#!/usr/bin/env python

"""
uptux by initstring (gitlab.com/initstring)

This tool checks for configuration issues on Linux systems that may lead to
privilege escalation.

All functionality is contained in a single file, because installing packages
in restricted shells is a pain.
"""

import os
import sys
import socket
import getpass
import argparse
import datetime
import subprocess
import inspect
import glob
import re

BANNER = r'''

                      ____ ___      ___________
                     |    |   \_____\__    ___/_ _____  ___
                     |    |   /\____ \|    | |  |  \  \/  /
                     |    |  / |  |_> >    | |  |  />    <
                     |______/  |   __/|____| |____//__/\_ \
                               |__|                      \/

                        PrivEsc for modern Linux systems
                          github.com/initstring/uptux

'''

########################## Global Declarations Follow #########################

LOGFILE = 'log-uptux-{:%Y-%m-%d-%H.%M.%S}'.format(datetime.datetime.now())
PARSER = argparse.ArgumentParser(description=
                                 "PrivEsc for modern Linux systems,"
                                 " by initstring (gitlab.com/initstring)")
PARSER.add_argument('-n', '--nologging', action='store_true',
                    help='do not write output to a logfile')
PARSER.add_argument('-d', '--debug', action='store_true',
                    help='print some debugging info to the console')
ARGS = PARSER.parse_args()

## Known directories for storing systemd files.
SYSTEMD_DIRS = ['/etc/systemd/**/',
                '/lib/systemd/**/',
                '/run/systemd/**/',
                '/usr/lib/systemd/**/']

## Known directories for storing D-Bus configuration files
DBUS_DIRS = ['/etc/dbus-1/system.d/',
             '/etc/dbus-1/session.d']

# Target files that we know we cannot exploit
NOT_VULN = ['/dev/null',
            '.']

# Used to enable/disable the relative path checks of systemd
SYSTEMD_PATH_WRITABLE = False
########################## End of Global Declarations #########################

############################ Setup Functions Follow ###########################

# This is the place for functions that help set up the application.

def tee(text, **kwargs):
    """Used to log and print concurrently"""

    # Defining variables to print color-coded messages to the console.
    colors = {'green': '\033[92m',
              'blue': '\033[94m',
              'orange': '\033[93m',
              'red': '\033[91m',}
    end_color = '\033[0m'
    boxes = {'ok': colors['blue'] + '[*] ' + end_color,
             'note': colors['green'] + '[+] ' + end_color,
             'warn': colors['orange'] + '[!] ' + end_color,
             'vuln': colors['red'] + '[VULNERABLE] ' + end_color,
             'sus': colors['orange'] + '[INVESTIGATE] ' + end_color}

    # If this function is called with an optional 'box=xxx' parameter, these
    # will be prepended to the message.
    box = kwargs.get('box', '')
    if box:
        box = boxes[box]

    # First, just print the item to the console.
    print(box + text)

    # Then, write it to the log if logging is not disabled
    if not ARGS.nologging:
        try:
            with open(LOGFILE, 'a') as logfile:
                logfile.write(box + text + '\n')
        except PermissionError:
            ARGS.nologging = True
            print(boxes['warn'] + "Could not create a log file due to"
                  " insufficient permissions. Continuing with checks...")

def check_handler(check, check_name, check_desc):
    """Check handler

    This function takes a dictionary of check_desc,check_name and will
    iterate through them all.
    """
    tee("\n\n++++++++++  {}: {}  ++++++++++\n\n"
        .format(check_name, check_desc))
    tee("Starting module at {:%Y-%m-%d-%H.%M.%S}"
        .format(datetime.datetime.now()), box='ok')
    tee("\n")
    check()
    tee("\n")
    tee("Finished module at {:%Y-%m-%d-%H.%M.%S}\n"
        .format(datetime.datetime.now()), box='ok')

def get_function_order(function):
    """Helper function for build_checks_list"""
    # Grabs the line number of a function it is passed.
    order = function.__code__.co_firstlineno
    return order

def build_checks_list():
    """Dynamically build list of checks to execute

    This function will grab, in order, all functions that start with
    'uptux_check_' and populate a list. This is then used to run the checks.
    """
    # Start to build a list of functions we will execute.
    uptux_checks = []

    # Get the name of this python script and all the functions inside it.
    current_module = sys.modules[__name__]
    all_functions = inspect.getmembers(current_module, inspect.isfunction)

    # If the function name matches 'uptux_check_' we will include it.
    for function in all_functions:
        function_name = function[0]
        function_object = function[1]
        if 'uptux_check_' in function_name:
            uptux_checks.append(function_object)

    # Use the helper function to sort by line number in script.
    uptux_checks.sort(key=get_function_order)

    # Return the sorted list of functions.
    return uptux_checks

############################# End Setup Functions #############################

########################### Helper Functions Follow ###########################

# This is the place to put functions that are used by multiple "Individual
# Checks" (those starting with uptux_check_).

def shell_exec(command):
    """Executes Linux shell commands"""
    # Split the command into a list as needed by subprocess
    command = command.split()

    # Get both stdout and stderror from command. Grab the Python exception
    # if there is one.
    try:
        out_bytes = subprocess.check_output(command,
                                            stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as error:
        out_bytes = error.output
    except OSError as error:
        print('Could not run the following OS command. Sorry!\n'
              '   Command: {}'.format(command))
        print(error)
        sys.exit()

    # Return the lot as a text string for processing.
    out_text = out_bytes.decode('utf-8')
    out_text = out_text.rstrip()
    return out_text

def find_system_files(**kwargs):
    """Locates system files

    Expected kwargs: known_dirs, search_mask
    """

    # Define known Linux folders for storing service unit definitions
    return_list = set()

    # Recursively gather all service unit from the known directories
    # and add them to a deduplicated set.
    for directory in kwargs['known_dirs']:
        found_files = glob.glob(directory + kwargs['search_mask'])
        for item in found_files:
            # We don't care about files that point to /dev/null.
            if '/dev/null' not in os.path.realpath(item):
                return_list.add(item)

    if ARGS.debug:
        print("DEBUG, FOUND FILES")
        for item in return_list:
            print(item)

    return return_list

def regex_vuln_search(**kwargs):
    """Helper function for searching text files

    This function will take a list of file paths and search through
    them with a given regex. Relevant messages will be printed to the console
    and log.

    Expected kwargs: file_paths, regex, message_text, message_box
    """
    # Start a list of dictionaries for files with interesting content.
    return_list = []

    # Open up each individual file and read the text into memory.
    for file_name in kwargs['file_paths']:
        return_dict = {}

        # Continue if we can't access the file.
        if not os.access(file_name, os.R_OK):
            continue

        file_object = open(file_name, 'r')
        file_text = file_object.read()
        # Use the regex we pass in to the function to look for vulns.
        found = re.findall(kwargs['regex'], file_text)

        # Save the file name and the interesting lines of text
        if found:
            return_dict['file_name'] = file_name
            return_dict['text'] = found
            return_list.append(return_dict)

    # If the function is supplied with message info, print to console and log.
    # This function may be used instead as input to another function, so we
    # don't always want to print here.
    if return_list and kwargs['message_text'] and kwargs['message_box']:
        # Print to console and log the interesting file names and content.
        tee("")
        tee(kwargs['message_text'], box=kwargs['message_box'])
        for item in return_list:
            tee("  {}:".format(item['file_name']))
            for text in item['text']:
                tee("    {}".format(text))
            tee("")

    if ARGS.debug:
        print("DEBUG, SEARCH RESULTS")
        for item in return_list:
            print(item['file_name'])
            for text in item['text']:
                print("  {}".format(text))

    return return_list

def check_file_permissions(**kwargs):
    """Helper function to check permissions and symlink status

    This function will take a list of file paths, resolve them to their
    actual location (for symlinks), and determine if they are writeable
    by the current user. Will also alert on broken symlinks and whether
    the target directory for the broken link is writeable.

    Expected kwargs: file_paths, files_message_text, dirs_message_text,
    message_box
    """
    # Start deuplicated sets for interesting files and directories.
    writeable_files = set()
    writeable_dirs = set()

    for file_name in kwargs['file_paths']:

        # Ignore known not-vulnerable targets
        if file_name in NOT_VULN:
            continue

        # Is it a symlink? If so, get the real path and check permissions.
        # If it is broken, check permissions on the parent directory.
        if os.path.islink(file_name):
            target = os.readlink(file_name)

            # Some symlinks use relative path names. Find these and prepend
            # the directory name so we can investigate properly.
            if target[0] == '.':
                parent_dir = os.path.dirname(file_name)
                target = parent_dir + '/' + target

            if os.path.exists(target) and os.access(target, os.W_OK):
                writeable_files.add('{} -- symlink --> {}'
                                    .format(file_name, target))
            else:
                parent_dir = os.path.dirname(target)
                if os.access(parent_dir, os.W_OK):
                    writeable_dirs.add((file_name, target))

        # OK, not a symlink. Just check permissions.
        else:
            if os.access(file_name, os.W_OK):
                writeable_files.add(file_name)

    if writeable_files:
        # Print to console and log the interesting findings.
        tee("")
        tee(kwargs['files_message_text'], box=kwargs['message_box'])
        for item in writeable_files:
            tee("  {}".format(item))

    if writeable_dirs:
        # Print to console and log the interesting findings.
        tee("")
        tee(kwargs['dirs_message_text'], box=kwargs['message_box'])
        for item in writeable_dirs:
            tee("  {} --> {}".format(item[0], item[1]))

    if not writeable_files and not writeable_dirs:
        tee("")
        tee("No writeable targets. This is expected...",
            box='note')

def check_command_permission(**kwargs):
    """Checks permissions on commands returned from inside files

    Loops through a provided list of dictionaries with a file name and
    commands found within. Checks to see if they are writeable or missing and
    living in a writable directory.

    Expected kwargs: file_paths, regex, message_text, message_box
    """
    # Start an empty list for the return of the writable files/commands
    return_list = []

    for item in kwargs['commands']:
        return_dict = {}
        return_dict['text'] = []

        # The commands we have are long and may include parameters or even
        # multiple commands with pipes and ;. We try to split this all out
        # below.
        for command in item['text']:
            command = re.sub(r'[\'"]', '', command)
            command = re.split(r'[ ;\|]', command)

            # We now have a list of some commands and some parameters and
            # other garbage. Checking for os access will clean this up for us.
            # The lines below determine if we have write access to anything.
            # It also checks for the case where the target does not exist  but
            # the parent directory is writeable.
            for split_command in command:
                vuln = False

                # Ignore known not-vulnerable targets
                if split_command in NOT_VULN:
                    continue

                # Some systemd items will specicify a command with a path
                # relative to the calling item, particularly timer files.
                relative_path = os.path.dirname(item['file_name'])

                # First, check the obvious - is this a writable command?
                if os.access(split_command, os.W_OK):
                    vuln = True

                # What about if we assume it is a relative path?
                elif os.access(relative_path + '/' + split_command, os.W_OK):
                    vuln = True

                # Or maybe it doesn't exist at all, but is in a writeable
                # director?
                elif (os.access(os.path.dirname(split_command), os.W_OK)
                      and not os.path.exists(split_command)):
                    vuln = True

                # If so, pack it all up in a new dictionary which is used
                # below for output.
                if vuln:
                    return_dict['file_name'] = item['file_name']
                    return_dict['text'].append(split_command)
                    if return_dict not in return_list:
                        return_list.append(return_dict)

    if return_list and kwargs['message_text'] and kwargs['message_box']:
        # Print to console and log the interesting file names and content.
        tee("")
        tee(kwargs['message_text'], box=kwargs['message_box'])
        for item in return_list:
            tee("  {}:".format(item['file_name']))
            for text in item['text']:
                tee("    {}".format(text))
            tee("")

########################## Helper Functions Complete ##########################

########################### Individual Checks Follow ##########################

# Note: naming a new function 'uptux_check_xxxx' will automatically
# include it in execution. These will trigger in the same order listed
# in the script. The docstring will be pulled and used in the console and
# log file, so keep it short (one line).

def uptux_check_sysinfo():
    """Gather basic OS information"""
    # Gather a few basics for the report.
    uname = os.uname()
    tee("Host: {}".format(uname[1]))
    tee("OS: {}, {}".format(uname[0], uname[3]))
    tee("Kernel: {}".format(uname[2]))
    tee("Current user: {} (UID {} GID {})".format(getpass.getuser(),
                                                  os.getuid(),
                                                  os.getgid()))
    tee("Member of following groups:\n  {}".format(shell_exec('groups')))

def uptux_check_systemd_paths():
    """Check if systemd PATH is writeable"""
    # Define the bash command.
    command = 'systemctl show-environment'
    output = shell_exec(command)

    # Define the regex to find in the output.
    regex = re.compile(r'PATH=(.*$)')

    # Take the output from bash and split it into a list of paths.
    output = re.findall(regex, output)

    # This command may fail in some environments, only proceed if we have
    # a good match.
    if output:
        output = output[0].split(':')

        # Check each path - if it is writable, add it to a list.
        writeable_paths = []
        for item in output:
            if os.access(item, os.W_OK):
                writeable_paths.append(item)
    else:
        writeable_paths = False

    # Write the status to the console and log.
    if writeable_paths:
        tee("The following systemd paths are writeable. THIS IS ODD!\n"
            "See if you can combine this with a relative path Exec statement"
            " for privesc:",
            box='vuln')
        for path in writeable_paths:
            tee("  {}".format(path))
        global SYSTEMD_PATH_WRITABLE
        SYSTEMD_PATH_WRITABLE = True
    else:
        tee("No systemd paths are writeable. This is expected...",
            box='note')

def uptux_check_services():
    """Inspect systemd service unit files"""
    # Define known Linux folders for storing service unit definitions
    units = set()
    mask = '*.service'
    units = find_system_files(known_dirs=SYSTEMD_DIRS,
                              search_mask=mask)

    tee("Found {} service units to analyse...\n".format(len(units)),
        box='ok')

    # Test for write access to any service files.
    # Will resolve symlinks to their target and also check for broken links.
    text = 'Found writeable service unit files:'
    text2 = 'Found writeable directories referred to by broken symlinks'
    box = 'vuln'
    tee("")
    tee("Checking permissions on service unit files...",
        box='ok')
    check_file_permissions(file_paths=units,
                           files_message_text=text,
                           dirs_message_text=text2,
                           message_box=box)

    # Only check relative paths if we can abuse them
    if SYSTEMD_PATH_WRITABLE:
        # Look for relative calls to binaries.
        # Example: ExecStart=somfolder/somebinary
        regex = re.compile(r'^Exec(?:Start|Stop|Reload)='
                           r'(?:@[^/]'    # special exec
                           r'|-[^/]'      # special exec
                           r'|\+[^/]'     # special exec
                           r'|![^/]'      # special exec
                           r'|!![^/]'     # special exec
                           r'|)'          # or maybe no special exec
                           r'[^/@\+!-]'   # not abs path or special exec
                           r'.*',         # rest of line
                           re.MULTILINE)
        text = ('Possible relative path in an Exec statement.\n'
                'Unless you have writeable systemd paths, you won\'t be able to'
                ' exploit this:')
        box = 'sus'
        tee("")
        tee("Checking for relative paths in service unit files [check 1]...",
            box='ok')
        regex_vuln_search(file_paths=units,
                          regex=regex,
                          message_text=text,
                          message_box=box)

        # Look for relative calls to binaries but invoked by an interpreter.
        # Example: ExecStart=/bin/sh -c 'somefolder/somebinary'
        regex = re.compile(r'^Exec(?:Start|Stop|Reload)='
                           r'(?:@[^/]'    # special exec
                           r'|-[^/]'      # special exec
                           r'|\+[^/]'     # special exec
                           r'|![^/]'      # special exec
                           r'|!![^/]'     # special exec
                           r'|)'          # or maybe no special exec
                           r'.*?(?:/bin/sh|/bin/bash) '   # interpreter
                           r'(?:[\'"]|)'  # might have quotes
                           r'(?:-[a-z]+|)'# might have params
                           r'(?:[ ]+|)'   # might have more spaces now
                           r'[^/-]'       # not abs path or param
                           r'.*',         # rest of line
                           re.MULTILINE)
        text = ('Possible relative path invoked with an interpreter in an'
                ' Exec statement.\n'
                'Unless you have writable systemd paths, you won\'t be able to'
                ' exploit this:')
        box = 'sus'
        tee("")
        tee("Checking for relative paths in service unit files [check 2]...",
            box='ok')
        regex_vuln_search(file_paths=units,
                          regex=regex,
                          message_text=text,
                          message_box=box)

    # Check for write access to any commands invoked by Exec statements.
    # Thhs regex below is used to extract command lines.
    regex = re.compile(r'^Exec.*?=[!@+-]*(.*?$)',
                       re.MULTILINE)
    # We don't pass message info to this function as we need to perform more
    # processing on the output to determine what is writeable.
    tee("")
    tee("Checking for write access to commands referenced in service files...",
        box='ok')
    service_commands = regex_vuln_search(file_paths=units,
                                         regex=regex,
                                         message_text='',
                                         message_box='')

    # Another helper function to take the extracted commands and check for
    # write permissions.
    text = 'You have write access to commands referred to in service files:'
    box = 'vuln'
    check_command_permission(commands=service_commands,
                             message_text=text,
                             message_box=box)

def uptux_check_timer_units():
    """Inspect systemd timer unit files"""
    units = set()
    mask = '*.timer'
    units = find_system_files(known_dirs=SYSTEMD_DIRS,
                              search_mask=mask)

    tee("Found {} timer units to analyse...\n".format(len(units)),
        box='ok')

    # Test for write access to any timer files.
    # Will resolve symlinks to their target and also check for broken links.
    text = 'Found writeable timer unit files:'
    text2 = 'Found writeable directories referred to by broken symlinks'
    box = 'vuln'
    tee("")
    tee("Checking permissions on timer unit files...",
        box='ok')
    check_file_permissions(file_paths=units,
                           files_message_text=text,
                           dirs_message_text=text2,
                           message_box=box)

    # Timers may reference systemd services, which are already being checked.
    # But they may reference a specific script (often a '.target' file of the
    # same name. Check to see if the action called is writable.
    # The regex below is used to extract these targets.
    regex = re.compile(r'^Unit=*(.*?$)',
                       re.MULTILINE)

    # We don't pass message info to this function as we need to perform more
    # processing on the output to determine what is writeable.
    tee("")
    tee("Checking for write access to commands referenced in timer files...",
        box='ok')
    timer_commands = regex_vuln_search(file_paths=units,
                                       regex=regex,
                                       message_text='',
                                       message_box='')

    # Another helper function to take the extracted commands and check for
    # write permissions.
    text = 'You have write access to commands referred to in timer files:'
    box = 'vuln'
    check_command_permission(commands=timer_commands,
                             message_text=text,
                             message_box=box)

def uptux_check_socket_units():
    """Inspect systemd socket unit files"""
    units = set()
    mask = '*.socket'
    units = find_system_files(known_dirs=SYSTEMD_DIRS,
                              search_mask=mask)

    tee("Found {} socket units to analyse...\n".format(len(units)),
        box='ok')

    # Test for write access to any socket files.
    # Will resolve symlinks to their target and also check for broken links.
    text = 'Found writeable socket unit files:'
    text2 = 'Found writeable directories referred to by broken symlinks'
    box = 'vuln'
    tee("")
    tee("Checking permissions on socket unit files...",
        box='ok')
    check_file_permissions(file_paths=units,
                           files_message_text=text,
                           dirs_message_text=text2,
                           message_box=box)

    # Check for write access to any socket files created by a service.
    # This can be interesting - I've seen a systemd service run a REST
    # API on a AF_UNIX socket. This would be missed by normal privesc
    # checks.
    # Thhs regex below is used to extract command lines.
    regex = re.compile(r'^Listen.*?=[!@+-]*(.*?$)',
                       re.MULTILINE)

   # We don't pass message info to this function as we need to perform more
    # processing on the output to determine what is writeable.
    tee("")
    tee("Checking for write access to AF_UNIX sockets...",
        box='ok')
    socket_files = regex_vuln_search(file_paths=units,
                                     regex=regex,
                                     message_text='',
                                     message_box='')

    # Another helper function to take the extracted commands and check for
    # write permissions.
    text = ('You have write access to AF_UNIX socket files invoked by a'
            ' systemd service.\n'
            'This could be interesting. \n'
            'You can attach to these files to look for an exploitable API.')
    box = 'sus'
    check_command_permission(commands=socket_files,
                             message_text=text,
                             message_box=box)

def uptux_check_socket_apis():
    """Look for web servers on UNIX domain sockets"""
    # Use Linux ss tool to find sockets in listening state
    command = 'ss -xlp -H state listening'
    output = shell_exec(command)

    # We get Unicode back from the above command, let's fix
    output = str(output)

    root_sockets = []
    socket_replies = {}

    # We want to grab all the strings that look like socket paths
    sockets = re.findall(r' (/.*?) ', output)
    abstract_sockets = re.findall(r' (@.*?) ', output)

    for socket_path in sockets:
        # For now, we are only interested in sockets owned by root
        if os.path.exists(socket_path) and os.stat(socket_path).st_uid == 0:
            root_sockets.append(socket_path)

    tee("Trying to connect to {} unix sockets owned by uid 0..."
        .format(len(root_sockets)), box='ok')
    tee("")

    # Cycle through each and try to send a raw HTTP GET
    for socket_target in root_sockets:

        # Define a raw HTTP GET request
        http_get = ('GET / HTTP/1.1\r\n'
                    'Host: localhost\r\n'
                    '\r\n\r\n')

        # Try to interact with the socket like a web API
        client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        client.settimeout(5)
        try:
            client.connect(socket_target)
            client.sendall(http_get.encode())

            reply = client.recv(8192).decode()

            # If we get a reply to this, we assume it is an API
            if reply:
                socket_replies[socket_target] = reply

        except (socket.error, UnicodeDecodeError):
            continue

    # If we have some replies, print to console
    # Hack-ish string replacement to get a nice indent
    if socket_replies:
        tee("The following root-owned sockets replied as follows",
            box='sus')
        for socket_path in socket_replies:
            tee("  " + socket_path + ":")
            tee("    " + socket_replies[socket_path]
                .replace('\n', '\n    '))
            tee("")

def uptux_check_dbus_issues():
    """Inspect D-Bus configuration items"""
    units = set()
    mask = '*.conf'
    units = find_system_files(known_dirs=DBUS_DIRS,
                              search_mask=mask)

    tee("Found {} D-Bus conf files to analyse...\n".format(len(units)),
        box='ok')

    # Test for write access to any files.
    # Will resolve symlinks to their target and also check for broken links.
    text = 'Found writeable D-Bus conf files:'
    text2 = 'Found writeable directories referred to by broken symlinks'
    box = 'vuln'
    tee("")
    tee("Checking permissions on D-Bus conf files...",
        box='ok')
    check_file_permissions(file_paths=units,
                           files_message_text=text,
                           dirs_message_text=text2,
                           message_box=box)

    # Checking for overly permission policies in D-Bus configuration files.
    # For example, normally "policy" is defined as a username. When defined in
    # an XML tag on its own, it applies to everyone.
    tee("")
    tee("Checking for overly permissive D-Bus configuration rules...",
        box='ok')

    regex = re.compile(r'<policy>.*?</policy>',
                       re.MULTILINE|re.DOTALL)

    text = ('These D-Bus policies may be overly permissive as they do not'
            ' specify a user or group.')
    box = 'sus'
    regex_vuln_search(file_paths=units,
                      regex=regex,
                      message_text=text,
                      message_box=box)

########################## Individual Checks Complete #########################

def main():
    """Main function"""
    print(BANNER)

    # Dynamically build list of checks to execute.
    uptux_checks = build_checks_list()

    # Use the handler to execute each check.
    for check in uptux_checks:
        check_name = check.__name__
        check_desc = check.__doc__
        check_handler(check, check_name, check_desc)

    # Good luck!
    tee("")
    tee("All done, good luck!", box='note')

if __name__ == "__main__":
    main()


# Katylub, Exercise 1
# Write a program that asks the user to input any positive integer and outputs the
# sum of all numbers between one and that number.

#Use positive number
n = int(input("Input a positive number: "))

#I have found the information doing research - www.w3resource.com
sum_num = (n * (n + 1)) / 2
print(sum_num)


import sys
sys.setrecursionlimit(10**7)

def minSumPath(grid):
    N = len(grid)
    M = len(grid[0])
    dp = [[-1 for i in range(M+1)] for i in range(N+1)]
    def help(i,j, c):
        if i== N-1 and j == M-1:return c + grid[i][j]
        if dp[i][j] != -1: return dp[i][j]
        r = help(i,j+1, c) if j<M-1 else float('inf')
        d = help(i+1,j, c) if i<N-1 else float('inf')
        dp[i][j] = min(r,d) + grid[i][j]
        return dp[i][j]
    return help(0,0,0)

# follow-up
# try to solve using real dp approach

# I didn't comeup with this solution, but it is best beginners solution ever
class Solution:
    def minPathSum(self, grid):
        n = len(grid)
        m = len(grid[0])
        for i in range(1,n):
            grid[i][0] = grid[i-1][0] + grid[i][0]
        for j in range(1, m):
            grid[0][j] = grid[0][j-1] + grid[0][j]

        for i in range(1,n):
            for j in range(1, m):
                grid[i][j] = grid[i][j] + min(grid[i-1][j], grid[i][j-1])
        return grid[n-1][m-1]


from IntProgramModule.OpcodeProgram import OpcodeProgram

# Day 2 - IntProgramParser
class IntProgramParser:

    def __init__(self, initialInstructionPointerValue, opcodeProgramSpanDict):
        if isinstance(initialInstructionPointerValue, int) == False:
            raise TypeError('ERROR :: "initialInstructionPointerValue" must be of type "Int"!')
        if initialInstructionPointerValue < 0:
            raise ValueError('ERROR :: "initialInstructionPointerValue" must be a positive number!')
        if isinstance(opcodeProgramSpanDict, dict) == False:
            raise TypeError('ERROR :: "opcodeProgramSpanDict" must be of type "dict"!')
        if len(opcodeProgramSpanDict) == 0:
            raise ValueError('ERROR :: "opcodeProgramSpanDict" dictionary is EMPTY!')

        self.instructionPointer = initialInstructionPointerValue
        self.opcodeProgramSpanDict = opcodeProgramSpanDict

    def Parse(self, opcodes : []):

        currentOpcode = opcodes[self.instructionPointer]
        opcodeProgramSpan = self.opcodeProgramSpanDict[currentOpcode]

        startLocation = self.instructionPointer
        stopLocation = startLocation + opcodeProgramSpan
        stepSize = 1

        rawOpCodeProgram = opcodes[startLocation : stopLocation : stepSize]

        if len(rawOpCodeProgram) < opcodeProgramSpan:
            parameter_Opcode = 99
            parameter_Params = []

            opcodeProgram = OpcodeProgram(
                parameter_Opcode,
                parameter_Params
            )
        else:
            opcodePosition = 0
            paramStartPosition = 1
            paramEndPosition = len(rawOpCodeProgram) - 1

            parameter_Opcode = rawOpCodeProgram[opcodePosition]
            parameter_Params = rawOpCodeProgram[paramStartPosition : len(rawOpCodeProgram) : stepSize]

            opcodeProgram = OpcodeProgram(
                parameter_Opcode,
                parameter_Params
            )

            self.instructionPointer += opcodeProgramSpan

        return opcodeProgram


#!/usr/bin/env python3
import sys
import socket
from threading import Thread, Lock
from struct import pack, unpack
from http.server import BaseHTTPRequestHandler, HTTPServer

NO_TYPE = 0
GPIO_26_OFF = 1
GPIO_26_ON = 2
GPIO_27_OFF = 3
GPIO_27_ON = 4

HEADER_LENGTH = 4

TESTING = False

device_connection = None
http_port = 8087

class Message:
    def __init__(self, message):
        self.type = None
        self.type = unpack('I', message)[0]
        self.type = socket.ntohl(self.type)

    def construct_message(self):
        message = pack('I', socket.htonl(self.type))
        return message

class WebServerDriver(object):
    def __init__(self, device_port_number, vlan_id):
        self.device_connected = False
        self.device_connection = None
        self.device_socket = None
        self.device_socket = socket.socket(
            socket.AF_INET, socket.SOCK_STREAM)
        self.device_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        # self.device_socket.bind(('0.0.0.0', 8188))
        self.device_socket.bind((f'192.168.{vlan_id}.1', device_port_number))
        self.device_socket.listen(1)
        self.device_socket_listener_thread = Thread(target=self.accept_device_connection, args=())
        self.device_socket_listener_thread.start()

    # For derived class to overwrite
    def run_http_server(self, http_port, dev_conn):
        raise NotImplementedError()

    def accept_device_connection(self):
        # while True:
        (conn, address) = self.device_socket.accept()
        self.device_connection = conn
        self.run_http_server(http_port, conn)


#-*- coding:utf-8 -*-
# utilities used by QR code generator

# Galois Field exp table and log table
GF_EXP = [1] * 512
GF_LOG = [0] * 256

def trans_to_binary(num, bits):
    """Transform int valuable to binary string

    Args:
        num: int valuable, e.g. 10
        bits: the number of bits of the binary string, e.g. 4

    Returns:
        A string that represent the binary value of `num`, e.g. '1010'

    """
    binary = bin(num)[2:]
    if len(binary) < bits:
        length = bits - len(binary)
        for i in range(length):
            binary = '0' + binary
    elif len(binary) > bits:
        binary = binary[len(binary) - bits:]
    return binary

def trans_bin_to_int(binary):
    """Transform binary string to list of demical

    Every 8-bit binary number take as a demical value,
    transform to lists of demical.

    Args:
        binary: the binary string whose length must be divisible by 8
          e.g. '01001010'

    Returns:
        The list of demical value, e.g. [4, 10]

    """
    dem_list = []
    for i in xrange(0, len(binary) / 8):
        dem_list.append(int(binary[i * 8: (i + 1) * 8], 2))
    return dem_list

def xor(bin1, bin2):
    """Xor operation for every bit int bin1 and bin2

    Args:
        bin1: the first oprand(binary string), e.g. '1010'
        bin2: the second oprand, the two string must have the same length
          e.g. '0011'

    Returns:
        The result string, e.g. '1001'

    """
    n = len(bin1)
    dem1 = int(bin1, 2)
    dem2 = int(bin2, 2)
    result = dem1 ^ dem2
    return trans_to_binary(result, n)

def RS_encode(msg_in, nsym):
    """Generate Reed Solemn correction code

        Args:
            msg_in: the data list, every element is either 0 or 1
            nsym: the length of required RS correction code

        Returns:
            A binary list which contain the origin data and the
              correction code
    """
    gen = rs_generator_poly(nsym)
    msg_out = [0] * (len(msg_in) + nsym)
    for i in range(0, len(msg_in)):
        msg_out[i] = msg_in[i]
    for i in range(0, len(msg_in)):
        coef = msg_out[i]
        if coef != 0:
            for j in range(0, len(gen)):
                msg_out[i+j] ^= gf_mul(gen[j], coef)
    for i in range(0, len(msg_in)):
        msg_out[i] = msg_in[i]
    return msg_out

def BCH_encode(data, nsym):
    """Generate BCH correction code

        Args:
            msg_in: the binary string
            nsym: the length of required BCH correction code

        Returns:
            A binary string which contain the origin data and the
              correction code
    """
    gen = [1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1]
    msg_in = [0] * len(data)
    msg_out = [0] * (len(data) + nsym)
    for i in range(0, len(data)):
        msg_in[i] = ord(data[i]) - ord('0')
        msg_out[i] = msg_in[i]
    for i in range(0, len(msg_in)):
        coef = msg_out[i]
        if coef != 0:
            for j in range(0, len(gen)):
                msg_out[i+j] ^= gf_mul(gen[j], coef)
    for i in range(0, len(msg_in)):
        msg_out[i] = msg_in[i]
    # print msg_out
    bin_encode = ''
    for msg in msg_out:
        bin_encode += trans_to_binary(msg, 1)
    return bin_encode

def generate_gf_table():
    x = 1
    for i in range(1, 255):
        x <<= 1
        if x & 0x100:
            x ^= 0x11d
        GF_EXP[i] = x
        GF_LOG[x] = i
    for i in range(255, 512):
        GF_EXP[i] = GF_EXP[i-255]

def gf_mul(x, y):
    if x == 0 or y == 0:
        return 0
    return GF_EXP[GF_LOG[x] + GF_LOG[y]]

def gf_poly_mul(p, q):
    r = [0] * (len(p)+len(q)-1)
    for j in range(0, len(q)):
        for i in range(0, len(p)):
            r[i+j] ^= gf_mul(p[i], q[j])
    return r

def rs_generator_poly(nsym):
    g = [1]
    for i in range(0, nsym):
        g = gf_poly_mul(g, [1, GF_EXP[i]])
    return g


"""
Unit test for the CDK constructs
"""
import pytest
from aws_cdk import (
    core as cdk,
    aws_ec2 as ec2,
)

import constants
import collections
from src.forensic_auto_capture.auto_forensics import AutoForensicsConstruct
StackEnvironment = collections.namedtuple('StackEnvironment', ['stack', 'vpc'])

"""
Testing the Auto Forensics
"""

@pytest.fixture(scope="session")
def stack_environment():
    app = cdk.App()
    stack = cdk.Stack(app, "TestDeploySimulation", env=cdk.Environment(account=constants.ACCOUNT_ID,
                                                                       region=constants.REGION))
    vpc = ec2.Vpc.from_lookup(stack, "TestVPC", vpc_name=constants.VPC_NAME)
    yield StackEnvironment(stack=stack, vpc=vpc)
    app.synth()

@pytest.fixture(scope="session")
def auto_forensics_construct(stack_environment):
    return AutoForensicsConstruct(stack_environment.stack, "UNITAutoForensics",
                                  member_account_id=constants.ACCOUNT_ID,
                                  security_account_id=constants.ACCOUNT_ID,
                                  region=constants.REGION,
                                  vpc=stack_environment.vpc,
                                  supported_azs=constants.SUPPORTED_AZS)

"""
Testing the forensics_construct
"""

@pytest.fixture(scope="session")
def machine_image():
   return ec2.LookupMachineImage(name=constants.MACHINE_LOOKUP_NAME, owners=[constants.IMAGE_ACCOUNT])

@pytest.mark.order(1)
def test_forensic_image(auto_forensics_construct, machine_image):
    auto_forensics_construct.forensic_image = machine_image
    assert True

@pytest.mark.order(2)
def test_forensic_resource(auto_forensics_construct):
    auto_forensics_construct.build_forensic_resource()
    assert True

@pytest.mark.order(3)
def test_build_functions(auto_forensics_construct):
    auto_forensics_construct.build_functions()
    assert True

@pytest.mark.order(4)
def test_build_step_function(auto_forensics_construct):
    auto_forensics_construct.build_step_function()
    assert True

@pytest.mark.order(5)
def test_build_invoke(auto_forensics_construct, machine_image):
    auto_forensics_construct.build_invoke()
    auto_forensics_construct.invoke_construct.invoke_from_securityhub()
    assert True


class Matrix:
    def __init__(self, values):
        valideMatrix = True
        self.values = []
        self.shape = (-1, -1)
        shape0 = -1
        shape1 = -1
        if isinstance(values, tuple):
            if len(values) == 2:
                if isinstance(values[0], int) and isinstance(values[1], int):
                    if values[0] > 1 and values[1] > 1:
                        self.values = Matrix.init_list(values[0], values[1])
                        self.shape = values
        elif isinstance(values, list):
            shape0 = len(values)
            for float_or_list in values:
                if isinstance(float_or_list, list)\
                    and len(float_or_list) > 1:
                    if shape1 == -1:
                        shape1 = len(float_or_list)
                    elif shape1 != len(float_or_list):
                        valideMatrix = False
                    for elem in float_or_list:
                        if not isinstance(elem, float):
                            valideMatrix = False
                else:
                    valideMatrix = False
        else:
            valideMatrix = False

        if valideMatrix:

            self.values = values
            self.shape = (shape0, shape1)
        if len(self.values) < 1:
            print("wrong data format")
            quit()
    @staticmethod
    def is_v(first, second):
        if second is None:
            if isinstance(first, Vector):
                return True
            return False
        elif isinstance(first, Vector) or isinstance(second, Vector) :
            return True
        return False

    def T(self):
        ret = Matrix.init_list(self.shape[1], self.shape[0])
        print(ret)
        if self.shape[1] == 1:
            for x in range(0, self.shape[0]):
                ret[x] = self.values[x][0]
        else:
            for x in range(0, self.shape[1]):
                for y in range(0, self.shape[0]):
                    ret[x][y] = self.values[x]
        if Matrix.is_v(self, None):
            return Vector(ret)
        else:
            return Matrix(ret)

    def dot(self, other):
        if self.shape != other.shape:
            print('ShapeError("dot : only vectors of same dimensions.")')
            return None
        ret = 0
        for x in range(0, self.shape[1]):
            if self.shape[0] == 1:
                ret += self.values[x] * other.values[x]
            else:
                for y in range(0, self.shape[0]):
                    ret += (self.values[y][x] * other.values[y][x])
        return ret
    @staticmethod
    def init_list(x, y):
        ret = []
        if x == 1:
            ret = [0.0] * y
        else:
            for i in range(x):
                ret.append([0.0] * y)

        return ret

    def __add__(self, other):
        if self.shape != other.shape:
            print('ShapeError("add : only vectors of same dimensions.")')
            return None
        ret = Matrix.init_list(self.shape[0], self.shape[1])

        for x in range(0, self.shape[1]):
            if self.shape[0] == 1:
                ret[x] = self.values[x] + other.values[x]
            else:
                for y in range(0, self.shape[0]):
                    ret[y][x] = (self.values[y][x] + other.values[y][x])
        if Matrix.is_v(self, other):
            return Vector(ret)
        else:
            return Matrix(ret)

    def __radd__(self, other):
        return self.__add__(other)

    def __sub__(self, other):
        if self.shape != other.shape:
            print('ShapeError("add : only vectors of same dimensions.")')
            return None
        ret = Matrix.init_list(self.shape[0], self.shape[1])

        for x in range(0, self.shape[1]):
            if self.shape[0] == 1:
                ret[x] = self.values[x] - other.values[x]
            else:
                for y in range(0, self.shape[0]):
                    ret[y][x] = (self.values[y][x] - other.values[y][x])
        if Matrix.is_v(self, other):
            return Vector(ret)
        else:
            return Matrix(ret)

    def __rsub__(self, other):
        return self.__sub__(other)

    def __truediv__(self, scalar):
        if scalar == 0:
            print('ValueError("cant div by 0".)')
            return None
        ret = Matrix.init_list(self.shape[0], self.shape[1])
        for x in range(0, self.shape[1]):
            if self.shape[0] == 1:
                ret[x] = self.values[x] / scalar
            else:
                for y in range(0, self.shape[0]):
                    ret[y][x] = self.values[y][x] / scalar
        if Matrix.is_v(self, None):
            return Vector(ret)
        else:
            return Matrix(ret)

    def __rtruediv__(self, scalar):
        print('ValueError("A scalar cannot be divided by a Vector".)')
        return None

    def __mul__(self, other):
        ret = Matrix.init_list(self.shape[0], self.shape[1])
        for x in range(0, self.shape[1]):
            if self.shape[0] == 1:
                ret[x] = self.values[x] * scalar
            else:
                for y in range(0, self.shape[0]):
                    ret[y][x] = self.values[y][x] * scalar
        if Matrix.is_v(self, None):
            return Vector(ret)
        else:
            return Matrix(ret)

    def __repr__(self):
        return "\n" + str(self.values) + \
            "\n" + str(self.shape)

    def __str__(self):
        return "\nmatrix: " + str(self.values) + \
            "\nshape: " + str(self.shape)

class Vector(Matrix):
    def __init__(self, values):
        valideRow = True
        valideColumn = True

        length = 0
        self.values = []
        self.shape = (0, 0)
        if isinstance(values, tuple):
            if len(values) == 2:
                if isinstance(values[0], int) and isinstance(values[1], int):
                    if values[0] != values[1]:
                        for i in range(values[0], values[1]):
                            self.values.append([float(i)])
                        self.shape = (len(self.values), 1)

        elif isinstance(values, int):
            if values > 0:
                for i in range(0, values):
                    self.values.append([float(i)])
                self.shape = (values, 1)

        elif isinstance(values, list):
            length = len(values)
            if length > 0:
                for float_or_list in values:
                    if valideRow:
                        if not isinstance(float_or_list, float):
                            valideRow = False
                    if valideColumn:
                        if not isinstance(float_or_list, list):
                            valideColumn = False
                        else:
                            for elem in float_or_list:
                                if valideColumn:
                                    if not isinstance(elem, float):
                                        valideColumn = False
            if valideRow:
                self.values = values
                self.shape = (1, length)
            if valideColumn:
                self.values = values
                self.shape = (length, len(values[0]))
            if len(self.values) < 1:
                print("wrong data format")
                quit()

    def __mul__(self, scalar):
        ret = Matrix.init_list(self.shape[0], self.shape[1])
        for x in range(0, self.shape[1]):
            if self.shape[0] == 1:
                ret[x] = self.values[x] * scalar
            else:
                for y in range(0, self.shape[0]):
                    ret[y][x] = self.values[y][x] * scalar
        if Matrix.is_v(self, None):
            return Vector(ret)
        else:
            return Matrix(ret)

    def __rmul__(self, scalar):
        print('ValueError("A scalar cannot be multiplied by a Vector".)')
        return None

    def __repr__(self):
        return "\n" + str(self.values) + \
            "\n" + str(self.shape)

    def __str__(self):
        return "\nvector: " + str(self.values) + \
            "\nshape: " + str(self.shape)


# Licensed under a 3-clause BSD style license - see LICENSE.rst

from astropy import log
from astropy.time import Time

from sofia_redux.scan.info.astrometry import AstrometryInfo
from sofia_redux.scan.utilities import utils
from sofia_redux.scan.coordinate_systems.epoch.epoch import Epoch, J2000
from sofia_redux.scan.coordinate_systems.equatorial_coordinates import \
    EquatorialCoordinates
from sofia_redux.scan.coordinate_systems.horizontal_coordinates import \
    HorizontalCoordinates
from sofia_redux.scan.coordinate_systems.geodetic_coordinates import \
    GeodeticCoordinates

__all__ = ['SofiaAstrometryInfo']

class SofiaAstrometryInfo(AstrometryInfo):

    default_fits_date = "1970-01-01T00:00:00.0"

    def __init__(self):
        """
        Initialize astrometry information for SOFIA observations.

        The SOFIA astrometry information should be relevant to all instruments
        that may be mounted on SOFIA.  All equatorial observations use the
        J2000 equinox.  Additionally, SOFIA is considered a ground-based
        telescope since it still encounters atmospheric effects.
        """
        super().__init__()
        self.date = None
        self.start_time = None
        self.file_date = None
        self.epoch = J2000
        self.requested_equatorial = None
        self.horizontal = None
        self.equatorial = None
        self.ground_based = True

    def apply_configuration(self):
        """
        Update astrometry information with the FITS header configuration data.

        Returns
        -------
        None
        """
        if self.options is None:
            return
        self.parse_time()
        self.parse_astrometry()

    def parse_time(self):
        """
        Parse and apply time related keywords from the FITS header.

        The following FITS header keywords are used::

          KEY       DESCRIPTION
          --------  -----------
          DATE      Determines the file date (ISOT format, UTC scale)
          DATE-OBS  The exposure start time (ISOT format, UTC scale)
          UTCSTART  The UTC of the exposure start (hh:mm:ss.### UTC format)
          UTCEND    The UTC of the exposure end (hh:mm:ss.### UTC format)

        The modified Julian date (MJD) is calculated, and precessions are
        initialized relative to the J2000 equinox.

        Returns
        -------
        None
        """
        options = self.options
        if options is None:
            return
        self.file_date = options.get_string("DATE")
        self.date = options.get_string("DATE-OBS",
                                       default=self.default_fits_date)

        start_time = options.get_string("UTCSTART")
        if start_time is not None:
            self.utc.start = utils.parse_time(start_time)

        end_time = utils.get_string(options.get("UTCEND"))
        if end_time is not None:
            self.utc.end = utils.parse_time(end_time)

        t_ind = self.date.find('T')
        if t_ind != -1:
            self.time_stamp = self.date
            self.date = self.time_stamp[:t_ind]
            start_time = self.time_stamp[(t_ind + 1):]
        elif start_time is not None:
            self.time_stamp = self.date + 'T' + start_time
        else:
            self.time_stamp = self.date

        self.start_time = start_time
        self.set_mjd(Time(self.time_stamp, format='isot', scale='utc').mjd)
        self.calculate_precessions(J2000)

    def parse_astrometry(self):
        """
        Parse and apply coordinate related keywords from the FITS header.

        The following FITS header keywords are used::

          KEY       DESCRIPTION
          --------  -----------
          EQUINOX   The equinox of the observation (year)
          OBSRA     The right-ascension of the observed source (hours)
          OBSDEC    The declination of the observed source (degrees)

        The modified Julian date (MJD) is calculated, and precessions are
        initialized relative to the J2000 equinox.

        Returns
        -------
        None
        """
        options = self.options
        if options is None:
            return
        if "EQUINOX" in options:
            self.epoch = Epoch.get_epoch(options.get_float("EQUINOX"))

        if "OBSRA" in options and "OBSDEC" in options:
            self.requested_equatorial = EquatorialCoordinates(epoch=self.epoch)
            self.requested_equatorial.ra = options.get_hms_time(
                'OBSRA', angle=True)
            self.requested_equatorial.dec = options.get_dms_angle('OBSDEC')

        if self.is_requested_valid():
            self.equatorial = self.requested_equatorial.copy()
            self.calculate_precessions(self.equatorial.epoch)
        else:
            log.warning("No valid OBSRA/OBSDEC in header.")
            self.requested_equatorial = None
            self.equatorial = self.guess_reference_coordinates()
            self.calculate_precessions("J2000")

    def is_requested_valid(self, header=None):
        """
        Check whether the requested OBSRA/DEC values in the header are valid.

        Parameters
        ----------
        header : astropy.io.fits.header.Header, optional
            The header to read.  If not supplied, the FITS header for the scan
            will be read instead.

        Returns
        -------
        valid : bool
        """
        if header is None:
            if self.options is None:
                return False
            ra = self.options.get_float('OBSRA')
            dec = self.options.get_float('OBSDEC')
        else:
            ra = header.get('OBSRA')
            dec = header.get('OBSDEC')

        if not self.valid_header_value(ra):
            return False
        elif not self.valid_header_value(dec):
            return False
        elif ra == 0 and dec == 0:
            return False
        else:
            return True

    @classmethod
    def coordinate_valid(cls, coordinate):
        """
        Check whether a coordinate is valid.

        Parameters
        ----------
        coordinate : Coordinate2D

        Returns
        -------
        valid : bool
        """
        if coordinate is None:
            return False
        elif not cls.valid_header_value(coordinate.x.value):
            return False
        elif not cls.valid_header_value(coordinate.y.value):
            return False
        else:
            return True

    def guess_reference_coordinates(self, header=None, telescope=None):
        """
        Guess the reference coordinates of a scan.

        Parameters
        ----------
        header : astropy.io.fits.Header, optional
            The header to read.  The default is to read stored OBSRA/OBSDEC
            values in the configuration.
        telescope : SofiaTelescopeInfo, optional
            A telescope object to extract the boresight equatorial coordinates
            if all other avenues to the coordinates failed.

        Returns
        -------
        coordinates : EquatorialCoordinates
        """
        if self.coordinate_valid(self.object_coordinates):
            log.debug("Referencing scan to object coordinates OBJRA/OBJDEC.")
            return self.object_coordinates.copy()

        elif self.is_requested_valid(header=header):
            log.debug("Referencing scan to requested coordinates.")
            if self.requested_equatorial is not None:
                return self.requested_equatorial.copy()
            else:
                return EquatorialCoordinates(
                    [header['OBSRA'], header['OBSDEC']],
                    epoch=header.get('EQUINOX', 'J2000'))

        elif telescope is not None and self.coordinate_valid(
                telescope.boresight_equatorial):
            log.debug(
                "Referencing scan to initial telescope boresight TELRA/TELDEC")
            return telescope.boresight_equatorial.copy()

        else:
            log.warning("Referencing scan to initial scan position.")
            return None

    def validate_astrometry(self, scan):
        """
        Validate astrometry information from a scan.

        Returns
        -------
        None
        """
        if scan is None:
            return
        first_integration = scan.get_first_integration()
        if first_integration is None:
            raise ValueError("No integrations exist for scan.")
        last_integration = scan.get_last_integration()
        first = first_integration.get_first_frame_index()
        last = last_integration.get_last_frame_index()

        if self.is_nonsidereal:
            oe0 = first_integration.frames.object_equatorial[first]
            oe1 = last_integration.frames.object_equatorial[last]
            self.object_coordinates = EquatorialCoordinates(epoch=oe0.epoch)
            self.object_coordinates.ra = 0.5 * (oe0.ra + oe1.ra)
            self.object_coordinates.dec = 0.5 * (oe0.dec + oe1.dec)
            self.equatorial = self.object_coordinates.copy()

        h0 = first_integration.frames.horizontal[first]
        h1 = last_integration.frames.horizontal[last]
        self.horizontal = HorizontalCoordinates()
        self.horizontal.az = 0.5 * (h0.x + h1.x)
        self.horizontal.el = 0.5 * (h0.y + h1.y)

        s0 = first_integration.frames.site[first]
        s1 = last_integration.frames.site[last]
        self.site = GeodeticCoordinates()
        self.site.longitude = 0.5 * (s0.x + s1.x)
        self.site.latitude = 0.5 * (s0.y + s1.y)
        log.info(f"Location: {self.site}")

        log.info(f"Mean telescope VPA is {scan.get_telescope_vpa():.6f}")


import math

first_pair = int(input())
second_pair = int(input())
first_diff = int(input())
second_diff = int(input())

for first in range(first_pair, first_pair + first_diff + 1):
    for second in range(second_pair, second_pair + second_diff + 1):
        first_sqrt = int(math.floor(math.sqrt(first)))
        second_sqrt = int(math.floor(math.sqrt(second)))
        is_prime1 = True
        is_prime2 = True

        for first1 in range(2, first_sqrt + 1):
            if first % first1 == 0:
                is_prime1 = False
        for second2 in range(2, second_sqrt + 1):
            if second % second2 == 0:
                is_prime2 = False
        if is_prime1 == True and is_prime2 == True:
            print(f"{first}{second}")

7.Nested Loops - More Exercises/13. Prime Pairs.py


from oslo_serialization import jsonutils as json

from tempest.lib.common import rest_client

class CredentialsClient(rest_client.RestClient):
    api_version = "v3"

    def create_credential(self, **kwargs):
        """Creates a credential.

        Available params: see http://developer.openstack.org/
                              api-ref-identity-v3.html#createCredential
        """
        post_body = json.dumps({'credential': kwargs})
        resp, body = self.post('credentials', post_body)
        self.expected_success(201, resp.status)
        body = json.loads(body)
        body['credential']['blob'] = json.loads(body['credential']['blob'])
        return rest_client.ResponseBody(resp, body)

    def update_credential(self, credential_id, **kwargs):
        """Updates a credential.

        Available params: see http://developer.openstack.org/
                              api-ref-identity-v3.html#updateCredential
        """
        post_body = json.dumps({'credential': kwargs})
        resp, body = self.patch('credentials/%s' % credential_id, post_body)
        self.expected_success(200, resp.status)
        body = json.loads(body)
        body['credential']['blob'] = json.loads(body['credential']['blob'])
        return rest_client.ResponseBody(resp, body)

    def show_credential(self, credential_id):
        """To GET Details of a credential."""
        resp, body = self.get('credentials/%s' % credential_id)
        self.expected_success(200, resp.status)
        body = json.loads(body)
        body['credential']['blob'] = json.loads(body['credential']['blob'])
        return rest_client.ResponseBody(resp, body)

    def list_credentials(self):
        """Lists out all the available credentials."""
        resp, body = self.get('credentials')
        self.expected_success(200, resp.status)
        body = json.loads(body)
        return rest_client.ResponseBody(resp, body)

    def delete_credential(self, credential_id):
        """Deletes a credential."""
        resp, body = self.delete('credentials/%s' % credential_id)
        self.expected_success(204, resp.status)
        return rest_client.ResponseBody(resp, body)


import torch
from torch import nn
import torch.nn.functional as F

def conv_params(in_size, out_size):
    filters = [3, 2, 5, 4]
    strides = [1, 2, 3]  # max_stride = 3
    pads = [0, 1, 2, 3]  # max pad

    if out_size == 1:
        return 1, 0, in_size

    for filter_size in filters:
        for pad in pads:
            for stride in strides:
                if ((out_size - 1) * stride == (in_size - filter_size) + 2 * pad):
                    return stride, pad, filter_size
    return None, None, None

class StdConv(nn.Module):
    def __init__(self, nin, nout, filter_size=3, stride=2, padding=1, drop=0.1):
        super().__init__()
        self.conv = nn.Conv2d(nin, nout, filter_size, stride=stride, padding=padding)
        self.bn = nn.BatchNorm2d(nout)
        self.drop = nn.Dropout(drop)

    def forward(self, x):
        return self.drop(self.bn(F.relu(self.conv(x))))

def flatten_conv(x, k):
    bs, nf, gx, gy = x.size()
    x = x.permute(0, 2, 3, 1).contiguous()
    return x.view(bs, -1, nf // k)

class OutConv(nn.Module):
    def __init__(self, k, nin, num_classes, bias):
        super().__init__()
        self.k = k
        self.oconv1 = nn.Conv2d(nin, (num_classes) * k, 3, padding=1)
        self.oconv2 = nn.Conv2d(nin, 4 * k, 3, padding=1)
        self.oconv1.bias.data.zero_().add_(bias)

    def forward(self, x):
        return [flatten_conv(self.oconv1(x), self.k),
                flatten_conv(self.oconv2(x), self.k)]

class SSDHead(nn.Module):
    def __init__(self, grids, anchors_per_cell, num_classes, drop=0.3, bias=-4.):
        super().__init__()
        self.drop = nn.Dropout(drop)

        self.sconvs = nn.ModuleList([])
        self.oconvs = nn.ModuleList([])

        self.anc_grids = grids

        self._k = anchors_per_cell

        self.sconvs.append(StdConv(512, 256, stride=1, drop=drop))

        for i in range(len(grids)):

            if i == 0:
                stride, pad, filter_size = conv_params(7, grids[i])  # get '7' by base model
            else:
                stride, pad, filter_size = conv_params(grids[i - 1], grids[i])

            if stride is None:
                print(grids[i - 1], ' --> ', grids[i])
                raise Exception('cannot create model for specified grids')

            self.sconvs.append(StdConv(256, 256, filter_size, stride=stride, padding=pad, drop=drop))
            self.oconvs.append(OutConv(self._k, 256, num_classes=num_classes, bias=bias))

    def forward(self, x):
        x = self.drop(F.relu(x))
        x = self.sconvs[0](x)
        out_classes = []
        out_bboxes = []
        for sconv, oconv in zip(self.sconvs[1:], self.oconvs):
            x = sconv(x)
            out_class, out_bbox = oconv(x)
            out_classes.append(out_class)
            out_bboxes.append(out_bbox)

        return [torch.cat(out_classes, dim=1),
                torch.cat(out_bboxes, dim=1)]

def one_hot_embedding(labels, num_classes):
    return torch.eye(num_classes)[labels.data.cpu()]

class BCE_Loss(nn.Module):
    def __init__(self, num_classes):
        super().__init__()
        self.num_classes = num_classes

    def forward(self, pred, targ):
        t = one_hot_embedding(targ, self.num_classes)
        t = torch.Tensor(t[:, 1:].contiguous()).cuda()
        x = pred[:, 1:]
        w = self.get_weight(x, t)
        return F.binary_cross_entropy_with_logits(x, t, w, size_average=False) / (self.num_classes - 1)

    def get_weight(self, x, t): return None

class FocalLoss(BCE_Loss):
    def get_weight(self, x, t):
        alpha, gamma = 0.25, 1
        p = x.sigmoid()
        pt = p * t + (1 - p) * (1 - t)
        w = alpha * t + (1 - alpha) * (1 - t)
        w = w * (1 - pt).pow(gamma)
        return w.detach()


# This code is for the analysis of the citizen reports and the DCGAN output. There are two parts to it, and you might not want to run it all.
# From line 171 onwards, read the comments and comment out what you do not want to do.
# First you can specify a date and look at the results for that date, this can be done next for a morning/afternoon resolution for a particular month
#
# Second, the analysis is done at a morning/afternoon resolution for a sightly extended time range. In this case, it's a collection of days that are known to visually align well.
# Adjust the days of interest in the code.
import numpy as np
import geopandas as gpd
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import sys, os
import shapely
from shapely.geometry import Point
sns.set_style("whitegrid")
sns.set_context("talk", font_scale=1.75)
from tqdm import tqdm
import h5py
import torch
import matplotlib.colors as mcolors
import pickle

def plot_grids(grid, cr_grid, date, measure):
 fig , ax = plt.subplots(1,2, figsize = (15,15), sharex=True, sharey=True)
 # plot the two grids
 ax[0] = grid.plot(column='emission', ax = ax[0], cmap = 'rocket_r')
 ax[0].set_title('DCGAN emissions results')
 sm = plt.cm.ScalarMappable(cmap = 'rocket_r')
 sm.set_array(grid['emission'])
 cbar = plt.colorbar(sm, ax=ax[0], shrink=0.4)

 ax[1]= cr_grid.plot(column='smell value', ax=ax[1], cmap = 'rocket_r')
 ax[1].set_title('Citizen reports')
 sm2 = plt.cm.ScalarMappable(cmap = 'rocket_r')
 sm2.set_array(cr_grid['smell value'])
 cbar = plt.colorbar(sm2, ax=ax[1], shrink=0.4)
 plt.show()
 fig.savefig(f'images/grid_comparison_{date}_{measure}.pdf')

def boundary_setup(n_cells):
    xmin, ymin, xmax, ymax = -80.1892, 40.2246, -79.6912, 40.5963
    cell_size_x = (xmax - xmin) / n_cells
    cell_size_y = (ymax - ymin) / n_cells
    crs = "+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +a=6371007.181 +b=6371007.181 +units=m +no_defs"
    grid_cells = []
    for x0 in np.arange(xmin, xmax, cell_size_x):
        for y0 in np.arange(ymin, ymax, cell_size_y):
            x1 = x0 + cell_size_x
            y1 = y0 + cell_size_y
            grid_cells.append(shapely.geometry.box(x0, y0, x1, y1))
    cell = gpd.GeoDataFrame(grid_cells, columns=['geometry'], crs=crs)

    bbox = shapely.geometry.box(xmin, ymin, xmax, ymax)
    bbox_gdf = gpd.GeoDataFrame(geometry=[bbox], crs=crs)
    return cell, bbox_gdf, crs

def preprocess(df, bbox_gdf, crs):
 geometry = [Point(xy) for xy in zip(df['skewed longitude'], df['skewed latitude'])]
 gdf = gpd.GeoDataFrame(df, geometry=geometry)
 gdf = gdf.drop(['skewed longitude', 'skewed latitude'], axis=1)
 gdf.crs = crs

 points_in_bbox = gpd.sjoin(gdf, bbox_gdf, how="inner",predicate='within')
 joined_df = points_in_bbox.drop(columns=['index_right'])
 return joined_df

# these two functions can definitely be made into one - ran out of tim
def merge_sum(result_gdf, cell,date,group=None, plot = False):
    merged = gpd.sjoin(result_gdf, cell, how='left', predicate='within')
    cell_max_smell = merged.groupby('index_right')['smell value'].sum()
    cell = cell.merge(cell_max_smell, how='left', left_index=True, right_index=True)
    cell['smell value'] = cell['smell value'].fillna(0)
    if plot:
        ax = cell.plot(column='smell value', figsize=(12, 8), cmap='BuGn', edgecolor="white")
        sm = plt.cm.ScalarMappable(cmap='BuGn')
        sm.set_array(cell['smell value'])

        norm = mcolors.Normalize()
        sm.set_norm(norm)
        cbar = plt.colorbar(sm, ax=ax)
        cbar.set_label('Smell Value')
        ax.set_title(f'Summed smell value for {date}')
    return cell

def merge_average(result_gdf, cell,date, group=None, plot = False):
    merged = gpd.sjoin(result_gdf, cell, how='left', predicate='within')
    cell_max_smell = merged.groupby('index_right')['smell value'].mean()
    cell = cell.merge(cell_max_smell, how='left', left_index=True, right_index=True)
    cell['smell value'] = cell['smell value'].fillna(0)
    if plot:
        ax = cell.plot(column='smell value', figsize=(12, 8), cmap='BuGn', edgecolor="white")

        sm = plt.cm.ScalarMappable(cmap='BuGn')
        sm.set_array(cell['smell value'])

        norm = mcolors.Normalize(vmin=0, vmax=5)
        sm.set_norm(norm)
        cbar = plt.colorbar(sm, ax=ax)
        cbar.set_label('Smell Value')
        ax.set_title(f'Average smell value for {date}')
    return cell

def make_dcgan_gdf(array, geoms):
    try:
        grid = pd.DataFrame(array)
        grid.columns = ['emission']
        grid['geometry'] = geoms
        geo_df = gpd.GeoDataFrame(grid, geometry='geometry')
        geo_df.crs = 'EPSG:4326'
    except (ValueError, AttributeError):
        print('geo_df not working for some reason so skipping this one')
        geo_df = None
    return geo_df

def get_impact_stats(df, date, plot=False, title=None, plotsave=False):
 if df is None:
    print('no matching data for this time')
    return None, None
 else:
    df.drop(columns=['geometry'], inplace=True)

    means = df.groupby('smell value').mean()
    means.rename(columns={'emission': 'Mean'}, inplace=True)

    std = df.groupby('smell value').std()
    std.rename(columns={'emission': 'STD'}, inplace=True)

    counts = df.groupby('smell value').count()
    counts.rename(columns={'emission': 'Count'}, inplace=True)

    corr = df.corr()
    print(corr)

    stats = pd.concat([means, counts, std], axis=1)

    if plot == True:
        fig, ax = plt.subplots(figsize=(10, 10))
        plt.plot(df['smell value'], df['emission'], 'rx', alpha=0.5, markersize=20, markeredgewidth=6)
        plt.plot(stats.index, stats['Mean'], linewidth=6)
        plt.fill_between(stats.index, stats['Mean']-stats['STD'], stats['Mean']+stats['STD'], alpha=0.2)
        if title == 'Average':
            plt.xlabel('Average smell value')
        elif title == 'Sum':
            plt.xlabel('Summed smell value')
        plt.ylabel('PM2.5 concentration $\mu g/m^3$')
        plt.tight_layout()
    if plotsave == True:
        plt.savefig(f'images/impact_analysis_{date}_{title}.pdf')

    return stats, corr['emission']['smell value']

def percentage_agreement(hysplit, cr):
    cr['smell value'] = cr['smell value'].replace(0, np.nan)
    df1 = cr.dropna()
    try:
        merged_df = df1.merge(hysplit, on='geometry', suffixes=('_df1', '_df2'))
    except ValueError:
        print('value error')
        return 0, None

    nonzero_vals = len(merged_df.where(merged_df['emission'] > 0).dropna())
    print('nonzero vals', nonzero_vals)

    total_values = len(df1)
    print('total values', total_values)
    pa = (nonzero_vals / total_values) * 100
    return pa, merged_df

# ADJUST VARIABLES
test_num = 22
dcganPATH = f'analysis/grids/run{test_num}/'
input_info = torch.load(f'../results_images/test_{test_num}/state11579_full_input.pt')

dates = []
for i in range(len(input_info)):
    dates.append(input_info[i][-9])
# inverse scale
dates = np.array(dates)
dates = dates * (1680300000 - 1470002400) + 1470002400
dates = pd.to_datetime(dates, unit='s')
dcgan_output = []
with h5py.File(f'../results_images/test_{test_num}/tensors_state11579.h5', 'r') as hf:
    for dataset_name in hf:
        dcgan_output.append(hf[dataset_name][:])

full={}
# scale the dcgan output
for i in range(len(dcgan_output)):
    grid = dcgan_output[i].squeeze()
    flat_grid = grid.flatten('F')
    flat_grid_rescaled = (flat_grid - (-1.0)) * (35.0 - 0.0) / (1.0 - (-1.0)) + 0.0
    full[dates[i]] = flat_grid_rescaled

sorted_tuples = sorted(zip(dates, dcgan_output))
sorted_date_dcgan = [(date, dcgan) for date, dcgan in sorted_tuples]
sorted_dcgan = [dcgan for date, dcgan in sorted_tuples]
sorted_dates = [date for date, dcgan in sorted_tuples]

df = pd.DataFrame.from_dict(full, orient='index')
df.index = pd.to_datetime(df.index)
df = df.sort_index()

# load all the cr grids
smell_reports = pd.read_csv('../../../citizen_data/raw/smell_reports.csv', usecols=["epoch time","skewed latitude", "skewed longitude","smell value"]).set_index("epoch time")
smell_reports.index = pd.to_datetime(smell_reports.index, unit='s')
# grouped at daily resolution
cell, bbox, crs = boundary_setup(64)
grouped_by_hour = smell_reports.groupby(smell_reports.index.floor('H'))

# MAY NEED TO CHANGE THIS ACCORDING TO FILE STRUCTURE
geoms = pickle.load(open('../../HYSPLIT_results/geoms.pkl', 'rb'))

# SPECIFY THE DATETIME YOU WANT TO LOOK AT and PLOT (hourly resolution)
interested_date = '2022-10-22-22'
results = {}
for dayhour, rows in tqdm(grouped_by_hour):
    datehr = dayhour.strftime('%Y-%m-%d-%H')
    if datehr == interested_date:
        print('match', dayhour)
        index = sorted_dates.index(dayhour)
        print('index:', index)
        print('date:', sorted_dates[index])
        group_df = smell_reports.loc[rows.index]
        print('group_df', group_df)
        if len(group_df) == 0:
            print('no data for this hour')
            continue

        else:
            result_gdf = preprocess(group_df, bbox, crs)
            merged_avg = merge_average(result_gdf, cell, dayhour, plot=False)
            merged_sum = merge_sum(result_gdf, cell, dayhour, plot=False)

            dcgan = make_dcgan_gdf(df.iloc[index], geoms)

        # plot all
        # fig, ax = plt.subplots(1,3, figsize=(30,10), sharex=True, sharey=True)
        # merged_avg.plot(column='smell value', ax=ax[0], cmap='BuGn', edgecolor="white")
        # sm = plt.cm.ScalarMappable(cmap='BuGn')
        # sm.set_array(merged_avg['smell value'])
        # norm = mcolors.Normalize(vmin=0, vmax=5)
        # sm.set_norm(norm)
        # cbar = plt.colorbar(sm, ax=ax[0], shrink=0.5)
        # # cbar.set_label('Smell Value')
        # ax[0].set_title(f'Average smell value for {dayhour}')
        # merged_sum.plot(column='smell value', ax=ax[1], cmap='BuGn', edgecolor="white")
        # sm2 = plt.cm.ScalarMappable(cmap='BuGn')
        # sm2.set_array(merged_sum['smell value'])
        # cbar2 = plt.colorbar(sm2, ax=ax[1], shrink=0.5)
        # cbar2.set_label('Smell Value')
        # fig.tight_layout()
        # ax[1].set_title(f'Summed smell value for {dayhour}')
        # dcgan.plot(column='emission', cmap='BuGn', edgecolor="white", ax=ax[2])
        # ax[2].set_title(f'DCGAN output for {dayhour}')
        # sm3 = plt.cm.ScalarMappable(cmap='BuGn')
        # sm3.set_array(dcgan['emission'])
        # cbar3 = plt.colorbar(sm3, ax=ax[2], shrink=0.5)
        # cbar3.set_label('Emissions')

        # actual analysis
            pa_average, df_average = percentage_agreement(dcgan, merged_avg)
            pa_sum, df_sum = percentage_agreement(dcgan, merged_sum)
            print('average', pa_average)
            print('sum', pa_sum)
            impactstats_avg, corr_avg = get_impact_stats(df_average, datehr, plot=True, title='Average', plotsave=True)
            impactstats_sum, corr_sum = get_impact_stats(df_sum, datehr, plot=True, title='Sum', plotsave=True)

# Morning afternoon analysis
smell_reports['Year'] = smell_reports.index.year
smell_reports['Month'] = smell_reports.index.month
smell_reports['Day'] = smell_reports.index.day
smell_reports['Hour'] = smell_reports.index.hour
ranges = [range(6, 14 + 1), range(15, 22 + 1)]
names = ['morning', 'afternoon']
# DEFINE WHICH MONTH TO LOOK AT
yr, mo = 2022,10
month_results = {}
combined_df_average = pd.DataFrame()
smell_reports_range = smell_reports[(smell_reports['Year']==yr) & (smell_reports['Month']==mo)]
for day in range(22,23):
    for name, hours in zip(names, ranges):
        time = f'{yr}-{mo}-{day}-{name}'
        start = pd.Timestamp(f'{yr}-{str(mo).zfill(2)}-{str(day).zfill(2)}-{str(hours[0]).zfill(2)}')
        end = pd.Timestamp(f'{yr}-{str(mo).zfill(2)}-{str(day).zfill(2)}-{str(hours[-1]).zfill(2)}')
        mask = (df.index >= start) & (df.index <= end)
        subset = df.loc[mask]
        column_sums = subset.sum(axis=0)
        summed_df = pd.DataFrame({'time section total emission': column_sums})
        dcgan = make_dcgan_gdf(summed_df, geoms)
        group_df = smell_reports_range.loc[start:end]
        if len(group_df) == 0:
            print('no data for this hour')
            continue
        else:
            cr_result_df = preprocess(group_df, bbox, crs)
            merged_avg = merge_average(cr_result_df, cell, time, plot=False)
            merged_sum = merge_sum(cr_result_df, cell, time, plot=False)
            pa_average, df_average = percentage_agreement(dcgan, merged_avg)
            pa_sum, df_sum = percentage_agreement(dcgan, merged_sum)

            df_add_to_list = df_average.copy()
            df_add_to_list = df_add_to_list.drop(columns=['geometry'])
            combined_df_average=  pd.concat([combined_df_average,df_add_to_list] , axis=0)

            impactstats_avg, corr_avg = get_impact_stats(df_average, time, plot=False, title='Average', plotsave=False)
            impactstats_sum, corr_sum = get_impact_stats(df_sum, time, plot=True, title='Sum', plotsave=True)

            month_results[time] = {
                'date': time,
                'percentage agreement average': pa_average,
                'percentage agreement sum': pa_sum,
                'impact stats average': impactstats_avg,
                'impact stats sum': impactstats_sum,
                'correlation average': corr_avg,
                'correlation sum': corr_sum}

# UNCOMMENT TO SAVE THE RESULTS
# month_results_df = pd.DataFrame.from_dict(month_results, orient='index')
# save to csv, change the name of the file
# month_results_df.to_csv(f'stats/run{test_num}_11579_morningafternoon_{yr}{mo}.csv')

# PLOT
# imapct plot for the month just here
# get all the impact stats average out
impact_stats = month_results.get('impact stats average')
fig, ax = plt.subplots()
plt.plot(combined_df_average['smell value'], combined_df_average['emission'], 'rx', alpha=0.5)
means = combined_df_average.groupby('smell value').mean()
std = combined_df_average.groupby('smell value').std()
plt.plot(means.index, means['emission'])
plt.fill_between(std.index, means['emission']-std['emission'], means['emission']+std['emission'], alpha=0.2)
plt.savefig(f'images/impact_analysis_{yr}{mo}_average.pdf')

# This is a particular code that analyses a specified range of dates.
ranges = [range(6, 14 + 1), range(14, 22 + 1)]
names = ['morning', 'afternoon']
years = [2022,2022,2022,2022,2022,2022,2022]
months = [2,3,6,9,10,11,12]
month_results = {}
pa_list = []
feb_daysofinterest = [1,7,15,16,22]
march_daysofinterest = [5,10,11,15,16,17]
june_daysofinterest = [30]
sept_daysofinterest = [16,17]
oct_daysofinterest = [11,12,14,21,22,23,25]
nov_daysofinterest = [2,4,10,22]
dec_daysofinterest = [5,29,30]
combined_df_average = pd.DataFrame()
combined_df_sum = pd.DataFrame()
all_daysofinterest = [feb_daysofinterest, march_daysofinterest, june_daysofinterest, sept_daysofinterest, oct_daysofinterest, nov_daysofinterest, dec_daysofinterest]

for yr, mo, daysofinterest in zip(years, months, all_daysofinterest):
    print(yr, mo)
    smell_reports_range = smell_reports[(smell_reports['Year']==yr) & (smell_reports['Month']==mo)]
    for day in daysofinterest:
        for name, hours in zip(names, ranges):
            time = f'{yr}-{mo}-{day}-{name}'
            print(time)
            start = pd.Timestamp(f'{yr}-{str(mo).zfill(2)}-{str(day).zfill(2)}-{str(hours[0]).zfill(2)}')
            end = pd.Timestamp(f'{yr}-{str(mo).zfill(2)}-{str(day).zfill(2)}-{str(hours[-1]).zfill(2)}')
            mask = (df.index >= start) & (df.index <= end)
            subset = df.loc[mask]
            column_sums = subset.sum(axis=0)
            summed_df = pd.DataFrame({'time section total emission': column_sums})
            dcgan = make_dcgan_gdf(summed_df, geoms)
            # get the cr reports in this time range
            group_df = smell_reports_range.loc[start:end]
            if len(group_df) == 0:
                print('no data for this hour')
                continue
            else:
                cr_result_df = preprocess(group_df, bbox, crs)
                merged_avg = merge_average(cr_result_df, cell, time, plot=False)
                merged_sum = merge_sum(cr_result_df, cell, time, plot=False)
                pa_average, df_average = percentage_agreement(dcgan, merged_avg)
                print('pa average', pa_average)
                pa_list.append(pa_average)
                pa_sum, df_sum = percentage_agreement(dcgan, merged_sum)
                print('pa sum', pa_sum)
                df_add_to_list = df_average.copy()
                df_add_to_list = df_add_to_list.drop(columns=['geometry'])
                combined_df_average=  pd.concat([combined_df_average,df_add_to_list] , axis=0)

                df_sum_add_to_list = df_sum.copy()
                df_sum_add_to_list = df_sum_add_to_list.drop(columns=['geometry'])
                combined_df_sum=  pd.concat([combined_df_sum,df_sum_add_to_list] , axis=0)

# save these dfs, comment out if already done and then just load them in
combined_df_average.to_csv(f'combined_df_average_periodofimpact.csv')
combined_df_sum.to_csv(f'combined_df_sum_periodofimpact.csv')

# loading in code:
# combined_df_average = pd.read_csv('combined_df_average_periodofimpact.csv')
# combined_df_sum = pd.read_csv('combined_df_sum_periodofimpact.csv')

grouped = combined_df_average.groupby('smell value')
fig, ax = plt.subplots(1,1, figsize=(10,10))
hists = []
means =[]
smells = []
binwidth = 2
for name, group in grouped:
    smells.append(name)
    means.append(group['emission'].mean())
    ax.scatter(group['smell value'], group['emission'], label=name, color='blue', alpha=0.5)
ax.scatter(smells, means, color='red', marker='x')
ax.plot(smells, means, 'black')
plt.xlabel('Average smell value')
plt.ylabel('PM2.5 concentration ($\mu g/m^3$)')
plt.tight_layout()
plt.savefig(f'images/average_smellvalue_vs_emission_forperiodofinterest.pdf')


import win32com.client as win32
from win32com.client import DispatchEx
from win32com.client import DispatchEx
import openpyxl
import pandas as pd
import openpyxl
import os
from openpyxl import workbook
from openpyxl.utils import get_column_letter
from openpyxl.styles import Alignment
from openpyxl.cell import Cell
import random
from copy import copy
import time

from openpyxl.formula.translate import Translator

import logging
logging.basicConfig(filename='pipelineanalysis.log', level=logging.INFO)

from win32com.client import constants

logging.info('Test 2 File Started.')
#Import library, record, and print the date of the workflow execution.
from datetime import date
today = date.today()
newDate = today.strftime("%Y%m%d")
print("Today's date:", newDate)

#Creates variable for the necessary file destinations.
tsp5_updated = 'C:\\Users\\Administrator\\Downloads\\tsp5_updated.xlsx'
pl_template_rows = 'C:\\Users\\Administrator\\Downloads\\PL_rows.xlsx'
test_copy_merge = 'C:\\Users\\Administrator\\Downloads\\copy_merge.xlsx'
data_only = 'C:\\Users\\Administrator\\Downloads\\data_only.xlsx'
PL_file = 'C:\\Users\\Administrator\\Downloads\\PL_20210521_Workflow.xlsx'
updated_file = 'C:\\Users\\Administrator\\Downloads\\PL_Updated_Workflow.xlsx'

#os.remove(data_only)
#os.remove(test_copy_merge)
#os.remove(updated_file)

excel = win32.gencache.EnsureDispatch('Excel.Application')

row_file = openpyxl.load_workbook(tsp5_updated)
row_sheet = row_file.worksheets[0]

hdrs = []
for cell in row_sheet[1]:
    hdrs.append(cell.value)
print(hdrs)

maxRow = row_sheet.max_row - 1
pl_max_row = maxRow + 15

strRow = str(maxRow)
str_pl_max = str(pl_max_row)

pl_rows_def_names = openpyxl.load_workbook(pl_template_rows)

pl_rows_def_names.save(pl_template_rows)
pl_rows_def_names.close()

#Merges the updated tsp5_demand file and the autofilled template file into the master Excel file as tabs.
excel.Visible = False
PL_template_workbook = excel.Workbooks.Open(pl_template_rows)
tsp5_workbook = excel.Workbooks.Open(tsp5_updated)
tsp5_workbook.Worksheets(1).Move(Before = PL_template_workbook.Worksheets(1))
excel.DisplayAlerts = False
PL_template_workbook.SaveAs(test_copy_merge)
excel.DisplayAlerts = True
excel.Visible = True
excel.Application.Quit()
print('Files have been successfully merged.')
logging.info('Workbooks Merged into Final Document')

header_merge_file = openpyxl.load_workbook(test_copy_merge)
hdr_search = header_merge_file.worksheets[1]
maxCol = hdr_search.max_column

hdrs2 = []
indxs = []
data_indxs = []
matched_hdrs = []
j = 0
i = 0
for cell in hdr_search[16]:
    hdrs2.append(cell.value)
print(hdrs2)

while j < len(hdrs):
    while i < len(hdrs2):
        if hdrs[j] == hdrs2[i]:
            print(hdrs[j] + ' and ' + hdrs2[i] + ' are a match at column ' + str(i) + '.')
            data_indxs.append(j)
            indxs.append(i)
            matched_hdrs.append(hdrs[j])
        i = i + 1
    j = j + 1
    i = 0
print(indxs)
print(matched_hdrs)

header_merge_file.close()

data_only_file = openpyxl.load_workbook(test_copy_merge, data_only = True)
data_only_file.save(data_only)
data_only_sheet = data_only_file.worksheets[1]
data_only_file.close()

#Opens the updated tsp5  file using Openpyxl as a workbook and creates a worksheet for the tab.
main_PL_file = openpyxl.load_workbook(test_copy_merge)
data_sheet = main_PL_file.worksheets[0]
pl_sheet = main_PL_file.worksheets[1]

#Creates a string holding the number of opportunity numbers found in the data sheet.
baseRows = data_sheet.max_row
strRows = str(baseRows)

#Defines copyRange to bring in a specified range of cells to be copied over for transfer.
def copyRange(startCol, startRow, endCol, endRow ,sheet):
    rangeSelected=[]
    for i in range(startRow, endRow + 1, 1):
        rowSelected = []
        for j in range(startCol, endCol + 1, 1):
            rowSelected.append(data_only_sheet.cell(row = i, column = j).value)
            rangeSelected.append(rowSelected)
    return rangeSelected

#Define pasteRange to dictate where the cells should be pasted in the recieving document.
def pasteRange(startCol, startRow, endCol, endRow, sheetRecieving, copiedData):
    countRow = 1
    for i in range(0, len(copiedData), 1):
        countCol = 1
        for j in range(startCol, endCol+1, 1):
            sheetRecieving.cell(row = startRow + i, column = startCol,value = copiedData[i][0])
            countCol += 1
        countRow += 1

#Defines createData to copy and paste the data from one sheet to another. Copies the opportunity column from the tsp5
#data sheet and pastes it into the pre-formatted PL_template opportunity number column starting in column B and on row
#17. Prints at the end of the process to confirm.
def oppNumCopy():
    i = 0
    while i < len(indxs):
        #selectedRange = copyRange(indxs[i], 17, indxs[i], baseRows + 15, data_only)
        selectedRange = copyRange(indxs[i], 17, indxs[i], baseRows + 15, data_only_sheet)
        pastingRange = pasteRange(indxs[i], 17, indxs[i], baseRows + 15, pl_sheet, selectedRange)
        #print('Opportunity Number range copied and pasted.')
        print('Copying ' + hdrs[data_indxs[i]] + ' to ' + hdrs2[indxs[i]])
        #print(indxs[i])
        i = i + 1

oppNumCopy()

main_PL_file.save(test_copy_merge)

#Defines the header name.
pl_hdr_range = openpyxl.workbook.defined_name.DefinedName('PL_Hdr_Latest', attr_text = "'PL " + newDate + "'!$B$16:$EO$16")
#main_PL_file.defined_names.append(pl_hdr_range)

#Defines the data name.
pl_data_range = openpyxl.workbook.defined_name.DefinedName('PL_Data_Latest', attr_text = "'PL " + newDate + "'!$B$16:$EO$" + str_pl_max)
#main_PL_file.defined_names.append(pl_data_range)

plz_inwork_group_range = openpyxl.workbook.defined_name.DefinedName('PLZ_InWork_Group', attr_text = "'PL " + newDate + "'!$B$16:$EO$" + str_pl_max)
#main_PL_file.defined_names.append(plz_inwork_group_range)

plz_othergroup_a_range = openpyxl.workbook.defined_name.DefinedName('PLZ_OtherGroup_A', attr_text = "'PL " + newDate + "'!$B$16:$EO$" + str_pl_max)
#main_PL_file.defined_names.append(plz_othergroup_a_range)

plz_othergroup_ar_range = openpyxl.workbook.defined_name.DefinedName('PLZ_OtherGroup_AR', attr_text = "'PL " + newDate + "'!$B$16:$EO$" + str_pl_max)
#main_PL_file.defined_names.append(plz_othergroup_ar_range)

plz_othergroup_b_range = openpyxl.workbook.defined_name.DefinedName('PLZ_OtherGroup_B', attr_text = "'PL " + newDate + "'!$B$16:$EO$" + str_pl_max)
#main_PL_file.defined_names.append(plz_othergroup_b_range)

pl_oppno_latest_range = openpyxl.workbook.defined_name.DefinedName('PL_Oppno_Latest', attr_text = "'PL " + newDate + "'!$B$16:$B$" + str_pl_max)
#main_PL_file.defined_names.append(pl_oppno_latest_range)

pl_rowno_latest_range = openpyxl.workbook.defined_name.DefinedName('PL_RowNo_Latest', attr_text = "'PL " + newDate + "'!$B$17:$B$" + str_pl_max)
#main_PL_file.defined_names.append(pl_rowno_latest_range)

#final_workbook = openpyxl.load_workbook(updated_file, read_only = True)
#print('Done.')

#Open file using win32
#excel.Visible = True
#final_workbook = excel.Workbooks.Open(updated_file)

#Open file using pandas
#final_workbook = pd.read_excel(updated_file)

#Open file using openpyxl
#final_workbook = openpyxl.load_workbook(updated_file, read_only=True)
#final_workbook.remove('PL 20210430')
#final_workbook.save(updated_file)
#final_workbook.close()

#os.remove(data_only)
#os.remove(test_copy_merge)


# bigger_circles4.py

# Written by @zmagsar
# Nov 23, 2020

import turtle as t

t.speed(0)
t.pensize(20)
t.colormode(255)

for x in range(50):
    t.color((0, 0, 200 - x * 2), (0, 0, 255))
    t.penup()
    t.goto(0, 350 - 15 * x)
    t.right(45)
    t.pendown()
    t.begin_fill()
    t.circle(5 * x)
    t.end_fill()


import json
from typing import List, Union

import requests
from tenacity import (retry, retry_if_exception_type, stop_after_attempt,
                      wait_fixed)

from semanticscholar.SemanticScholarException import \
    BadQueryParametersException, ObjectNotFoundException

class ApiRequester:
    '''
    This class handles calls to Semantic Scholar API.
    '''

    def __init__(self, timeout) -> None:
        '''
        :param float timeout: an exception is raised \
            if the server has not issued a response for timeout seconds.
        '''
        self._timeout = timeout

    @property
    def timeout(self) -> int:
        '''
        :type: :class:`int`
        '''
        return self._timeout

    @timeout.setter
    def timeout(self, timeout: int) -> None:
        '''
        :param int timeout:
        '''
        self._timeout = timeout

    @retry(
        wait=wait_fixed(30),
        retry=retry_if_exception_type(ConnectionRefusedError),
        stop=stop_after_attempt(10)
    )
    def get_data(
                self,
                url: str,
                parameters: str,
                headers: dict,
                payload: dict = None
            ) -> Union[dict, List[dict]]:
        '''Get data from Semantic Scholar API

        :param str url: absolute URL to API endpoint.
        :param str parameters: the parameters to add in the URL.
        :param str headers: request headers.
        :param dict payload: data for POST requests.
        :returns: data or empty :class:`dict` if not found.
        :rtype: :class:`dict` or :class:`List` of :class:`dict`
        '''

        url = f'{url}?{parameters}'
        method = 'POST' if payload else 'GET'
        payload = json.dumps(payload) if payload else None
        r = requests.request(
            method, url, timeout=self._timeout, headers=headers, data=payload)

        data = {}
        if r.status_code == 200:
            data = r.json()
            if len(data) == 1 and 'error' in data:
                data = {}
        elif r.status_code == 400:
            data = r.json()
            raise BadQueryParametersException(data['error'])
        elif r.status_code == 403:
            raise PermissionError('HTTP status 403 Forbidden.')
        elif r.status_code == 404:
            data = r.json()
            raise ObjectNotFoundException(data['error'])
        elif r.status_code == 429:
            raise ConnectionRefusedError('HTTP status 429 Too Many Requests.')
        elif r.status_code in [500, 504]:
            data = r.json()
            raise Exception(data['message'])

        return data


# -*- coding: utf-8 -*-
# @Author: amish
# @Date:   2021-02-27 17:04:59
# @Last Modified by:   Amish Sharma
# @Last Modified time: 2021-05-08 17:13:57

import pytesseract
import re, cv2, math
import numpy as np
from scipy import ndimage
from imutils.perspective import four_point_transform
from modules import glob, threshold, information

def check_orientation(img):
	newdata = pytesseract.image_to_osd(np.array(img))
	orgi_confidence = int(re.search('(?<=Orientation confidence: )\d+', newdata).group(0))
	if orgi_confidence > 0:
		return rotate(img)
	else:
		return img

def rotate(image, center=None, scale=1.0):
	angle = 360 - int(re.search('(?<=Rotate: )\d+', pytesseract.image_to_osd(image)).group(0))
	(h, w) = image.shape[:2]

	if center is None:
		center = (w / 2, h / 2)
	mmm = cv2.getRotationMatrix2D(center, angle, scale)
	rotated = cv2.warpAffine(image, mmm, (w, h))
	return rotated

def process_image (img):
	thresh = threshold.threshold_image(img)
	edgeImg = np.asarray(thresh, np.uint8)
	orig = img.copy()
	gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
	img_edges = cv2.Canny(gray, 100, 100, apertureSize=3)

	contours, heirarchy = cv2.findContours(edgeImg, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
	level1 = []

	for i, tupl in enumerate(heirarchy[0]):
		if tupl[3] == -1:
			tupl = np.insert(tupl, 0, [i])
			level1.append(tupl)
	significant = []
	tooSmall = edgeImg.size * 5 / 100
	for tupl in level1:
		contour = contours[tupl[0]];
		area = cv2.contourArea(contour)
		if area > tooSmall:
			significant.append([contour, area])

			# Draw the contour on the original image
			cv2.drawContours(img, [contour], 0, (0,255,0),2, cv2.LINE_AA, maxLevel=1)

	significant.sort(key=lambda x: x[1])

	f = 0;
	for j in glob.v:
		peri = cv2.arcLength(significant[0][0], True)
		approx = cv2.approxPolyDP(significant[0][0], j * peri, True)
		if len(approx) == 4:
			warped = four_point_transform(orig, approx.reshape(4, 2))
			f = 1
			break

	if f == 0:
		mx = (0,0,0,0)    # biggest bounding box so far
		mx_area = 0
		for cont in contours:
			x,y,w,h = cv2.boundingRect(cont)
			area = w*h
			if area > mx_area:
				mx = x,y,w,h
				mx_area = area
		x,y,w,h = mx
		roi = img[y:y+h,x:x+w]
		lines = cv2.HoughLinesP(img_edges, 1, math.pi / 180.0, 100, minLineLength=100, maxLineGap=5)
		angles = []
		try:
			for [[x1, y1, x2, y2]] in lines:
				angle = math.degrees(math.atan2(y2 - y1, x2 - x1))
				angles.append(angle)
			median_angle = np.median(angles)
			img_rotated = ndimage.rotate(roi, median_angle)
			return img_rotated
		except:
			return None
	else:
		try:
			warped = check_orientation(warped)
		except:
			pass

		text = pytesseract.image_to_string(warped)
		if information.info(text) > 5:
			return warped
		else:
			mx = (0,0,0,0)
			mx_area = 0
			for cont in contours:
				x,y,w,h = cv2.boundingRect(cont)
				area = w*h
				if area > mx_area:
					mx = x,y,w,h
					mx_area = area
			x,y,w,h = mx
			roi = img[y:y+h,x:x+w]
			lines = cv2.HoughLinesP(img_edges, 1, math.pi / 180.0, 100, minLineLength=100, maxLineGap=5)
			angles = []
			for [[x1, y1, x2, y2]] in lines:
				angle = math.degrees(math.atan2(y2 - y1, x2 - x1))
				angles.append(angle)
			median_angle = np.median(angles)
			img_rotated = ndimage.rotate(roi, median_angle)
			return img_rotated
		return


import pytest

from pathlib import Path
from unittest import mock
from cvs import commands, errors

@pytest.mark.parametrize("path", ["..", "../../"])
def test_adding_wrong_path(path: str, test_view):
    command = commands.AddCommand(test_view)
    try:
        command(path)
    except errors.RepoNotFoundError:
        assert True
    else:
        assert False

@pytest.mark.parametrize("path", ["___", "unknown path"])
def test_adding_non_existing_path(path: str, test_view):
    command = commands.AddCommand(test_view)
    try:
        command(path)
    except errors.InvalidPathError:
        assert True
    else:
        assert False

def test_validating_init_command(test_view):
    with mock.patch("cvs.config.MAIN_PATH", return_value=Path("tests/")):
        command = commands.InitCommand(test_view)
        try:
            command()
        except errors.RepoAlreadyExistError:
            assert True
        else:
            assert False

def test_init_command(test_view, test_dir):
    commands.InitCommand(test_view)()
    assert Path(".cvs").exists()
    assert test_view.buffer[0].startswith("Инициализирован")

def test_commit_output(test_view, test_dir):
    commands.InitCommand(test_view)()
    commands.CommitCommand(test_view)("test")
    assert test_view.buffer[1].startswith("Нечего коммитить")

def test_log_output(test_view, test_dir):
    commands.InitCommand(test_view)()
    commands.LogCommand(test_view)()
    assert len(test_view.buffer) == 1

def test_status_command(test_view, test_dir):
    commands.InitCommand(test_view)()
    command = commands.StatusCommand(test_view)
    command()
    assert test_view.buffer[1].startswith("HEAD")