#!/usr/bin/env python3

# file    :  fuzzman.py
# repo    :  https://github.com/fuzzah/fuzzaide
# author  :  https://github.com/fuzzah
# license :  MIT
# check repository for more information

import os
import sys
import glob
import shutil
import signal
from time import sleep, time
from pprint import pprint

from multiprocessing import cpu_count

from fuzzaide.common import which
from fuzzaide.common.fuzz_stats import is_afl_fuzzer_stats_old, get_afl_stat_name
from .args import get_launch_args
from .running_process import RunningAFLProcess, TimeoutExpired
from .const import *

class FuzzManager:
    """
    Literally GOD-class that needs a ton of rework :(
    Responsible for:
        1. Generating fuzzer commands for different run modes (normal, --builds, --cmd-file)
        2. Running commands
        3. Monitoring and restarting of commands
        4. Statistics counting
        5. Desicion making on termination of fuzzing job

    Limitations:
        1. Only AFL, AFL++ supported
        2. Only interactive output mode supported
    """

    def __init__(self, args):
        self.procs = list()
        self.last_shown_screen_idx = 0
        self.args = args
        self.waited_for_child = False
        self.start_time = int(time())
        self.cores_specified = False
        self.num_from_file = 0

    @staticmethod
    def extract_instance_count(amount):
        """
        Gets number of instances from strings like "5", "10%" or "66.6%"
        """

        count = 0
        perc = amount.endswith("%")
        try:
            if perc:
                count = float(amount.split("%", 1)[0])
            else:
                count = int(amount)
        except ValueError:
            sys.exit(
                "Error in --builds argument: '%s' is not convertible to number of instances (examples: 3, 66.6%%)"
                % (amount,)
            )

        return count, perc

    def extract_complex_mode_params(self):
        """
        Iterate over --builds arguments and extract group name, path, number/percent of cpu cores
        """

        args = self.args

        params = []
        for build_spec in args.builds:
            bspec = build_spec.split(":")  # 0:1:2 -> NAME:PATH:N[%]
            num_spec_parts = len(bspec)
            name = None
            path = None
            count = None
            perc = False
            if num_spec_parts == 1:
                path = bspec[0]
            elif num_spec_parts == 2:
                if "%" in bspec[1] or bspec[1].isnumeric():
                    path = bspec[0]
                    count, perc = self.extract_instance_count(bspec[1])
                else:
                    name = bspec[0]
                    path = bspec[1]
            elif num_spec_parts == 3:
                name = bspec[0]
                path = bspec[1]
                count, perc = self.extract_instance_count(bspec[2])
            else:
                sys.exit(
                    "Error in --builds argument: format of one build is [NAME:]<dir/bin path>[:N[%]] (examples: -h/--help)"
                )

            if path.startswith("~"):
                path = os.path.expanduser(path)
            params.append([name, path, count, perc])

        if args.verbose:
            print("Params of --builds: ")
            pprint(params)

        all_dirs = all(os.path.isdir(p) for _, p, _, _ in params)
        all_bins = all(os.path.isfile(p) for _, p, _, _ in params)

        if not all_dirs and not all_bins:
            sys.exit(
                "Error: --builds should point EITHER to directories OR to binaries"
            )

        # build directories provided -> each one should contain binary with same app name
        if all_dirs:
            for i, (_, dirpath, _, _) in enumerate(params):
                path = os.path.normpath(os.path.join(dirpath, args.program[0]))
                if not os.path.isfile(path):
                    sys.exit(
                        "Error in --builds argument: directory '%s' does not contain '%s' (path checked: '%s')"
                        % (dirpath, args.program[0], path)
                    )
                params[i][1] = path

            return params

        # exact binaries specified (full or partial paths or in PATH)
        for i, (_, binpath, _, _) in enumerate(params):
            if which(binpath) is None:
                sys.exit(
                    "Error in --builds argument: file %s not found so it cannot be tested"
                    % (binpath,)
                )

        return params

    def adjust_complex_mode_params(self, params):
        """
        For complex mode (--builds). Converts percent ratios to number of instances
        and makes sure that each build is used at least once.
        params is a list of 4-item lists: name, path, count/percent, is_percent
        """

        # sum percents as specified by user
        percsum = 0.0
        for _, _, count, is_perc in params:
            if count is not None and is_perc:
                percsum += count

        # normalize percents to 100 if required
        if percsum > 100.0:
            if self.args.verbose:
                print(
                    "Info: sum of percents in --builds is %.2f%% which is not 100%%. Will proportionally adjust it"
                    % (percsum,)
                )
            for i, (_, _, count, is_perc) in enumerate(params):
                if is_perc:
                    params[i][2] = count * 100.0 / percsum
            percsum = 100.0

        # count builds without number or percent of cores specified
        count_none = sum(1 for _, _, count, _ in params if count is None)

        if count_none > 0:
            # set percents for builds without cores specified
            for i, (_, _, count, _) in enumerate(params):
                if count is None:
                    params[i][2] = (100.0 - percsum) / count_none
                    params[i][3] = True

        sum_cores_exact = sum(count for _, _, count, is_perc in params if not is_perc)
        number_of_free_cores = self.args.instances - sum_cores_exact
        if number_of_free_cores < 0 or (number_of_free_cores == 0 and percsum > 0.0):
            sys.exit(
                "Error in --builds argument: not enough processor cores to fit desired configuration"
            )

        if percsum > 0.0:
            for i, (_, _, count, is_perc) in enumerate(params):
                # make 1 core for builds specified as percent
                if is_perc and (
                    count <= 0.0 or number_of_free_cores * params[i][2] / 100.0 < 1
                ):
                    params[i][2] = 1
                    params[i][3] = False

        # final normalization of percents
        percsum = sum(p for _, _, p, is_perc in params if is_perc)
        if percsum != 0.0 and percsum != 100.0:
            for i, (_, _, count, is_perc) in enumerate(params):
                if is_perc:
                    params[i][2] = count * 100.0 / percsum
            percsum = 100.0

        sum_cores_exact = sum(count for _, _, count, is_perc in params if not is_perc)
        number_of_free_cores = self.args.instances - sum_cores_exact
        if number_of_free_cores < 0 or (number_of_free_cores == 0 and percsum > 0.0):
            sys.exit(
                "Error in --builds argument: not enough processor cores to fit desired configuration"
            )

        # convert all the rest percent ratios to number of cores
        if percsum > 0.0:
            for i, (_, _, p, is_perc) in enumerate(params):
                if is_perc:
                    params[i].append(p)
                    params[i][2] = 0
                else:
                    params[i].append(0.0)
                params[i].append(i)  # order

            core_percent = 100.0 / number_of_free_cores
            while True:  # TODO: there must be a better way :(
                params = sorted(params, key=lambda x: -x[4])
                for i, (_, _, _, _, perc, _) in enumerate(params):
                    if perc <= 0.0:
                        continue
                    params[i][4] -= core_percent
                    percsum -= core_percent
                    params[i][2] += 1
                    if percsum <= core_percent:
                        break
                if percsum <= core_percent:
                    break

            number_of_used_cores = sum(c for _, _, c, _, _, _ in params)
            if number_of_used_cores != self.args.instances:
                if self.args.verbose:
                    print(
                        "Math in fuzzman is junky! Fixing error with delta of %d cores"
                        % (number_of_used_cores - self.args.instances,)
                    )
                params = sorted(params, key=lambda x: -x[4])
                params[0][2] -= number_of_used_cores - self.args.instances
                if self.args.verbose:
                    pprint(params)

                number_of_used_cores = sum(c for _, _, c, _, _, _ in params)
                if number_of_used_cores != self.args.instances:
                    sys.exit(
                        "Math in fuzzman is really junky! Error with delta of %d cores! "
                        "Please create an issue with verbose run screenshot.\nFor now you may want to specify different percent/amount of cores"
                        % (number_of_used_cores - self.args.instances,)
                    )

            params = sorted(params, key=lambda x: x[5])  # return original order

        params = list(
            map(lambda x: x[0:3], params)
        )  # leave only name, path and number of cores

        number_of_used_cores = sum(c for _, _, c in params)
        if self.cores_specified and number_of_used_cores != self.args.instances:
            sys.exit(
                "Error in --builds argument: less cores specified in --builds (%d) than in -n (%d)"
                % (number_of_used_cores, self.args.instances)
            )

        if self.args.verbose:
            print(
                "Using %d cores out of total %d available in OS"
                % (number_of_used_cores, cpu_count())
            )

        return params

    def load_custom_cmds(self, path):
        """
        Load custom fuzzer commands from file specified by path.
        Returns list of commands to run instead of fuzzman-generated commands.
        Each element in result list is [worker_name, command]
        """

        cmds = []
        if not os.path.isfile(path):
            sys.exit("Error: file '%s' doesn't exist or it's not a file" % path)

        with open(path, "rt") as f:
            lines = f.readlines()

        for line in lines:
            line = line.strip()
            if len(line) < 1 or line[0] == "#":
                continue

            cmd = line.split(":", 1)
            if len(cmd) != 2:
                sys.exit(
                    "Error: bad command in file %s: %s\nCorrect format:\n  name : command"
                    % (path, line)
                )

            worker = []
            for s in cmd:
                s = s.strip()
                if len(s) < 1:
                    sys.exit(
                        "Error: empty worker name or command in file %s: %s\nCorrect format:\n  name : command"
                        % (path, line)
                    )

                worker.append(s)

            if self.args.verbose:
                print("Loaded custom command: %s" % worker)

            cmds.append(worker)

        # some sanity checks
        if len(cmds) < 1:
            sys.exit("Error: custom commands file doesn't contain any commands to run")

        unique_names = set(name for name, _ in cmds)
        if len(unique_names) < len(cmds):
            sys.exit("Error: custom commands file shouldn't contain duplicate names")

        if not self.args.cmd_file_allow_duplicates:
            unique_cmds = set(cmd for _, cmd in cmds)
            if len(unique_cmds) < len(cmds):
                sys.exit(
                    "Error: custom commands file shouldn't contain duplicate commands (use --cmd-file-allow-duplicates to override)"
                )

        if self.cores_specified and self.args.instances != len(cmds):
            sys.exit(
                "Error: you have specified number of cores = %d but custom commands file %s contains %d commands"
                % (self.args.instances, path, len(cmds))
            )

        return cmds

    def start(self, env=None):
        """
        Start instances either in normal mode, complex mode (--builds) or custom commands mode (--cmd-file)
        """

        if self.args.instances is None:
            self.cores_specified = False
            self.args.instances = cpu_count()
        else:
            self.cores_specified = True

        args = self.args

        if args.instances < 1:
            args.instances = 1

        if which(args.fuzzer_binary) is None:
            sys.exit(
                "File %s not found so it cannot be used as fuzzer" % args.fuzzer_binary
            )

        if not os.path.exists(args.input_dir):
            try:
                os.makedirs(args.input_dir)
            except OSError:
                sys.exit("Can't create input directory %s" % args.input_dir)
        elif not os.path.isdir(args.input_dir):
            sys.exit("Can't use %s as input directory" % args.input_dir)

        if len(glob.glob(os.path.join(args.input_dir, "*"))) < 1:
            path = os.path.join(args.input_dir, "1")
            if args.verbose:
                print("Creating simple input corpus: %s" % path)
            try:
                with open(path, "w") as f:
                    f.write("12345")
            except OSError:
                sys.exit("Wasn't able to create input corpus")

        if args.cleanup and os.path.isdir(args.output_dir):
            print("Removing directory '%s'" % args.output_dir, file=sys.stderr)
            try:
                shutil.rmtree(args.output_dir, ignore_errors=True)
            except shutil.Error:
                sys.exit(
                    "Wasn't able to remove output directory '%s'" % args.output_dir
                )

        if args.cmd_file is not None:
            custom_cmds = self.load_custom_cmds(args.cmd_file)
            for i, (worker_name, cmd) in enumerate(custom_cmds):
                worker_env = os.environ.copy()
                worker_env["AFL_FORCE_UI"] = "1"
                if env is not None:
                    worker_env.update(env)

                print("Starting worker #%d {%s}: %s" % (i + 1, worker_name, cmd))
                self.procs.append(
                    RunningAFLProcess(
                        name=worker_name,
                        groupname="custom",
                        cmd=cmd,
                        env=worker_env,
                        verbose=args.verbose,
                    )
                )
            self.start_time = int(time())
            return

        complex_mode = args.builds is not None and len(args.builds) > 0

        params = []
        used_builds = []
        if complex_mode:
            params = self.extract_complex_mode_params()
            if args.verbose:
                print('"Raw" params:')
                pprint(params)

            params = self.adjust_complex_mode_params(params)
            if args.verbose:
                print("Adjusted params:")
                pprint(params)

            for name, path, num_cores in params:
                used_builds.extend([[name, path]] * num_cores)
        else:  # normal run mode
            if which(args.program[0]) is None:
                sys.exit("File %s not found so it cannot be tested" % args.program[0])
            used_builds = [[None, args.program[0]]] * args.instances

        if args.verbose:
            print("Builds in use:")
            pprint(used_builds)

        # TODO: maybe split this method for basic and complex modes?
        if args.dump_cmd_file:
            print("# Fuzzer commands for use with --cmd-file option of fuzzman")

        for i, (groupname, path) in enumerate(used_builds):
            dictionary = ""

            if i == 0:
                role = "-M"
                worker_name = "m"
                if args.dict:
                    dictionary = " -x " + args.dict

            else:
                role = "-S"
                worker_name = "s"

            power_schedule = " -p explore" if i % 2 == 0 else " -p fast"

            worker_name += str(i + 1)

            if args.no_power_schedules:
                power_schedule = ""

            cmd = (
                args.fuzzer_binary
                + " -i "
                + args.input_dir
                + " -o "
                + args.output_dir
                + " -m "
                + args.memory_limit
                + dictionary
                + " "
                + role
                + " "
                + worker_name
                + power_schedule
            )

            if args.more_args:
                cmd += " " + args.more_args
            cmd += " -- " + path + " " + " ".join(args.program[1:])

            if args.dump_cmd_file:
                worker_env = dict()
            else:
                worker_env = os.environ.copy()
                worker_env["AFL_FORCE_UI"] = "1"

            if env is not None:
                worker_env.update(env)

            cmd = cmd.strip()

            if args.dump_cmd_file:
                wenv = " ".join(k + "=" + v for k, v in worker_env.items())
                if len(wenv) > 0:
                    print("%s : env %s %s" % (worker_name, wenv, cmd))
                else:
                    print("%s : %s" % (worker_name, cmd))
            else:
                print("Starting worker #%d {%s}: %s" % (i + 1, worker_name, cmd))
                self.procs.append(
                    RunningAFLProcess(
                        name=worker_name,
                        groupname=groupname,
                        cmd=cmd,
                        env=worker_env,
                        verbose=args.verbose,
                    )
                )

        if args.dump_cmd_file:
            sys.exit(0)

        self.start_time = int(time())

    def stop(self, grace_sig=signal.SIGINT):
        """
        Stop all fuzzer workers
        """

        if len(self.procs) < 1:
            return

        if self.args.dump_screens:
            print("Dumping status screens")
            self.dump_status_screens()
        else:
            print("Stopping processes")

        self.job_status_check(onlystats=True)

        for proc in self.procs:
            proc.stop(grace_sig=grace_sig)

        term_wait = 1.0
        if self.args.verbose:
            print("Waiting %.1f seconds to check for leftover processes" % (term_wait,))
        sleep(term_wait)

        for proc in self.procs:
            proc.stop(force=True)
        self.procs = []

    def health_check(self):
        """
        Check if fuzzer workers are still running, also print each worker status
        """

        if len(self.procs) < 1:
            return False

        print("Checking status of workers")
        num_ok = sum(1 for proc in self.procs if proc.health_check())

        print("%d/%d workers report OK status" % (num_ok, len(self.procs)))
        return num_ok > 0

    def display_next_status_screen(self, outfile=sys.stdout, static_dump=False):
        """
        Show interactive status screen of one fuzzer for approximately 5 seconds
        """

        if len(self.procs) < 1:
            print("No status screen to show")
            return

        if len(self.procs) == 1:
            self.last_shown_screen_idx = 0

        outbuf = getattr(outfile, "buffer", outfile)

        # helper for drawing workaround on linux with fancy boxes mode
        mqj = b"mqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqj"

        instance = self.procs[self.last_shown_screen_idx]
        if self.is_process_still_running(instance.proc):
            outbuf.write(CURSOR_HIDE)
            if static_dump:
                num_dumps = 1
            else:
                num_dumps = 100

            for _ in range(num_dumps):
                data = instance.get_output(24)
                need_drawing_workaround = not self.args.no_drawing_workaround and len(
                    data
                )

                if need_drawing_workaround:
                    data[0] = data[0].replace(mqj, b"")

                for line in data:
                    outbuf.write(line)

                if need_drawing_workaround:
                    outbuf.write(SET_G1 + bSTG + mqj + bSTOP + cRST + RESET_G1)

                if not static_dump:
                    sleep(0.05)
            outbuf.write(CURSOR_SHOW)
        else:  # process is not running
            if not self.waited_for_child:
                try:
                    instance.proc.wait(3.0)  # wait to prevent zombie-processes
                except TimeoutExpired:
                    pass  # timeout waiting: hanged process?
                else:
                    self.waited_for_child = True

            data = instance.get_output(29)
            for line in data:
                outbuf.write(line)
            if not static_dump:
                sleep(5.0)

        outbuf.write(bSTOP + cRST + RESET_G1 + CURSOR_SHOW)

        if len(self.procs) > 0:
            self.last_shown_screen_idx += 1
            self.last_shown_screen_idx %= len(self.procs)

    @staticmethod
    def is_process_still_running(proc):
        return proc.poll() is None

    def dump_status_screens(self, outfile=sys.stdout):
        """
        Print status screens of all fuzzer instances
        """

        self.last_shown_screen_idx = 0
        outbuf = getattr(outfile, "buffer", outfile)
        for _ in self.procs:
            outbuf.write(
                b"\n" * 40
            )  # messy "workaround" for overlapping status screens (tmux, etc)
            self.display_next_status_screen(outfile=outfile, static_dump=True)

        outbuf.write(b"\n\n")

    def get_fuzzer_stats(self, output_dir, idx, instance):
        """
        Form a dictionary from fuzzer_stats file of given fuzzer instance
        """

        if instance.name is None or len(instance.name) < 1:
            print(
                "Wasn't able to get stats of instance #%d because somehow it has no name"
                % (idx,),
                file=sys.stderr,
            )
            return None

        stats_file_path = os.path.join(output_dir, instance.name, "fuzzer_stats")
        if not os.path.isfile(stats_file_path):
            print(
                "Wasn't able to get stats of instance %s because somehow it has no fuzzer_stats file"
                % (instance.name,),
                file=sys.stderr,
            )
            return None

        try:
            with open(stats_file_path, "rt") as f:
                data = f.readlines()
        except OSError:
            print(
                "Wasn't able to get stats of instance %s because of fail to open '%s'"
                % (instance.name, stats_file_path),
                file=sys.stderr,
            )
            return None

        if data is None:
            print(
                "Wasn't able to get data of instance %s because its fuzzer_stats file '%s' is empty"
                % (instance.name, stats_file_path),
                file=sys.stderr,
            )
            return None

        stats = dict()
        for line in data:
            if ":" not in line:
                continue

            k, v = line.split(":", 1)
            k = k.strip()
            v = v.strip()
            stats[k] = v

        if len(stats) < 1:
            return None

        return stats

    @staticmethod
    def update_stat_timestamp(stats_dict, stat_name, saved_newest_stamp):
        """
        Use this method to update last (newest) path (crash, hang, etc) timestamp.
        Example: newest_path_stamp = update_stat_timestamp(stats, "last_path", newest_path_stamp)
        """

        stamp = stats_dict.get(stat_name)

        if stamp is None:
            return saved_newest_stamp

        try:
            stamp = int(stamp)
        except ValueError:
            return saved_newest_stamp

        if stamp > saved_newest_stamp:
            return stamp

        return saved_newest_stamp

    @staticmethod
    def format_seconds(seconds):
        """
        Returns time in AFL-like format: days, hrs, min, sec
        """

        s = seconds % 60
        m = (seconds // 60) % 60
        h = (seconds // 3600) % 24
        d = seconds // 86400

        if d > 0:
            return "%d days, %d hrs, %d min, %d sec" % (d, h, m, s)
        elif h > 0:
            return "%d hrs, %d min, %d sec" % (h, m, s)
        elif m > 0:
            return "%d min, %d sec" % (m, s)

        return "%d sec" % (s,)

    def job_status_check(self, onlystats=False):
        """
        Enumerate fuzzer_stats files, print stats, return True if stopping required
        """

        output_dir = self.args.output_dir
        if output_dir is None or len(output_dir) < 1:
            return False

        newest_path_stamp = 0
        newest_hang_stamp = 0
        newest_crash_stamp = 0

        sum_execs = 0
        sum_paths = 0
        sum_hangs = 0
        sum_crashes = 0
        sum_restarts = 0

        use_old_style = None

        for idx, instance in enumerate(self.procs, start=1):
            stats = self.get_fuzzer_stats(output_dir, idx, instance)

            if not stats:
                continue

            if use_old_style is None:
                use_old_style = is_afl_fuzzer_stats_old(stats)
                if use_old_style is None:
                    continue

            crashes = int(
                stats.get(get_afl_stat_name("unique_crashes", use_old_style), 0)
            )
            hangs = int(stats.get(get_afl_stat_name("unique_hangs", use_old_style), 0))
            paths_total = int(
                stats.get(get_afl_stat_name("paths_total", use_old_style), 0)
            )
            paths_found = int(
                stats.get(get_afl_stat_name("paths_found", use_old_style), 0)
            )

            sum_restarts += instance.total_restarts
            sum_crashes += crashes
            sum_hangs += hangs
            sum_paths += paths_total
            sum_execs += int(stats.get("execs_done", 0))

            if not onlystats:
                if instance.proc.poll():
                    status = "NOT "
                else:
                    status = ""

                if instance.groupname is not None:
                    print(
                        "Worker %s of group %s is %srunning"
                        % (instance.name, instance.groupname, status)
                    )
                else:
                    print("Worker %s is %srunning" % (instance.name, status))

                print(
                    "\tcrashes: %d, hangs: %d, paths total: %d"
                    % (crashes, hangs, paths_total)
                )
                print(
                    "\tpaths discovered: %d (%.2f%% of total paths)"
                    % (paths_found, 100.0 * paths_found / (paths_total or 1))
                )

            newest_path_stamp = self.update_stat_timestamp(
                stats, get_afl_stat_name("last_path", use_old_style), newest_path_stamp
            )
            newest_hang_stamp = self.update_stat_timestamp(
                stats, get_afl_stat_name("last_hang", use_old_style), newest_hang_stamp
            )
            newest_crash_stamp = self.update_stat_timestamp(
                stats,
                get_afl_stat_name("last_crash", use_old_style),
                newest_crash_stamp,
            )

        print("\nStats of this fuzzing job:")
        job_duration = int(time()) - self.start_time
        print("Duration: %s" % (self.format_seconds(job_duration),))

        if newest_path_stamp == 0:
            if not onlystats:
                print("\nNo more stats to display (yet)")
            return False

        e = float(sum_execs)
        c = ""
        if e >= 1000000000:
            e /= 1000000000
            c = "B"
        elif e >= 1000000:
            e /= 1000000
            c = "M"
        elif e >= 1000:
            e /= 1000
            c = "K"

        if len(c) > 0:
            if c == "B":
                print("   Execs: %.4f%c" % (e, c))
            else:
                print("   Execs: %.2f%c" % (e, c))
        else:
            print("   Execs: %.0f" % (e,))

        now = int(time())

        newest_path_delta = now - newest_path_stamp
        newest_path_fmt = self.format_seconds(newest_path_delta)
        print("   Paths: %d.\tLast new path: %s ago" % (sum_paths, newest_path_fmt))

        if sum_hangs > 0:
            delta = now - newest_hang_stamp
            seconds_fmt = self.format_seconds(delta)
            print("   Hangs: %d.\tLast new hang: %s ago" % (sum_hangs, seconds_fmt))
        else:
            print("   Hangs: 0")

        if sum_crashes > 0:
            delta = now - newest_crash_stamp
            seconds_fmt = self.format_seconds(delta)
            print(" Crashes: %d.\tLast new crash: %s ago" % (sum_crashes, seconds_fmt))
        else:
            print(" Crashes: 0")

        if sum_restarts > 0:
            print("Fuzzer restarts: %d" % (sum_restarts,))

        # now decide if we need to stop
        if (
            self.args.no_paths_stop is not None
            and self.args.no_paths_stop <= newest_path_delta
        ):
            if (
                self.args.minimal_job_duration is not None
                and job_duration < self.args.minimal_job_duration
            ):
                return False
            return True

        return False

def main():
    args = get_launch_args()

    retcode = 7
    fuzzman = FuzzManager(args)

    stdoutbuf = getattr(sys.stdout, "buffer", sys.stdout)

    def handler(_signo, _stack_frame):
        stdoutbuf.write(bSTOP + cRST + RESET_G1 + CURSOR_SHOW)
        print()
        fuzzman.stop()
        sys.exit(0)

    signal.signal(signal.SIGINT, handler)

    fuzzman.start()

    while True:
        stdoutbuf.write(TERM_CLEAR)
        if not fuzzman.health_check():  # this check also prints alive status of workers
            retcode = 1
            break

        sleep(5.0)
        stdoutbuf.write(TERM_CLEAR)
        fuzzman.display_next_status_screen()  # this displays fuzzer output in real time for ~5 seconds

        stdoutbuf.write(TERM_CLEAR)

        # this function displays stats and decides if we need to stop current fuzzing job
        if fuzzman.job_status_check():
            print("STOP CONDITION MET. Stopping current fuzzing job...")
            retcode = 0
            break
        sleep(5.0)

    fuzzman.stop()

    if retcode == 0:
        print("STOP CONDITION MET")

    return retcode

if __name__ == "__main__":
    sys.exit(main())


import requests
from bs4 import BeautifulSoup
from fake_useragent import UserAgent

ua = UserAgent()
header = {'user-agent': ua.firefox}

google_page = 'https://www.google.com'
autocomplete = 'https://www.'

def target_url():
    input_url = input("BS4 PARSER."
                      "\n\t(Default display: google.com) "
                      "\n\nINPUT URL:")

    if input_url == "":
        input_url = google_page
    elif autocomplete not in input_url:
        input_url = autocomplete + input_url

    requested_url = requests.get(input_url, headers=header)
    soup = BeautifulSoup(requested_url.content, 'lxml')  # html.parser

    print(soup.prettify())

    with open("scraped.txt", "w", encoding="utf-8") as f:
        f.write(soup.prettify())
        f.close()

if __name__ == '__main__':
    target_url()


import cv2 as cv
import matplotlib.pyplot as plt
import numpy as np

img = cv.imread('El.jpg')

blank = np.zeros(img.shape[:2],dtype='uint8')#for same size as image

mask = cv.circle(blank,(img.shape[1]//2,img.shape[0]//2),100,255,-1)
##cv.imshow('Circle',circle)

#gray = cv.cvtColor(img,cv.COLOR_BGR2GRAY)
#cv.imshow('Gray',gray)

#mask = cv.bitwise_and(gray,gray,mask=circle)
masked = cv.bitwise_and(img,img,mask=mask)# for color histogram
#cv.imshow('Mask',mask)
# grayscale histogram --> distribution of pixels across the image
#gray_hist = cv.calcHist([gray],[0],None,[256],[0,256])
#(images,channels,mask,histSize,ranges,hist=None,accumulate=None)
#gray_hist = cv.calcHist([gray],[0],mask,[256],[0,256])
#plt.figure()
#plt.title('Grayscale Histogram')
#plt.xlabel('Bins')
#plt.ylabel('# of pixels')
#plt.plot(gray_hist)
#plt.xlim([0,256])
#plt.show()

#Color Histogram

plt.figure()
plt.title('Color Histogram')
plt.xlabel('Bins')
plt.ylabel('# of pixels')

colors = ('b','g','r')

for i,col in enumerate(colors):
    hist = cv.calcHist([img],[i],None,[256],[0,256])
    plt.plot(hist,color=col)
    plt.xlim([0,256])
plt.show()

cv.waitKey(0)
cv.destroyAllWindows()


""" This script imports data from csv files and returns a dataframe.

Note
----
1. CSV - If only folder is specified and no filename, all csv's will be read in sorted order by name. \n
2. CSV - Doesn't handle cases when user provides \n
    \t 1. file_name of type str and folder_name of type list(str) \n
    \t 2. file_name and folder_name both of type list(str)

To Do \n
    \t 1. Remove Import_XBOS from Import_Data.

Authors \n
@author Marco Pritoni <marco.pritoni@gmail.com> \n
@author Jacob Rodriguez  <jbrodriguez@ucdavis.edu> \n
@author Pranav Gupta <phgupta@ucdavis.edu> \n

Last modified: November 12 2018 \n

"""

import os
import glob
import numpy as np
import pandas as pd

class Import_Data:

    """ This class imports data from csv files """

    def __init__(self):
        """ Constructor.

        This class stores the imported data.

        """
        self.data = pd.DataFrame()

    def import_csv(self, file_name='*', folder_name='.', head_row=0, index_col=0, convert_col=True, concat_files=False):
        """ Imports csv file(s) and stores the result in data.

        Note
        ----
        1. If folder exists out of current directory, folder_name should contain correct regex
        2. Assuming there's no file called "\*.csv"

        Parameters
        ----------
        file_name       : str
            CSV file to be imported. Defaults to '\*', i.e. all csv files in the folder.
        folder_name     : str
            Folder where file resides. Defaults to '.', i.e. current directory.
        head_row        : int
            Skips all rows from 0 to head_row-1
        index_col       : int
            Skips all columns from 0 to index_col-1
        convert_col     : bool
            Convert columns to numeric type
        concat_files    : bool
            Appends data from files to result dataframe

        """

        # Import a specific or all csv files in folder
        if isinstance(file_name, str) and isinstance(folder_name, str):
            try:
                self.data = self._load_csv(file_name, folder_name, head_row, index_col, convert_col, concat_files)
            except Exception as e:
                raise e

        # Import multiple csv files in a particular folder.
        elif isinstance(file_name, list) and isinstance(folder_name, str):

            for i, file in enumerate(file_name):
                if isinstance(head_row, list):
                    _head_row = head_row[i]
                else:
                    _head_row = head_row

                if isinstance(index_col, list):
                    _index_col = index_col[i]
                else:
                    _index_col = index_col

                try:
                    data_tmp = self._load_csv(file, folder_name, _head_row, _index_col, convert_col, concat_files)
                    if concat_files:
                        self.data = self.data.append(data_tmp, ignore_index=False, verify_integrity=False)
                    else:
                        self.data = self.data.join(data_tmp, how="outer")
                except Exception as e:
                    raise e

        else:
            # Current implementation can't accept,
            # 1. file_name of type str and folder_name of type list(str)
            # 2. file_name and folder_name both of type list(str)
            raise NotImplementedError("Filename and Folder name can't both be of type list.")

    def _load_csv(self, file_name, folder_name, head_row, index_col, convert_col, concat_files):
        """ Load single csv file.

        Parameters
        ----------
        file_name       : str
            CSV file to be imported. Defaults to '*' - all csv files in the folder.
        folder_name     : str
            Folder where file resides. Defaults to '.' - current directory.
        head_row        : int
            Skips all rows from 0 to head_row-1
        index_col       : int
            Skips all columns from 0 to index_col-1
        convert_col     : bool
            Convert columns to numeric type
        concat_files    : bool
            Appends data from files to result dataframe

        Returns
        -------
        pd.DataFrame()
            Dataframe containing csv data

        """

        # Denotes all csv files
        if file_name == "*":

            if not os.path.isdir(folder_name):
                raise OSError('Folder does not exist.')
            else:
                file_name_list = sorted(glob.glob(folder_name + '*.csv'))

                if not file_name_list:
                    raise OSError('Either the folder does not contain any csv files or invalid folder provided.')
                else:
                    # Call previous function again with parameters changed (file_name=file_name_list, folder_name=None)
                    # Done to reduce redundancy of code
                    self.import_csv(file_name=file_name_list, head_row=head_row, index_col=index_col,
                                    convert_col=convert_col, concat_files=concat_files)
                    return self.data

        else:
            if not os.path.isdir(folder_name):
                raise OSError('Folder does not exist.')
            else:
                path = os.path.join(folder_name, file_name)

                if head_row > 0:
                    data = pd.read_csv(path, index_col=index_col, skiprows=[i for i in range(head_row-1)])
                else:
                    data = pd.read_csv(path, index_col=index_col)

                # Convert time into datetime format
                try:
                    # Special case format 1/4/14 21:30
                    data.index = pd.to_datetime(data.index, format='%m/%d/%y %H:%M')
                except:
                    data.index = pd.to_datetime(data.index, dayfirst=False, infer_datetime_format=True)

        # Convert all columns to numeric type
        if convert_col:
            # Check columns in dataframe to see if they are numeric
            for col in data.columns:
                # If particular column is not numeric, then convert to numeric type
                if data[col].dtype != np.number:
                    data[col] = pd.to_numeric(data[col], errors="coerce")

        return data

class Import_XBOS(Import_Data):

    """ This class imports data from XBOS """

    def __init__(self):
        """ Constructor.

        This class imports the dataclient module & stores the imported data.

        """

        import dataclient
        self.weather_data = pd.DataFrame()
        self.power_data = pd.DataFrame()
        self.temp_data = pd.DataFrame()
        self.hsp_data = pd.DataFrame()
        self.csp_data = pd.DataFrame()

    def get_weather_power_tstat(self, site, start, end, data_type=['weather', 'power']):
        """ Get weather and power data.

        Parameters
        ----------
        site        : str
            Site name.
        start       : str
            Start date.
        end         : str
            End date.
        data_type   : str
            Type of data needed (all, weather, power, temperature, hsp, csp)

        """

        m = dataclient.MDALClient("corbusier.cs.berkeley.edu:8088")

        request = {
            "Variables": {
                "greenbutton": {
                    "Definition": """SELECT ?meter ?meter_uuid FROM %s WHERE {
                        ?meter rdf:type brick:Green_Button_Meter .
                        ?meter bf:uuid ?meter_uuid
                    };""" % site,
                },
                "weather": {
                    "Definition": """SELECT ?t ?t_uuid FROM %s WHERE {
                        ?t rdf:type/rdfs:subClassOf* brick:Weather_Temperature_Sensor .
                        ?t bf:uuid ?t_uuid
                    };""" % site,
                },
                "tstat_state": {
                    "Definition": """SELECT ?t ?t_uuid ?tstat FROM %s WHERE {
                        ?t rdf:type/rdfs:subClassOf* brick:Thermostat_Status .
                        ?t bf:uuid ?t_uuid
                        ?t bf:isPointOf ?tstat .
                        ?tstat rdf:type brick:Thermostat
                    };""" % site,
                },
                "tstat_hsp": {
                    "Definition": """SELECT ?t ?t_uuid ?tstat FROM %s WHERE {
                        ?t rdf:type/rdfs:subClassOf* brick:Supply_Air_Temperature_Heating_Setpoint .
                        ?t bf:uuid ?t_uuid .
                        ?t bf:isPointOf ?tstat .
                        ?tstat rdf:type brick:Thermostat
                    };""" % site,
                },
                "tstat_csp": {
                    "Definition": """SELECT ?t ?t_uuid ?tstat FROM %s WHERE {
                        ?t rdf:type/rdfs:subClassOf* brick:Supply_Air_Temperature_Cooling_Setpoint .
                        ?t bf:uuid ?t_uuid .
                        ?t bf:isPointOf ?tstat .
                        ?tstat rdf:type brick:Thermostat
                    };""" % site,
                },
                "tstat_temp": {
                    "Definition": """SELECT ?t ?t_uuid ?tstat FROM %s WHERE {
                        ?t rdf:type/rdfs:subClassOf* brick:Temperature_Sensor .
                        ?t bf:uuid ?t_uuid .
                        ?t bf:isPointOf ?tstat .
                        ?tstat rdf:type brick:Thermostat
                    };""" % site,
                },
            },
        }

        # outside air temp
        request['Composition'] = ['weather']
        request['Aggregation'] = {'weather': ['MEAN']}
        request['Time'] = {
            'Start': start,
            'End': end,
            'Window': '15m',
            'Aligned': True
        }
        resp_weather = m.query(request)
        self.weather_data = resp_weather.df

        # power
        request['Composition'] = ['greenbutton']
        request['Aggregation'] = {'greenbutton': ['MEAN']}
        resp_power = m.query(request)
        self.power_data = resp_power.df

        # tstat temperature
        request['Composition'] = ['tstat_temp', 'tstat_hsp', 'tstat_csp']
        request['Aggregation'] = {'tstat_temp': ['MEAN']}
        resp_temp  = m.query(request)
        self.temp_data = resp_temp

        # tstat heat setpoint
        request['Composition'] = ['tstat_hsp']
        request['Aggregation'] = {'tstat_hsp': ['MAX']}
        resp_hsp = m.query(request)
        self.hsp_data = resp_hsp

        # tstat cool setpoint
        request['Composition'] = ['tstat_csp']
        request['Aggregation'] = {'tstat_csp': ['MAX']}
        resp_csp = m.query(request)
        self.csp_data = resp_csp

        mapping = {
            'weather': resp_weather,
            'power': resp_power,
            'temperature': resp_temp,
            'hsp': resp_hsp,
            'csp': resp_csp
        }

        first = True
        for dat in data_type:
            if first:
                try:
                    self.data = mapping[dat].df
                    first = False
                except:
                    raise SystemError('Undefined data_type (Make sure all characters are lowercase)')
            else:
                try:
                    self.data = self.data.join(mapping[dat].df)
                except:
                    raise SystemError('Undefined data_type (Make sure all characters are lowercase)')

        return mapping

class Import_MDAL(Import_Data):

    def __init__(self):

        import dataclient
        self.m = dataclient.MDALClient("corbusier.cs.berkeley.edu:8088")

    def get_meter(self, site, start, end, var="meter", agg='MEAN', window='24h', aligned=True, return_names=True):

        point_type = 'Green_Button_Meter'
        request = self.compose_MDAL_dic(point_type=point_type, site=site, start=start, end=end,  var=var, agg=agg, window=window, aligned=aligned)
        resp = self.m.query(request)

        if return_names:
            resp = self.replace_uuid_w_names(resp)

        return resp

    def get_weather(self, site, start, end,  var="weather", agg='MEAN', window='24h', aligned=True, return_names=True):

        point_type = 'Weather_Temperature_Sensor'
        request = self.compose_MDAL_dic(point_type=point_type, site=site, start=start, end=end,  var=var, agg=agg, window=window, aligned=aligned)
        resp = self.m.query(request)

        if return_names:
            resp = self.replace_uuid_w_names(resp)

        return resp

    def get_tstat(self, site, start, end,  var="tstat_temp", agg='MEAN', window='24h', aligned=True, return_names=True):

        point_map = {"tstat_state" : "Thermostat_Status",
                     "tstat_hsp" : "Supply_Air_Temperature_Heating_Setpoint",
                     "tstat_csp" : "Supply_Air_Temperature_Cooling_Setpoint",
                     "tstat_temp": "Temperature_Sensor" }

        if isinstance(var,list):
            point_type = [point_map[point_type] for point_type in var] # list of all the point names using BRICK classes
        else:
            point_type = point_map[var] # single value using BRICK classes

        request = self.compose_MDAL_dic(point_type=point_type, site=site, start=start, end=end,  var=var, agg=agg, window=window, aligned=aligned)
        resp = self.m.query(request)

        if return_names:
            resp = self.replace_uuid_w_names(resp)

        return resp

    def compose_MDAL_dic(self, point_type, site, start, end,  var, agg, window, aligned, points=None, return_names=False):

        request = {}
        # add Time Details - single set for one or multiple series
        request['Time'] = {
            'Start': start,
            'End': end,
            'Window': window,
            'Aligned': aligned
                           }
        # define Variables
        request["Variables"] = {}
        request['Composition'] = []
        request['Aggregation'] = {}

        if isinstance(point_type, str): # if point_type is a string -> single type of point requested
            request["Variables"][var] =  self.compose_BRICK_query(point_type=point_type,site=site) # pass one point type at the time
            request['Composition'] = [var]
            request['Aggregation'][var] = [agg]

        elif isinstance(point_type, list): # loop through all the point_types and create one section of the brick query at the time

            for idx, point in enumerate(point_type):
                request["Variables"][var[idx]] =  self.compose_BRICK_query(point_type=point,site=site) # pass one point type at the time
                request['Composition'].append(var[idx])

                if isinstance(agg, str): # if agg is a string -> single type of aggregation requested
                    request['Aggregation'][var[idx]] = [agg]
                elif isinstance(agg, list): # if agg is a list -> expected one agg per point
                    request['Aggregation'][var[idx]] = [agg[idx]]

        #pprint.pprint(request)
        return request

    def compose_BRICK_query(self, point_type,site):

        if point_type == "Green_Button_Meter":
            BRICK_query = {"Definition": """SELECT ?point ?uuid FROM %s WHERE {
                                                        ?point rdf:type brick:%s .
                                                        ?point bf:uuid ?uuid
                                                                              };""" % (site,point_type)
                          }
        if point_type == "Weather_Temperature_Sensor":
            BRICK_query = {"Definition": """SELECT ?point ?uuid FROM %s WHERE {
                                                   ?point rdf:type/rdfs:subClassOf* brick:%s .
                                                   ?point bf:uuid ?uuid
                                                                            };""" % (site,point_type)
                          }

        if point_type in ["Thermostat_Status","Supply_Air_Temperature_Heating_Setpoint",
                          "Supply_Air_Temperature_Cooling_Setpoint","Temperature_Sensor"]: ##  "tstat_state","tstat_hsp","tstat_csp","tstat_temp":
            BRICK_query = {"Definition": """SELECT ?point ?uuid ?equip FROM %s WHERE {
                                            ?point rdf:type/rdfs:subClassOf* brick:%s .
                                            ?point bf:uuid ?uuid .
                                            ?point bf:isPointOf ?equip .
                                            ?equip rdf:type brick:Thermostat };""" % (site,point_type)
                          }

        return BRICK_query

    def parse_context(self, context):

        metadata_table = pd.DataFrame(context).T

        return metadata_table

    def strip_point_name(self, col):

        return col.str.split("#",expand=True)[1]

    def get_point_name(self, context):

        metadata_table = self.parse_context(context)

        return metadata_table.apply(self.strip_point_name, axis=1)

    def replace_uuid_w_names(self, resp):
        col_mapper = self.get_point_name(resp.context)["?point"].to_dict()
        resp.df.rename(columns = col_mapper, inplace=True)
        return resp


class Solution:
    import itertools
    import sys
    def threeSumClosest(self, nums: 'List[int]', target: 'int') -> 'int':
        min_num = sys.maxsize
        res = 0
        for i in list(map(sum,list(itertools.combinations(nums, 3)))):
            if abs(i - target) < min_num:
                min_num = abs(i - target)
                res = i
        return res


from django.db import models
from django.contrib.postgres.fields import ArrayField

import datetime
from django.dispatch import receiver
from django.db.models import Q

import re

class Location(models.Model):
    # loc is my own classification of location
    loc = models.CharField(max_length=20)
    warehouse_location = models.CharField(max_length=10, default="USLA")
    area = models.CharField(max_length=2)
    aisle_letter = models.CharField(max_length=3, default="")
    aisle_num = models.IntegerField()
    level = models.IntegerField()
    column = models.IntegerField()

    def __str__(self):
        loc_name = self.warehouse_location + "." + self.area + "." +\
                   self.aisle_letter + str(self.aisle_num) + "." + \
                   str(self.column) + "." + str(self.level)
        return loc_name

def make_location(warehouse_location, area, aisle_letter, aisle_num, column, level):
    location_inst = Location(warehouse_location=warehouse_location,
                             area=area,
                             aisle_letter=aisle_letter,
                             aisle_num=aisle_num,
                             column=column,
                             level=level,
                             )
    loc = ''
    if area == "PH" or area == "PA":
        loc = "P"
    elif area == "S":
        if aisle_num == 60:
            loc = "VC"
        else:
            loc = "S"
    elif area == "F":
        loc = "F"
    elif area == "VA" or area == "VB":
        if aisle_num == 44:
            loc = "VC"
        else:
            loc = "F"
    elif area == "VC" or area == "VD":
        loc = "VC"
    elif area == "H":
        if aisle_letter == "H":
            loc = "S"
        elif aisle_num == 8:
            loc = "S"
        elif aisle_num <= 6:
            loc = "VC"
    location_inst.loc = loc

    location_inst.save()
    return location_inst

def filter_item_query_by_loc(item_query, loc):
    if loc == "P":
        return item_query.filter( Q(rack_location__area="PH") | Q(rack_location__area="PA"))
    elif loc == "VC":
        q1 = Q(rack_location__area="VC")
        q2 = Q(rack_location__area="VD")
        q3 = (Q(rack_location__aisle_num=44) & Q(rack_location__area="VA"))
        q4 = (Q(rack_location__aisle_num=44) & Q(rack_location__area="VB"))
        q5 = (Q(rack_location__aisle_num=60) & Q(rack_location__area="S"))
        q6 = (Q(rack_location__area="H") & Q(rack_location__aisle_num__lte=6) & ~Q(rack_location__aisle_letter="H") )
        return item_query.filter(q1 | q2 | q3 | q4 | q5 | q6)
    elif loc == "S":
        q1 = (~Q(rack_location__aisle_num=60) & Q(rack_location__area="S"))
        q2 = (Q(rack_location__area="H" ) & Q(rack_location__aisle_letter="H"))
        q3 = (Q(rack_location__area="H") & Q(rack_location__aisle_num=8))
        return item_query.filter(q1 | q2 | q3)
    elif loc == "F":
        q1 = Q(rack_location__area="F")
        q2 = (~Q(rack_location__aisle_num=44) & Q(rack_location__area="VA"))
        q3 = (~Q(rack_location__aisle_num=44) & Q(rack_location__area="VB"))
        return item_query.filter(q1 | q2 | q3)
    return item_query

class DataDate(models.Model):
    date = models.DateTimeField()

class Items(models.Model):
    item_id = models.IntegerField()
    lab_id = models.IntegerField(default = 0)
    item_weight = models.FloatField(default = 0)
    avail_quantity = models.IntegerField()
    ship_quantity = models.IntegerField()
    location_code = models.CharField(max_length=20, default="")
    fifo_date = models.DateTimeField()
    iv_create_date = models.DateTimeField(default=datetime.datetime.now)
    rack_location = models.ForeignKey(Location, on_delete=models.CASCADE, default=None)
    rcv = models.CharField(max_length=20)
    data_date = models.ForeignKey(DataDate, on_delete=models.CASCADE, default=None)
    description = models.CharField(max_length=100, default="")
    item_code = models.CharField(max_length=50, default="")
    customer_code = models.IntegerField(default="0")
    last_out_date = models.DateTimeField(blank=True, null=True, default=None)

    def get_input_date(self):
        rcv = self.rcv
        recv_re = re.compile("^RECV")
        if recv_re.match(rcv):
            return self.iv_create_date
        else:
            return self.fifo_date


from collections import defaultdict

import numpy as np
# import faiss
import pickle

# from umls_utils import cui2st
# # from umls import umls_kb_st21pv as umls_kb

# class FaissVSM(object):

#     def __init__(self, k=10):
#         self.index = None
#         self.labels = []
#         self.k = k

#     def create(self, vecs_txt_path):
#         vectors = []
#         with open(vecs_txt_path, encoding='utf-8') as vecs_f:
#             for line_idx, line in enumerate(vecs_f):
#                 # label, vals = line.split('\t')
#                 elems = line.strip().split()
#                 label, vals = elems[0], elems[1:]
#                 self.labels.append(label)
#                 # vectors.append(np.array(list(map(float, vals.split())), dtype=np.float32))
#                 vectors.append(np.array(list(map(float, vals)), dtype=np.float32))

#                 if line_idx % 100000 == 0:
#                     print(line_idx)

#         vectors = np.vstack(vectors)
#         d = vectors.shape[1]
#         self.index = faiss.IndexFlatL2(d)
#         # self.index = faiss.IndexLSH(d, 2 * d)
#         self.index.add(vectors)

#     def load(self, index_path, labels_path):
#         self.index = faiss.read_index(index_path)
#         with open(labels_path, 'rb') as labels_f:
#             self.labels = pickle.load(labels_f)

#         # # aux
#         # self.label2cui = {l: l.split('%')[-1] for l in self.labels}

#         # self.labels_by_st = defaultdict(list)
#         # for label in self.labels:
#         #     st = cui2st(self.label2cui[label])
#         #     if st is None:  # failed to retrieve st
#         #         continue

#         #     self.labels_by_st[st].append(label)

#     # def load(self, index_path, labels_path):
#     #     self.index = faiss.read_index(index_path)
#     #     with open(labels_path, 'rb') as labels_f:
#     #         self.labels = pickle.load(labels_f)

#     #     # aux
#     #     self.label2cui = {l: l.split('%')[-1] for l in self.labels}

#     #     self.labels_by_st = defaultdict(list)
#     #     for label in self.labels:
#     #         st = cui2st(self.label2cui[label])
#     #         if st is None:  # failed to retrieve st
#     #             continue

#     #         self.labels_by_st[st].append(label)

#     def save(self, index_path, labels_path):
#         faiss.write_index(self.index, index_path)
#         with open(labels_path, 'wb') as labels_f:
#             pickle.dump(self.labels, labels_f)

#     def most_similar(self, query_vec):
#         dists, idxs = self.index.search(query_vec.reshape(1, -1), self.k)
#         dists, idxs = dists[0], idxs[0]
#         sims = [1 - d for d in dists]
#         idx_labels = [self.labels[i] for i in idxs]

#         r = list(zip(idx_labels, sims))
#         r = sorted(r, key=lambda x: x[1], reverse=False)
#         return r

#     # def most_similar_cuis(self, query_vec, k=10, alias_k=30, restrict_sts=[]):

#     #     sims_cuis = defaultdict(lambda:float('inf'))

#     #     q_most_similar = self.most_similar(query_vec, alias_k)

#     #     if len(restrict_sts) > 0:
#     #         relevant_labels = set()
#     #         for st in restrict_sts:
#     #             relevant_labels.update(self.labels_by_st[st])

#     #         # filter to only consider cuis belonging to given sts
#     #         q_most_similar = [(l, d) for l, d in q_most_similar if l in relevant_labels]

#     #     for label, dist in q_most_similar:
#     #         cui = self.label2cui[label]
#     #         if dist < sims_cuis[cui]:
#     #             sims_cuis[cui] = dist

#     #     r = list(sims_cuis.items())
#     #     return sorted(r, key=lambda x: x[1], reverse=False)[:k]

class VSM(object):

    def __init__(self, vecs_path, dtype='float32', delimiter=' ', normalize=True):
        self.vecs_path = vecs_path

        if dtype == 'float32':
            self.dtype = np.float32
        elif dtype == 'float16':
            self.dtype = np.float16
        else:
            self.dtype = np.float

        self.labels = []
        self.vectors = np.array([], dtype=self.dtype)
        self.indices = {}
        self.ndims = 0

        self.load_txt(vecs_path, delimiter)

        if normalize:
            self.normalize()

    def load_txt(self, vecs_path, delimiter):
        self.vectors = []
        with open(vecs_path, encoding='utf-8') as vecs_f:
            for line_idx, line in enumerate(vecs_f):
                elems = line.strip().split(delimiter)
                self.labels.append(elems[0])
                self.vectors.append(np.array(list(map(float, elems[1:])), dtype=self.dtype))

        self.vectors = np.vstack(self.vectors)
        self.indices = {l: i for i, l in enumerate(self.labels)}
        self.ndims = self.vectors.shape[1]

    def save_txt(self, vecs_path, delimiter='\t'):
        with open(vecs_path, 'w') as vecs_f:
            for label, vec in zip(self.labels, self.vectors):
                vec_str = ' '.join([str(round(v, 6)) for v in vec.tolist()])
                vecs_f.write('%s%s%s\n' % (label, delimiter, vec_str))

    def load_npz(self, npz_vecs_path):
        loader = np.load(npz_vecs_path)
        self.labels = loader['labels'].tolist()
        self.vectors = loader['vectors']

        self.labels_set = set(self.labels)
        self.indices = {l: i for i, l in enumerate(self.labels)}
        self.ndims = self.vectors.shape[1]

    def save_npz(self):
        npz_path = self.vecs_path.replace('.txt', '.npz')
        np.savez_compressed(npz_path,
                            labels=self.labels,
                            vectors=self.vectors)

    def normalize(self, norm='l2'):
        self.vectors = (self.vectors.T / np.linalg.norm(self.vectors, axis=1)).T

    def get_vec(self, label):
        return self.vectors[self.indices[label]]

    def similarity(self, label1, label2):
        v1 = self.get_vec(label1)
        v2 = self.get_vec(label2)
        return np.dot(v1, v2).tolist()

    def most_similar(self, vec, threshold=0.5, topn=10):
        sims = np.dot(self.vectors, vec).astype(self.dtype)
        sims_ = sims.tolist()
        r = []
        for top_i in sims.argsort().tolist()[::-1][:topn]:
            if sims_[top_i] > threshold:
                r.append((self.labels[top_i], sims_[top_i]))
        return r

    def sims(self, vec):
        return np.dot(self.vectors, np.array(vec)).tolist()

# class CUI_VSM(VSM):

#     def load_txt(self, vecs_path, delimiter='\t'):
#         self.vectors = []
#         with open(vecs_path, encoding='utf-8') as vecs_f:
#             for line_idx, line in enumerate(vecs_f):
#                 label, vals = line.strip().split('\t')  # TO-DO: fix ignored delimiter
#                 self.labels.append(label)
#                 self.vectors.append(np.array(list(map(float, vals.split())), dtype=self.dtype))

#                 if line_idx % 1000000 == 0 and line_idx >= 1000000:  # some output when loading large files
#                     print('Loading vecs - at idx %d' % line_idx)

#         self.vectors = np.vstack(self.vectors)
#         self.indices = {l: i for i, l in enumerate(self.labels)}
#         self.ndims = self.vectors.shape[1]

#         # aux
#         self.label2cui = {l: l.split('%')[-1] for l in self.labels}

#         self.labels_by_st = defaultdict(list)
#         for label in self.labels:
#             st = cui2st(self.label2cui[label])
#             if st is None:  # failed to retrieve st
#                 continue

#             self.labels_by_st[st].append(label)

#     def most_similar_cuis(self, vec, sort=True, restrict_sts=[]):

#         relevant_idxs = list(range(len(self.labels)))
#         relevant_labels = self.labels

#         if len(restrict_sts) > 0:
#             relevant_labels = []
#             for st in restrict_sts:
#                 relevant_labels += self.labels_by_st[st]
#             relevant_idxs = [self.indices[l] for l in relevant_labels]

#         sims = np.dot(self.vectors[relevant_idxs], vec).astype(self.dtype).tolist()

#         sims_cuis = defaultdict(float)
#         for label, score in zip(relevant_labels, sims):
#             cui = self.label2cui[label]
#             if score > sims_cuis[cui]:
#                 sims_cuis[cui] = score

#         r = list(sims_cuis.items())
#         if sort:
#             return sorted(r, key=lambda x: x[1], reverse=True)
#         else:
#             return r

if __name__ == '__main__':

    # p = 'models/VSMs/umls.2017AA.active.st21pv.scibert_scivocab_uncased.cuis.vecs'
    p = 'models/VSMs/umls.2017AA.active.st21pv.en_core_sci_lg.cuis.vecs'
    vsm = FaissVSM()
    # vsm.load()
    print('creating ...')
    vsm.create(p)
    print('saving ...')
    vsm.save(p.replace('.vecs', '.index'), p.replace('.vecs', '.labels'))
    # vsm = VSM(p)


rooms = open("input4.txt").read().split("\n")
rooms.pop()

def fx(rooms):

    idx=0
    while idx!= len(rooms):
        room=rooms[idx]
        idx+=1
        sector_id=int(room[-10:-7])
        room=room[:-11]
        newroom=''
        shift = sector_id
        allletters="abcdefghijklmnopqrstuvwxyz"

        for i in range(len(room)):
            if room[i]=="-": continue
            bubak=allletters.index(room[i])
            bubak+=shift
            bubak%=26
            newroom+=allletters[bubak]

        room=newroom
        if "northpole" in room:
            print(sector_id)

print(fx(rooms))


import cv2
import numpy as np
import matplotlib.pyplot as plt

# Function to convert image to grayscale, apply gaussian blur and detect edges
def canny(image):
    gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
    # To reduce image noise use Gaussian blur (from sharp image get smooth image)
    # cv2.GaussianBlur(input image, kernel matrix, standard deviation)
    blur = cv2.GaussianBlur(gray, (5, 5), 0)
    # Canny edge detection -> search for high value (color) difference between neighbour pixels i.e. edges
    # cv2.Canny (input image, first treshold, second treshhold)
    canny = cv2.Canny(blur, 50, 150)
    return canny

def display_lines(image, lines):
    line_image = np.zeros_like(image) # Black image
    if lines is not None:
        for x1, y1, x2, y2 in lines:
            # Draw lines
            # cv2.line(input image, start point, end point, color, thickness)
            cv2.line(line_image, (x1, y1), (x2, y2), (255,0,0), 10)
    return line_image

# Function to create white triangle and black mask of the input image.
# Apply AND bitwise operation on those 2 to get area of interest (lanes)
def region_of_interest(image):
    height = image.shape[0]
    # Define triangle with its verticies
    polygons = np.array([
    [(200, height), (1100, height), (550, 250)]
    ])
    # Return an array of zeros with the same shape and type as a given array(image)
    # i.e. make black image with the same size of input image
    mask = np.zeros_like(image)
    # On black mask apply white(255) triangle
    cv2.fillPoly(mask, polygons, 255)
    # Isolate area of interest and fill rest with zeros (black)
    # Preform AND operation on elements from 2 arrays (image, mask)
    masked_image = cv2.bitwise_and(image, mask)
    return masked_image

def make_coordinates(image, line_parameters):
    slope, intercept = line_parameters
    y1 = image.shape[0]
    y2 = int(y1*(3/5))
    x1 = int((y1 - intercept) / slope)
    x2 = int((y2 - intercept) / slope)
    return np.array([x1, y1, x2, y2])

def average_slope_intercept(image, lines):
    left_fit = []
    right_fit = []
    for line in lines:
        x1, y1, x2, y2 = line.reshape(4)
        parameters = np.polyfit((x1, x2), (y1, y2), 1)
        slope = parameters[0]
        intercept = parameters[1]
        if slope < 0:
            left_fit.append((slope, intercept))
        else:
            right_fit.append((slope, intercept))
    left_fit_average = np.average(left_fit, axis=0)
    right_fit_average = np.average(right_fit, axis=0)
    left_line = make_coordinates(image, left_fit_average)
    right_line = make_coordinates(image, right_fit_average)
    return np.array([left_line, right_line])

# Load image, make copy of it
image = cv2.imread('test_image.jpg')
lane_image = np.copy(image)
"""
canny_image = canny(lane_image)
cropped_image = region_of_interest(canny_image)
# https://bit.ly/2XQgX7I
# cv2.HoughLinesP(input image, rho, theta, treshold, placeholder array, min length of line, max allowed gap btw lines)
lines = cv2.HoughLinesP(cropped_image, 2, np.pi/180, 100, np.array([]), minLineLength=40, maxLineGap=5)

averaged_lines = average_slope_intercept(lane_image, lines)

line_image = display_lines(lane_image, averaged_lines)

# Blend image to another with defined weights(transparency, translucency)
# https://bit.ly/3iy2gxN
# cv2.addWeighted(original image, alpha, second image, beta, gamma)
combo_image = cv2.addWeighted(lane_image, 0.7, line_image, 1, 1)

# Display result image
cv2.imshow("Image Result", combo_image)
cv2.waitKey(0)
"""
cap = cv2.VideoCapture("test2.mp4")
while(cap.isOpened()):
    _, frame = cap.read()
    canny_image = canny(frame)
    cropped_image = region_of_interest(canny_image)
    lines = cv2.HoughLinesP(cropped_image, 2, np.pi/180, 100, np.array([]), minLineLength=40, maxLineGap=5)
    averaged_lines = average_slope_intercept(frame, lines)
    line_image = display_lines(frame, averaged_lines)
    combo_image = cv2.addWeighted(frame, 0.7, line_image, 1, 1)
    cv2.imshow("Video Result", combo_image)
    if cv2.waitKey(1) == ord('q'):
        break
cap.release()
cv2.destroyAllWindows()


import PIL
from PIL import Image, ImageDraw, ImageColor, ImageTk

class Render:

    @staticmethod
    def draw_grid(color: str = None):
        if color is None:
            color = 'green'

        img = Image.new(mode="RGB", size=(612, 612), color='black')

        image = ImageDraw.Draw(img)

        for i in range(1, 5):
            x, y = 123 * i, 123 * i
            image.line((x-2, 0, x-2, 612), width=3, fill=ImageColor.getrgb(color))
            image.line((0, y-2, 612, y-2), width=3, fill=ImageColor.getrgb(color))

        return img

    @staticmethod
    def render_unit(img, x: int, y: int):
        img.paste(Image.open("./assets/test.png"), (x,y))
        return img


#!/usr/bin/python3

from personal_data import user_id
from datetime import datetime
import requests
import json
import _phue
import logging
from numpy import mean
import scene
import data
from time import sleep
import toolbox
import sys

log_file = '/home/pi/ambient.log'
MASTER = 31

def trim_logs():
    with open(log_file, "r") as file:
        lines = file.readlines()

    max_number_of_lines = 1 * 24 * 60

    if len(lines) > max_number_of_lines:
        with open(log_file, "w") as file:
            file.writelines(lines[-max_number_of_lines:])

def __get_new(i=MASTER):
    response = requests.get("http://%s/api/%s/sensors/%d" % (
                            _phue.ip_address, user_id, i))
    json_data = json.loads(response.text)
    return json_data["state"]["lightlevel"]

def __get_history(i=MASTER, number=10):
    with open(log_file, "r") as file:
        lines = file.readlines()
    if len(lines) == 0:
        sleep(.1)
        with open(log_file, "r") as file:
            lines = file.readlines()

    if number != float('inf'):
        lines = lines[-number:]
    lines_copy = lines
    try:
        lines = [l.replace("\n", "") for l in lines]
        lines = [l[26:] for l in lines]
        lines = [l.split(";") for l in lines]
        lines = [[jj.replace(" ", "") for jj in j if jj] for j in lines]
        lines = [[jj for jj in j if jj] for j in lines]
        data = dict()
        for l in lines:
            for element in l:
                key, value = element.split(':')
                key = int(key)
                value = float(value)
                if key not in data:
                    data[key] = list()
                data[key].append(value)
        return data[i]
    except:
        print(lines_copy)
        raise

def get_history_mean(i=MASTER, number=10):
    return mean(__get_history(i, number))

def get_simulated_bri():
    g = __get_new()
    bri = toolbox.map(g, 1000, 36000, 0, 1)
    bri = min(bri, 1)
    bri = max(bri, 0)
    return bri

def get_simulated_ct(maximum=1.0, minimum=.35):
    mean = get_history_mean(number=60)
    ct = toolbox.map(mean, 0, 33000, maximum, minimum)
    ct = min(ct, maximum)
    ct = max(ct, minimum)
    return ct

def get_schmitt_trigger(low=16000, high=20000):
    history = __get_history(number=float('inf'))
    for j in history[::-1]:
        if j <= low:
            return False
        elif high <= j:
            return True

def log_all():
    message = ""
    for i in [11, 31, 34, 82]:
        lightlevel = __get_new(i)

        message += str(i)
        message += ": "
        message += str(lightlevel)
        message += "; "

    logging.basicConfig(filename=log_file,
                        encoding='utf-8',
                        level=logging.INFO,
                        format='%(asctime)s   %(message)s')

    logging.info(message)
    trim_logs()

def __minutes_in_current_day():
    now = datetime.now()
    midnight = now.replace(hour=0, minute=0, second=0, microsecond=0)
    minutes_passed = (now - midnight).total_seconds() // 60
    return int(minutes_passed)

limit_ambient = 28000.0
limit_minutes = int(2 * 60)
def should_turn_off():
    return False
    window_minutes = __minutes_in_current_day()
    if window_minutes < limit_minutes:
        return False
    history = __get_history(number=window_minutes)
    history_before = history[:-1]
    nof_elements = len([h for h in history if limit_ambient <= h])
    nof_elements_before = len([h for h in history_before if limit_ambient <= h])
    return (nof_elements == limit_minutes and
            nof_elements_before == limit_minutes - 1)

def should_be_off():
    return False
    window_minutes = __minutes_in_current_day()
    if window_minutes < limit_minutes:
        return False
    history = __get_history(number=window_minutes)
    nof_elements = len([h for h in history if limit_ambient <= h])
    return limit_minutes <= nof_elements

def turn_off_if_ambient_above_limit():
    if should_turn_off():
        scene.transition('off_wohnzimmer', time=10*60)

def auto_ct_enabled():
    try:
        with open('/home/pi/auto_ct_enabled', 'r') as file:
            content = file.read().strip()
        return content.lower() == "true"
    except:
        return False

def enable_auto_ct():
    with open('/home/pi/auto_ct_enabled', 'w') as f:
        f.write('true')

def disable_auto_ct():
    with open('/home/pi/auto_ct_enabled', 'w') as f:
        f.write('false')

def auto_ct():
    if auto_ct_enabled():
        ct_value = get_simulated_ct(maximum=1.0, minimum=0.0)
        for z in ["schlafzimmer", "wohnzimmer"]:
            do_this_room = True
            for light, light_data in eval("data." + z + "_lights"):
                if "bri" in light_data:
                    threshold = data.warm[light]["bri"] - 0.02
                    if _phue.get_bri(light) < threshold:
                        do_this_room = False
            if do_this_room:
                s = toolbox.scene_superposition(ct_value,
                                                eval("data.warm_" + z),
                                                1.0 - ct_value,
                                                eval("data.hell_" + z))
                scene.transition_dicionary(s, time=59)

def main():
    log_all()
    turn_off_if_ambient_above_limit()
    auto_ct()
    trim_logs()

if __name__ == "__main__":
    if len(sys.argv) == 1:
        main()
    else:
        for a in sys.argv[1:]:
            f = globals()[a.strip("-").replace("-", "_")]
            f()


import re
from telegram import ForceReply, Update
from telegram.ext import Application, CommandHandler, ContextTypes, MessageHandler, filters
import logging
MY_TOKEN = "YOUR TOKEN"

logging.basicConfig(
    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=logging.INFO
)
logger = logging.getLogger(__name__)

def clearExpresiion(input_str: str):
    return re.sub(r'[^\d+=*%^/\(\)\-]', "", input_str)

def calc(expresiion: str):
    cleared_expresion = clearExpresiion(expresiion)
    return eval(cleared_expresion)

async def start(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
    """Send a message when the command /start is issued."""
    logger.info("Start")
    await update.message.reply_text(
        "Добро пожаловать в калькулятор, введите выражение для расчета, например: 1 + 2"
    )

async def echo(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
    logger.info("User input {}".format(update.message.text))
    result = calc(update.message.text)
    logger.info("Result is: {}".format(update.message.text))
    message = f"Результат выражения: {result}"
    await update.message.reply_text(message)

def main() -> None:
    """Start the bot."""
    logger.info("Bot started!")
    # Create the Application and pass it your bot's token.
    application = Application.builder().token(MY_TOKEN).build()

    # on different commands - answer in Telegram
    application.add_handler(CommandHandler("start", start))

    # on non command i.e message - echo the message on Telegram
    application.add_handler(MessageHandler(
        filters.TEXT & ~filters.COMMAND, echo))

    # Run the bot until the user presses Ctrl-C
    application.run_polling()

if __name__ == "__main__":
    main()


#!/usr/bin/python3.7
from itertools import repeat
from string import ascii_uppercase
from json import dumps

def analyse_freq(text):
    freq = dict(zip(ascii_uppercase, repeat(0)))
    most_common = ['E', 'T', 'A', 'O', 'I', 'N', 'S', 'R', 'H', 'D', 'L', 'U', 'C', 'M', 'F', 'W', 'G', 'Y', 'P', 'B', 'V', 'K', 'X', 'J', 'Q', 'Z']
    for i in text:
        freq[i]+=1
    s = sorted(freq, key=(lambda x: freq[x]), reverse=True)
    return dict(zip(s, most_common))

texts = [open("spam/{}".format(i)).read() for i in range(7)]

for i, mapping in enumerate(map(analyse_freq, texts)):
    open("mappings/{}".format(i), "w").write(dumps(mapping))


import secrets

from rest_framework import serializers
from rest_framework.validators import UniqueTogetherValidator

from .models import *

class BuildingSerializer(serializers.ModelSerializer):
    id = serializers.ReadOnlyField()

    class Meta:
        model = Building
        fields = [
            'id',
            'owner',
            'building_name',
            'street',
            'area',
            'city',
            'state',
            'zip_code',
            'landmark',
            'latitude',
            'longitude',
            'gender_label',
            'display_pic',
            'in_time'
        ]

    def create(self, validated_data):
        display_pic = None
        if validated_data['display_pic']:
            display_pic = validated_data['display_pic']
            ext = display_pic.name.split('.')[-1]
            display_pic.name = secrets.token_urlsafe(30) + '.' + ext

        building = Building.objects.create(
            owner=validated_data['owner'],
            building_name=validated_data.get('building_name'),
            street=validated_data.get('street'),
            area=validated_data.get('area'),
            city=validated_data.get('city'),
            state=validated_data.get('state'),
            zip_code=validated_data.get('zip_code'),
            landmark=validated_data.get('landmark'),
            latitude=validated_data.get('latitude'),
            longitude=validated_data.get('longitude'),
            gender_label=validated_data.get('gender_label'),
            display_pic=display_pic,
            in_time=validated_data.get('in_time')
        )
        return building

    def update(self, instance, validated_data):
        if validated_data.get('display_pic') != instance.display_pic:
            display_pic = validated_data.get('display_pic')
            ext = display_pic.name.split('.')[-1]
            display_pic.name = secrets.token_urlsafe(30) + '.' + ext
            validated_data['display_pic'] = display_pic
            instance.display_pic = validated_data.get('display_pic', instance.display_pic)

        instance.gender_label = validated_data.get('gender_label', instance.gender_label)
        instance.landmark = validated_data.get('landmark', instance.landmark)
        instance.in_time = validated_data.get('in_time', instance.in_time)
        instance.save()
        return instance

class RoomSerializer(serializers.ModelSerializer):
    id = serializers.ReadOnlyField()

    class Meta:
        model = Room
        fields = [
            'id',
            'building',
            'description',
            'title',
            'rent',
            'total',
            'available',
            'occupancy',
            'is_verified',
        ]

    def create(self, validated_data):

        room = Room.objects.create(
            building=validated_data['building'],
            title=validated_data['title'],
            rent=validated_data['rent'],
            total=validated_data['total'],
            description=validated_data['description'],
            available=validated_data['available'],
            occupancy=validated_data['occupancy'],
            is_verified=False,
        )
        return room

    def update(self, instance, validated_data):
        instance.title = validated_data.get('title', instance.title)
        instance.rent = validated_data.get('rent', instance.rent)
        instance.description = validated_data.get('description', instance.description)
        instance.occupancy = validated_data.get('occupancy', instance.occupancy)
        instance.save()
        return instance

class PropertyDeedSerializer(serializers.ModelSerializer):
    id = serializers.ReadOnlyField()

    class Meta:
        model = PropertyDeed
        fields = [
            'id',
            'room',
            'owner',
            'registration_no',
            'issue_date',
            'expiry_date',
            'document'
        ]
        validators = [
            UniqueTogetherValidator(queryset=PropertyDeed.objects.all(), fields=['registration_no']),
            UniqueTogetherValidator(queryset=PropertyDeed.objects.all(), fields=['room'])
        ]

    def create(self, validated_data):
        document = None
        if validated_data['document']:
            document = validated_data['document']
            ext = document.name.split('.')[-1]
            document.name = secrets.token_urlsafe(30) + '.' + ext

        property_deed = PropertyDeed.objects.create(
            room=validated_data['room'],
            owner=validated_data['owner'],

            registration_no=validated_data['registration_no'],
            issue_date=validated_data['issue_date'],
            expiry_date=validated_data['expiry_date'],
            document=document
        )
        return property_deed

class BuildingPhotoSerializer(serializers.ModelSerializer):
    class Meta:
        model = BuildingPhoto
        read_only_fields = ['id', 'caption', 'photo']
        fields = ['id', 'caption', 'photo']

class PerkSerializer(serializers.ModelSerializer):
    class Meta:
        model = Perk
        read_only_fields = ['id', 'description']
        fields = ['id', 'description']

class BookingSerializer(serializers.ModelSerializer):
    id = serializers.ReadOnlyField()

    class Meta:
        model = Booking
        read_only_fields = ['id', 'booking_no', 'room', 'user', 'booking_date']
        fields = ['id', 'booking_no', 'room', 'user', 'booking_date']

class BookmarkSerializer(serializers.ModelSerializer):
    id = serializers.ReadOnlyField()

    class Meta:
        model = Bookmark
        fields = [
            'id',
            'user',
            'building'
        ]
        validators = [
            UniqueTogetherValidator(queryset=Bookmark.objects.all(), fields=['user', 'building'])
        ]

    def create(self, validated_data):
        bookmark = Bookmark.objects.create(
            user=validated_data['user'],
            building=validated_data['building']
        )
        return bookmark


from builtins import breakpoint
import cv2
import matplotlib.pyplot as plt
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
from matplotlib.figure import Figure
import numpy as np
import torch
import torchvision
import io
import matplotlib.pyplot as plt
import PIL.Image
from torchvision.transforms import ToTensor
from .transformers import make_affine_parameters

def convert_image_np(inp, dataset='mnist'):
    """Convert a Tensor to numpy image."""
    if dataset == 'mnist':
        mean, std = 0.1307, 0.3081
    else:
        mean = np.array([0.485, 0.456, 0.406])
        std = np.array([0.229, 0.224, 0.225])

    # convert to numpy (C,H,W) -> (H,W,C)
    inp = inp.numpy().transpose((1, 2, 0))

    # normalize
    inp = std * inp + mean
    inp = np.clip(inp, 0, 1)

    return inp

# def gen_plot_input(input_array, nr_plots):
#     """Create a pyplot plot and save to buffer."""
#     plt.figure()
#     for ix in range(nr_plots):
#         plt.subplot(nr_plots, 1, ix+1)
#         plt.plot(input_array[ix, 0, :], c='darkblue')
#     buf = io.BytesIO()
#     plt.savefig(buf, format='jpeg')
#     buf.seek(0)
#     return buf

def gen_plot_input(transformed_array, nr_plots):
    plt.figure()
    colors = ['darkgoldenrod', 'sienna', 'darkred']
    for ix in range(nr_plots):
        plt.subplot(nr_plots, 1, ix+1)
        plt.plot(transformed_array[ix, 0, :], c=colors[ix])
    buf = io.BytesIO()
    plt.savefig(buf, format='jpeg')
    buf.seek(0)
    return buf

def gen_plot_output(transformed_array, nr_plots, batch_size):
    plt.figure()
    # colors = ['darkgreen', 'darkgoldenrod', 'darkred']
    nr_samples = 3
    alphas = [0.5, 0.75, 1]
    # Plot the results side-by-side
    for data_ix in range(nr_plots):
        plt.subplot(nr_plots, 1, data_ix+1)
        for sample_ix in range(nr_samples):
            plt.plot(transformed_array[data_ix + (sample_ix * batch_size), 0, :],
                     alpha=alphas[sample_ix], c='sienna')
    buf = io.BytesIO()
    plt.savefig(buf, format='jpeg')
    buf.seek(0)
    return buf

def normalize(images):
    return images / torch.max(images)

def visualize_stn(model, data, x_high_res, opt):
    with torch.no_grad():

        # IMAGE VISUALIZATION
        if opt.xdim == 2:
            data = data[:16]  # just visualize the first 16
            in_grid = convert_image_np(torchvision.utils.make_grid(data.cpu()), opt.dataset.lower())
            in_grid = (in_grid * 255).astype(np.uint8)
            in_grid = np.transpose(in_grid, (2, 0, 1))
            if opt.model.lower() == 'cnn':
                return in_grid, None, None, None

            if x_high_res is not None:
                x_high_res = x_high_res[:16]  # just visualize the first 16

            if opt.model.lower() == 'stn':
                transformed_input_tensor, thetas = model.forward_localizer(data, x_high_res)

            elif opt.model.lower() == 'pstn':
                transformed_input_tensor, thetas, _ = model.forward_localizer(data, x_high_res)

            out_grid = convert_image_np(torchvision.utils.make_grid(transformed_input_tensor.cpu()),
                                        opt.dataset.lower())
            out_grid = (out_grid * 255).astype(np.uint8)
            out_grid = np.transpose(out_grid, (2, 0, 1))

            bbox_images = None
            if opt.num_param < 4 and opt.num_param > 1:
                affine_params = make_affine_parameters(thetas)
                bbox_images = visualize_bbox(data.cpu(), affine_params, opt)
            # TODO: implement bounding boxed for 4 param trafos

            return in_grid, out_grid, thetas, bbox_images

        # TIMESERIES VISUALIZATIONS
        if opt.xdim == 1:
            batch_size = data.shape[0]
            nr_plots = min(3, batch_size)  # just visualize the first 3
            input_array = data.cpu().numpy()
            plt.close(); plt.close(); plt.close()
            # Prepare the input plot
            plot_buf = gen_plot_input(input_array, nr_plots)
            image = PIL.Image.open(plot_buf)
            image = ToTensor()(image)
            plt.close(); plt.close(); plt.close()

            if opt.model.lower() == 'cnn':
                return image, None, None, None

            if opt.model.lower() == 'stn':
                transformed_input, theta = model.forward_localizer(data, x_high_res=None)
                transformed_input = transformed_input.cpu().numpy()
                # prepare the STN output plot
                out_plot_buf = gen_plot_input(transformed_input, nr_plots)
                out_image = PIL.Image.open(out_plot_buf)
                out_image = ToTensor()(out_image)
                return image, out_image, theta, None

            elif opt.model.lower() == 'pstn':
                transformed_input, sampled_theta, _ = model.forward_localizer(data, x_high_res=None)
                transformed_input = transformed_input.cpu().numpy()
                # prepare the P-STN output plot
                out_plot_buf = gen_plot_output(transformed_input, nr_plots, batch_size)
                out_image = PIL.Image.open(out_plot_buf)
                out_image = ToTensor()(out_image)
                return image, out_image, sampled_theta, None

def visualize_bbox(data, affine_params, opt):
    if opt.transformer_type == 'diffeomorphic':
        return None

    batch_size = data.shape[0]
    affine_params = torch.stack(affine_params.split([opt.N] * opt.test_samples * batch_size))
    sorted_params = torch.zeros(batch_size, opt.test_samples, opt.N, 2, 3, dtype=affine_params.dtype)
    for i in range(len(affine_params)):
        # im, sample, N, 2, 3
        sorted_params[i % batch_size, i // batch_size, :, :, :] = affine_params[i, :, :]

    images = []
    for j, (im, params) in enumerate(zip(data, sorted_params)):

        im = np.transpose(im.numpy(), (1, 2, 0))
        im = denormalize(im)

        if im.shape[2] == 1:
            im = np.stack((im[:, :, 0],) * 3, axis=-1)

        if torch.isnan(params).any(): continue

        im = add_bounding_boxes(im, params, opt.N, opt.test_samples, mode_='crop', heatmap=opt.heatmap)

        images.append(np.transpose(im, (2, 0, 1)))

    if len(images) > 0:

        images = torch.FloatTensor(images)
        if 'mnist' in opt.dataset.lower():
            images = convert_image_np(torchvision.utils.make_grid(images - 0.3), opt.dataset.lower())
            images = (images * 255).astype(np.uint8)
            images = np.transpose(images, (2, 0, 1))
        else:
            images = torchvision.utils.make_grid(images, normalize=True)

        return images

    return None

def denormalize(image):
    im = (image - np.min(image)) / (np.max(image) - np.min(image))
    return im

def add_bounding_boxes(image, affine_params, num_branches, num_samples, mode_='crop', heatmap=True):
    color = [(255, 0, 0), (0, 255, 0), (0, 0, 255), (255, 255, 0), (255, 0, 255), (0, 255, 255)]

    image *= 255
    im = image.astype(np.uint8).copy()

    if mode_ == 'crop':
        w = int(im.shape[0])
        h = int(im.shape[1])

    for j in range(num_samples):
        for i in range(num_branches):
            if mode_ == 'crop':
                x = affine_params[j, i, 0, 2]
                y = affine_params[j, i, 1, 2]
                s_x = affine_params[j, i, 0, 0]
                s_y = affine_params[j, i, 1, 1]

                # define bbox by top left corner and define coordinates system with origo in top left corner
                x = int((x + 1 - s_x) * w // 2)
                y = int((y + 1 - s_y) * h // 2)

                # 1. (x + 1) * w / 2
                # 2. - s_x * w / 2

                # => (x + 1 - s_x) * w / 2

                if heatmap:

                    overlay = im.copy()
                    cv2.rectangle(overlay, (x, y), (x + w // 2, y + h // 2), color[i % len(color)],
                                  -1)  # A filled rectangle

                    alpha = 0.05  # Transparency factor.

                    # Following line overlays transparent rectangle over the image
                    im = cv2.addWeighted(overlay, alpha, im, 1 - alpha, 0)
                else:
                    cv2.rectangle(im, (x, y), (x + int(w * s_x), y + int(h * s_y)), color[i % len(color)], 1)

    return im


#!/usr/lib/env python
#
# Module Documentation Override Check
#
# Verifies that any modules present in the _templates/overrides directory have all of their
# their exported functions included in the doc file.
import importlib
from pathlib import Path
import sys

modules_to_skip = ['metpy.xarray']

failed = False
for full_path in (Path('_templates') / 'overrides').glob('metpy.*.rst'):

    module = full_path.with_suffix('').name
    if module in modules_to_skip:
        continue

    # Get all functions in the module
    i = importlib.import_module(module)
    functions = set(i.__all__)

    # Get all lines in the file
    with open(full_path) as f:
        lines = f.read().splitlines()
    lines = {line.strip() for line in lines}

    # Check for any missing functions
    missing_functions = functions - lines

    if missing_functions:
        failed = True
        print('ERROR - The following functions are missing from the override file ' +
              str(full_path.name) + ': ' + ', '.join(missing_functions), file=sys.stderr)

# Report status
if failed:
    sys.exit(1)
else:
    print('Override check successful.')


import cv2
import numpy as np

image = cv2.imread('sample.jpg', 0)
cv2.imshow("Original", image)

ret, thresh1 = cv2.threshold(image, 127, 255, cv2.THRESH_BINARY)
cv2.imshow('Binary Threshold', thresh1)

ret, thresh2 = cv2.threshold(image, 127, 255, cv2.THRESH_BINARY_INV)
cv2.imshow('Threshold Binary Inverse', thresh2)

ret, thresh3 = cv2.threshold(image, 127, 255, cv2.THRESH_TRUNC)
cv2.imshow('Thresh Trunc', thresh3)

ret, thresh4 = cv2.threshold(image, 127, 255, cv2.THRESH_TOZERO)
cv2.imshow('Threshold zero', thresh4)

ret, thresh5 = cv2.threshold(image, 127, 255, cv2.THRESH_TOZERO_INV)
cv2.imshow('Threshold zero inv', thresh5)
cv2.waitKey()
cv2.destroyAllWindows()

blur_img = cv2.GaussianBlur(image, (3, 3), 0)

#adaptive threshsold
adaptive_threshold = cv2.adaptiveThreshold(image, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 3, 5)
cv2.imshow("Adaptive Mean Thresholding", adaptive_threshold)
cv2.waitKey()

#Otsu's Thresholding
_, otsus_threshold = cv2.threshold(image, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
cv2.imshow("Otsu's Threshold", otsus_threshold)
cv2.waitKey()

cv2.destroyAllWindows()


import unittest
import os.path
import os
from datetime import datetime
from models.engine.file_storage import FileStorage
from models import *
from console import HBNBCommand

@unittest.skipIf(os.getenv('HBNB_TYPE_STORAGE', '') == "db", "db")
class Test_FileStorage(unittest.TestCase):
    """
    Test the file storage class
    """

    def setUp(self):
        self.cli = HBNBCommand()
        self.store = FileStorage()

        test_args = {'updated_at': datetime(2017, 2, 12, 00, 31, 53, 331997),
                     'id': 'f519fb40-1f5c-458b-945c-2ee8eaaf4900',
                     'created_at': datetime(2017, 2, 12, 00, 31, 53, 331900)}
        self.model = BaseModel(test_args)

        self.test_len = 0
        if os.path.isfile("file.json"):
            self.test_len = len(self.store.all())

    def test_all(self):
        self.assertEqual(len(self.store.all()), self.test_len)

    def test_new(self):
        self.assertEqual(len(self.store.all()), self.test_len)
        self.model.save()
        self.assertEqual(len(self.store.all()), self.test_len + 1)
        a = BaseModel()
        a.save()
        self.assertEqual(len(self.store.all()), self.test_len + 2)
        self.cli.do_destroy("BaseModel f519fb40-1f5c-458b-945c-2ee8eaaf4900")
        self.cli.do_destroy("BaseModel " + a.id)

    def test_save(self):
        self.test_len = len(self.store.all())
        a = BaseModel()
        a.save()
        self.assertEqual(len(self.store.all()), self.test_len + 1)
        b = User()
        self.assertNotEqual(len(self.store.all()), self.test_len + 2)
        b.save()
        self.assertEqual(len(self.store.all()), self.test_len + 2)
        self.cli.do_destroy("BaseModel " + a.id)
        self.cli.do_destroy("User " + b.id)

    def test_reload(self):
        pass

if __name__ == "__main__":
    unittest.main()


"""
AIML Bot implements an interpreter for AIML, the Artificial Intelligence
Markup Language developed by Dr. Richard Wallace of the A.L.I.C.E. Foundation.
It can be used to implement a conversational AI program.
"""

from setuptools import setup
import glob
import os
import warnings

package_prefix = "Lib/site-packages/aiml_bot"

def get_long_description():
    """Load the long description from the README file. In the process,
    convert the README from .md to .rst using Pandoc, if possible."""
    rst_path = os.path.join(os.path.dirname(__file__), 'README.rst')
    md_path = os.path.join(os.path.dirname(__file__), 'README.md')

    try:
        # Imported here to avoid creating a dependency in the setup.py
        # if the .rst file already exists.

        # noinspection PyUnresolvedReferences,PyPackageRequirements
        from pypandoc import convert_file
    except ImportError:
        warnings.warn("Module pypandoc not installed. Unable to generate README.rst.")
    else:
        # First, try to use convert_file, assuming Pandoc is already installed.
        # If that fails, try to download & install it, and then try to convert
        # again.
        # noinspection PyBroadException
        try:
            # pandoc, you rock...
            rst_content = convert_file(md_path, 'rst')
            with open(rst_path, 'w') as rst_file:
                for line in rst_content.splitlines(keepends=False):
                    rst_file.write(line + '\n')
        except Exception:
            try:
                # noinspection PyUnresolvedReferences,PyPackageRequirements
                from pypandoc.pandoc_download import download_pandoc

                download_pandoc()
            except FileNotFoundError:
                warnings.warn("Unable to download & install pandoc. Unable to generate README.rst.")
            else:
                # pandoc, you rock...
                rst_content = convert_file(md_path, 'rst')
                with open(rst_path, 'w') as rst_file:
                    for line in rst_content.splitlines(keepends=False):
                        rst_file.write(line + '\n')

    if os.path.isfile(rst_path):
        with open(rst_path) as rst_file:
            return rst_file.read()
    else:
        # It will be messy, but it's better than nothing...
        with open(md_path) as md_file:
            return md_file.read()

setup(
    name="AIML Bot",
    version="0.0.3",
    author="Cort Stratton",
    author_email="cort@cortstratton.org",
    license="BSD-2-Clause",
    maintainer="Aaron Hosford",
    maintainer_email="hosford42@gmail.com",
    description="An interpreter package for AIML, the Artificial Intelligence Markup Language",
    long_description=get_long_description(),
    url="https://github.com/hosford42/aiml_bot",

    platforms=["any"],
    classifiers=[
        "Development Status :: 3 - Alpha",
        "Environment :: Console",
        "Intended Audience :: Developers",
        "Programming Language :: Python :: 3",
        "Operating System :: OS Independent",
        "Topic :: Communications :: Chat",
        "Topic :: Scientific/Engineering :: Artificial Intelligence"
    ],

    packages=["aiml_bot"],
    package_data={
        "aiml_bot": ["*.aiml"],
    },
    data_files=[
        ("doc/aiml_bot", glob.glob("*.txt")),
        ("doc/aiml_bot", glob.glob("*.md")),
    ]
)