import json
import os
import re
import time
import yaml
from datetime import timedelta

from api.iotrace_lat_hist_parser import LatencyHistograms
from core.test_run_utils import TestRun
from test_tools.fs_utils import check_if_directory_exists, create_directory
from test_utils.output import CmdException
from test_utils.size import Unit, Size
from utils.git import get_current_commit_hash, get_current_commit_message
from utils.git import get_current_octf_hash
from log.logger import create_log, Log

NOT_WHITESPACE = re.compile(r'[^\s]')

class IotracePlugin:
    def __init__(self):
        # Test controller's repo, copied to DUT
        self.repo_dir = os.path.join(os.path.dirname(__file__), "../../")

        # DUT's make/install work directory
        self.working_dir = "~"

        # Flag indicating if install iotrace
        self.reinstall = False

        self.pid = None

    def runtest_setup(self, item) -> object:
        try:
            with open(item.config.getoption('--dut-config')) as cfg:
                dut_config = yaml.safe_load(cfg)
        except Exception:
            raise Exception("You need to specify DUT config. "
                            "See the example_dut_config.py file.")

        try:
            TestRun.prepare(item, dut_config)

            test_name = item.name.split('[')[0]
            TestRun.LOGGER = create_log(item.config.getoption('--log-path'), test_name)
            TestRun.setup()

            self.working_dir = dut_config['working_dir']
            self.reinstall = item.config.getoption('--force-reinstall')
        except Exception as ex:
            raise Exception(f"Exception occurred during test setup:\n"
                            f"{str(ex)}\n{traceback.format_exc()}")

        TestRun.LOGGER.info(f"DUT info: {TestRun.dut}")

        # Prepare DUT for test
        TestRun.LOGGER.add_build_info(f'Commit hash:')
        TestRun.LOGGER.add_build_info(f"{get_current_commit_hash()}")
        TestRun.LOGGER.add_build_info(f'Commit message:')
        TestRun.LOGGER.add_build_info(f'{get_current_commit_message()}')
        TestRun.LOGGER.add_build_info(f'OCTF commit hash:')
        TestRun.LOGGER.add_build_info(f'{get_current_octf_hash()}')

        TestRun.LOGGER.info(f"DUT info: {TestRun.dut}")
        TestRun.LOGGER.write_to_command_log("Test body")
        TestRun.LOGGER.start_group("Test body")

    @staticmethod
    def runtest_teardown():
        TestRun.LOGGER.end_all_groups()

        with TestRun.LOGGER.step("Cleanup after test"):
            try:
                if TestRun.executor.is_active():
                    TestRun.executor.wait_for_connection()
            except Exception:
                TestRun.LOGGER.warning("Exception occured during platform cleanup.")

        TestRun.LOGGER.end()
        if TestRun.executor:
            TestRun.LOGGER.get_additional_logs()
        Log.destroy()
        TestRun.teardown()

    def start_tracing(self,
                      bdevs: list = [],
                      buffer: Size = None,
                      trace_file_size: Size = None,
                      timeout: timedelta = None,
                      label: str = None,
                      shortcut: bool = False):
        """
        Start tracing given block devices. Trace all available if none given.

        :param bdevs: Block devices to trace, can be empty
        (for all available)
        :param buffer: Size of the internal trace buffer in MiB
        :param trace_file_size: Max size of trace file in MiB
        :param timeout: Max trace duration time in seconds
        :param label: User defined custom label
        :param shortcut: Use shorter command
        :type bdevs: list of strings
        :type buffer: Size
        :type trace_file_size: Size
        :type timeout: timedelta
        :type label: str
        :type shortcut: bool
        """

        if len(bdevs) == 0:
            disks = TestRun.dut.disks
            for disk in disks:
                bdevs.append(disk.system_path)

        buffer_range = range(1, 1025)
        trace_file_size_range = range(1, 100000001)
        timeout_range = range(1, 4294967296)

        command = 'iotrace' + (' -S' if shortcut else ' --start-tracing')
        command += (' -d ' if shortcut else ' --devices ') + ','.join(bdevs)

        if buffer is not None:
            if not int(buffer.get_value(Unit.MebiByte)) in buffer_range:
                raise CmdException(f"Given buffer is out of range {buffer_range}.")
            command += ' -b ' if shortcut else ' --buffer '
            command += f'{int(buffer.get_value(Unit.MebiByte))}'

        if trace_file_size is not None:
            if not int(trace_file_size.get_value(Unit.MebiByte)) in trace_file_size_range:
                raise CmdException(f"Given size is out of range {trace_file_size_range}.")
            command += ' -s ' if shortcut else ' --size '
            command += f'{int(trace_file_size.get_value(Unit.MebiByte))}'

        if timeout is not None:
            if not int(timeout.total_seconds()) in timeout_range:
                raise CmdException(f"Given time is out of range {timeout_range}.")
            command += ' -t ' if shortcut else ' --time '
            command += f'{int(timeout.total_seconds())}'

        if label is not None:
            command += ' -l ' if shortcut else ' --label ' + f'{label}'

        self.pid = str(TestRun.executor.run_in_background(command))
        TestRun.LOGGER.info("Started tracing of: " + ','.join(bdevs))
        # Make sure there's a >0 duration in all tests
        time.sleep(2)

    @staticmethod
    def get_trace_repository_path(shortcut: bool = False) -> list:
        """
        Get the path to trace repository from iotrace

        :param shortcut: Use shorter command
        :type shortcut: bool
        :return: list of dictionaries with trace repository path
        :raises Exception: when cannot find the path
        """
        command = 'iotrace' + (' -C' if shortcut else ' --trace-config')
        command += ' -G ' if shortcut else ' --get-trace-repository-path '

        return parse_json(TestRun.executor.run_expect_success(command).stdout)[0]["path"]

    def check_if_tracing_active(self) -> bool:
        """
        Check if tracing is active

        :return: True if correct iotrace process found, False otherwise
        :rtype: bool
        """
        output = TestRun.executor.run('pgrep iotrace')

        if output.stdout == "":
            TestRun.LOGGER.info("Iotrace processes not found.")
            return False

        elif self.pid is not None and self.pid not in output.stdout:
            TestRun.LOGGER.info(f"Found other iotrace process with PID {output.stdout}")
            return False

        else:
            return True

    def stop_tracing(self) -> bool:
        """
        Stop tracing.

        :return: True if tracing was stopped, False no tracing was active
        :rtype: bool
        """
        TestRun.LOGGER.info("Stopping tracing")

        if self.check_if_tracing_active():
            TestRun.LOGGER.info(f"Sending termination signal to process {self.pid}")
            TestRun.executor.run(f'kill -2 {self.pid}')
            TestRun.executor.wait_cmd_finish(pid=self.pid, timeout=timedelta(seconds=60))

        if self.check_if_tracing_active():
            TestRun.LOGGER.info("Cannot kill iotrace, force kill")
            TestRun.executor.kill_process(self.pid)
            TestRun.fail("Cannot not stop iotrace")
            return False

        return True

    def kill_tracing(self) -> bool:
        """
        Kill tracing.

        :return: True if tracing was killed
        :rtype: bool
        :raises Exception: if failed to kill iotrace process
        """
        TestRun.LOGGER.info("Killing tracing")

        # Send -9
        kill_attempts = 30
        attempt = 0

        while self.check_if_tracing_active() and attempt < kill_attempts:
            TestRun.LOGGER.info("Sending -9 no. " + str(attempt + 1))
            attempt += 1
            TestRun.executor.run(f'kill -9 {self.pid}')
            time.sleep(2)

        if self.check_if_tracing_active():
            raise CmdException("Could not kill iotrace")

        return True

    @staticmethod
    def get_traces_list(prefix: str = None, shortcut: bool = False) -> list:
        """
        Returns list of traces

        :param prefix: Trace path or trace path's prefix when ended with '*'
        :param shortcut: Use shorter command
        :type prefix: str
        :type shortcut: bool
        :return: list of traces
        """
        command = 'iotrace' + (' -M' if shortcut else ' --trace-management')
        command += ' -L ' if shortcut else ' --list-traces '

        if prefix is not None:
            command += (' -p ' if shortcut else ' --prefix ') + f'{prefix}'
        output = TestRun.executor.run_expect_success(command)

        return parse_json(output.stdout)[0]["trace"]

    @staticmethod
    def get_trace_count(prefix: str = None, shortcut: bool = False) -> int:
        """
        Returns number of traces

        :param prefix: Trace path or trace path's prefix when ended with '*'
        :param shortcut: Use shorter command
        :type prefix: str
        :type shortcut: bool
        :return: number of traces
        """
        paths_parsed = IotracePlugin.get_traces_list(prefix, shortcut)
        return len(paths_parsed)

    @staticmethod
    def get_latest_trace_path(prefix: str = None, shortcut: bool = False) -> list:
        """
        Returns trace path of the most recent trace

        :param prefix: Trace path or trace path's prefix when ended with '*'
        :param shortcut: Use shorter command
        :type prefix: str
        :type shortcut: bool
        :return: trace path
        """
        paths_parsed = IotracePlugin.get_traces_list(prefix, shortcut)

        # Return the last element of trace list
        if len(paths_parsed):
            return paths_parsed[-1]['tracePath']
        else:
            return ""

    @staticmethod
    def get_trace_summary(trace_path: str, shortcut: bool = False) -> list:
        """
        Get trace summary of given trace path

        :param trace_path: trace path
        :param shortcut: Use shorter command
        :type trace_path: str
        :type shortcut: bool
        :return: Summary of trace
        :raises Exception: if summary is invalid
        """
        command = 'iotrace' + (' -M' if shortcut else ' --trace-management')
        command += ' -G' if shortcut else ' --get-trace-summary'
        command += (' -p ' if shortcut else ' --path ') + f'{trace_path}'

        output = TestRun.executor.run(command)
        if output.stdout == "":
            raise CmdException("Invalid summary", output)

        return parse_json(output.stdout)[0]

    @staticmethod
    def get_lba_histogram(trace_path: str,
                          bucket_size: Size = Size(0, Unit.Byte),
                          subrange_start: int = 0,
                          subrange_end: int = 0,
                          shortcut: bool = False) -> list:
        """
        Get lba histogram of given trace path

        :param trace_path: trace path
        :param bucket_size: bucket size
        :param subrange_start: subrange start
        :param subrange_end: subrange end
        :param shortcut: Use shorter command
        :type trace_path: str
        :type bucket_size: Size
        :type subrange_start: int
        :type subrange_end: int
        :type shortcut: bool
        :return: LBA histogram
        :raises Exception: if iotrace command or histogram is invalid
        """
        bucket_size_range = range(1, 4294967296)
        subrange_range = range(1, 9223372036854775808)
        if subrange_start and subrange_end:
            if subrange_start > subrange_end:
                subrange_start, subrange_end = subrange_end, subrange_start

        command = 'iotrace' + (' -P' if shortcut else ' --trace-parser')
        command += ' -B' if shortcut else ' --lba-histogram'
        command += (' -p ' if shortcut else ' --path ') + f'{trace_path}'

        if bucket_size is not None:
            if int(bucket_size.get_value(Unit.Byte)) not in bucket_size_range:
                raise CmdException(f"Given size is out of range {bucket_size_range}.")
            command += ' -b ' if shortcut else ' --bucket-size '
            command += f'{int(bucket_size.get_value(Unit.Byte))}'

        if subrange_start is not None:
            if subrange_start not in subrange_range:
                raise CmdException(f"Given start position is out of range {subrange_range}.")
            command += ' -s ' if shortcut else ' --subrange-start '
            command += f'{subrange_start}'

        if subrange_end is not None:
            if subrange_end not in subrange_range:
                raise CmdException(f"Given end position is out of range {subrange_range}.")
            command += ' -e ' if shortcut else ' --subrange-end '
            command += f'{subrange_end}'
        command += (' -f ' if shortcut else ' --format ') + 'json'

        output = TestRun.executor.run(command)
        if output.stdout == "":
            raise CmdException("Invalid histogram", output)

        return parse_json(output.stdout)

    @staticmethod
    def get_latency_histograms(trace_path: str, shortcut: bool = False) -> LatencyHistograms:
        """
        Get latency histogram of given trace path

        :param trace_path: trace path
        :param shortcut: Use shorter command
        :type trace_path: str
        :type shortcut: bool
        :return: latency histogram
        :rtype: LatencyHistograms
        :raises Exception: if histogram is invalid
        """
        command = 'iotrace' + (' -P' if shortcut else ' --trace-parser')
        command += ' -L' if shortcut else ' --latency-histogram'
        command += (' -p ' if shortcut else ' --path ') + f'{trace_path}'
        command += (' -f ' if shortcut else ' --format ') + 'json'

        output = TestRun.executor.run(command)
        if output.stdout == "":
            raise CmdException("Invalid histogram", output)

        return LatencyHistograms(parse_json(output.stdout)[0]['histogram'][0])

    @staticmethod
    def get_fs_statistics(trace_path: str, shortcut: bool = False) -> list:
        """
        Get filesystem statistics of given trace path

        :param trace_path: trace path
        :param shortcut: Use shorter command
        :type trace_path: str
        :type shortcut: bool
        :return: Filesystem statistics
        :raises Exception: if statistics are invalid
        """
        command = 'iotrace' + (' -P' if shortcut else ' --trace-parser')
        command += ' -F' if shortcut else ' --fs-statistics'
        command += (' -p ' if shortcut else ' --path ') + f'{trace_path}'
        command += (' -f ' if shortcut else ' --format ') + 'json'

        output = TestRun.executor.run(command)
        if output.stdout == "":
            raise CmdException("Invalid filesystem statistics", output)

        return parse_json(output.stdout)

    @staticmethod
    def get_trace_events(trace_path: str, raw: bool = False, shortcut: bool = False) -> list:
        """
        Get all trace events of given trace path

        :param trace_path: trace path
        :param raw: output without processing
        :param shortcut: Use shorter command
        :type trace_path: str
        :type raw: bool
        :type shortcut: bool
        :return: trace events
        :raises Exception: if traces are invalid
        """
        command = 'iotrace' + (' -P' if shortcut else ' --trace-parser')
        command += ' -P' if shortcut else ' --io'
        command += (' -p ' if shortcut else ' --path ') + f'{trace_path}'
        command += (' -f ' if shortcut else ' --format ') + 'json'

        if raw:
            command += ' -r ' if shortcut else ' --raw '

        output = TestRun.executor.run(command)
        if output.stdout == "":
            raise CmdException("Invalid traces", output)

        return parse_json(output.stdout)

    @staticmethod
    def get_trace_statistics(trace_path: str, dev_path: str = "", shortcut: bool = False) -> list:
        """
        Get statistics of particular trace

        :param trace_path: trace path
        :param dev_path: path of device which should be retrieved
        :param shortcut: Use shorter command
        :type trace_path: str
        :type shortcut: bool
        :return: If @dev_path specified - trace statistics, list of trace stats otherwise
        :raises Exception: if statistics are invalid
        """
        command = 'iotrace' + (' -P' if shortcut else ' --trace-parser')
        command += ' -S' if shortcut else ' --statistics'
        command += (' -p ' if shortcut else ' --path ') + f'{trace_path}'
        command += (' -f ' if shortcut else ' --format ') + 'json'

        output = TestRun.executor.run(command)
        if output.stdout == "":
            raise CmdException("Invalid IO statistics", output)

        if dev_path == "":
            return parse_json(output.stdout)[0]['statistics']

        expected_device_name  = dev_path.split('/dev/')[1].replace("/","")

        for trace_stat in parse_json(output.stdout)[0]['statistics']:
            traced_device = trace_stat["desc"]["device"]["name"]
            if  traced_device == expected_device_name:
                return trace_stat

        raise CmdException(f"No trace stats for device {dev_path}", output)

    @staticmethod
    def remove_traces(prefix: str, force: bool = False, shortcut: bool = False):
        """
        Removes record of the most recent trace

        :param prefix: Trace path or trace path's prefix when ended with '*'
        :param force: force removal
        :param shortcut: Use shorter command
        :type prefix: str
        :type force: bool
        :type shortcut: bool
        :raises Exception: if removal is invalid
        """
        command = 'iotrace' + (' -M' if shortcut else ' --trace-management')
        command += ' -R' if shortcut else ' --remove-traces'
        command += (' -p ' if shortcut else ' --prefix ') + f'{prefix}'

        if force:
            command += ' -f ' if shortcut else ' --force '

        output = TestRun.executor.run(command)
        if output.exit_code == 0:
            return parse_json(output.stdout)
        error_output = parse_json(output.stderr)[0]['trace']
        if error_output == "No traces removed.":
            return ""
        return error_output

    @staticmethod
    def set_trace_repository_path(trace_path: str, shortcut: bool = False):
        """
        :param trace_path: trace path
        :param shortcut: Use shorter command
        :type trace_path: str
        :type shortcut: bool
        :raises Exception: if setting path fails
        """
        if not check_if_directory_exists(trace_path):
            create_directory(trace_path)

        command = 'iotrace' + (' -C' if shortcut else ' --trace-config')
        command += ' -S ' if shortcut else ' --set-trace-repository-path '
        command += (' -p ' if shortcut else ' --path ') + f'{trace_path}'

        output = TestRun.executor.run(command)
        if output.exit_code == 0:
            return
        error_output = parse_json(output.stderr)[0]["trace"]
        if error_output == "No access to trace directory":
            raise CmdException("Invalid setting of the trace repository path", output)

    @staticmethod
    def help(shortcut: bool = False) -> str:
        """
        :param shortcut: Use shorter command
        :type shortcut: bool
        :return: io-tracer help
        :raises Exception: if command fails
        """
        return TestRun.executor.run_expect_success(
            'iotrace' + (' -H ' if shortcut else ' --help ')).stdout

    @staticmethod
    def version(shortcut: bool = False) -> dict:
        """
        :param shortcut: Use shorter command
        :type shortcut: bool
        :return: io-tracer version
        :rtype: dict {'iotrace': str, 'OCTF': str}
        :raises Exception: if command fails
        """
        output = parse_json(TestRun.executor.run_expect_success(
            'iotrace' + (' -V ' if shortcut else ' --version ')).stdout)

        version = {output[0]["system"]: output[0]["trace"],
                   output[1]["system"]: output[1]["trace"]}

        return version

def parse_json(output: str):
    """
    Parse a string with json messages to a list of python dictionaries

    :param output: JSON output to be parsed into python dicts
    :return: List of dictionaries with fields corresponding
    to json messages fields
    :rtype: List of dicts
    """
    msgs_list = []
    for obj in __decode_json_stream(output):
        msgs_list.append(obj)
    return msgs_list

# Get json objects from a stream of concatenated json messages
#
# raw_decode stops once it has a valid object and returns the last position
# where wasn't part of the parsed object. It's not documented, but you can
# pass this position back to raw_decode and it start parsing again from
# that position. Unfortunately, the Python json module doesn't accept
# strings that have prefixing whitespace. So we need to search to find
# the first none-whitespace part of your document.

def __decode_json_stream(document, pos=0, decoder=json.JSONDecoder()):
    while True:
        # Create json stream without whitespace
        match = NOT_WHITESPACE.search(document, pos)

        if not match:
            # No more data
            return
        pos = match.start()

        try:
            obj, pos = decoder.raw_decode(document, pos)
        except json.JSONDecodeError:
            raise Exception("Invalid json formatting")

        yield obj


from django.conf.urls import url
from mainapp import views

#template tagging
app_name = 'mainapp'
urlpatterns=[
    url(r'^timetablePage/$',views.timetabledisp,name='timetablePage'),
    url(r'^inputPage/$',views.input,name='inputPage'),

]


from abc import ABC, abstractmethod

import pandas as pd
import numpy as np
from sklearn.model_selection import KFold

from QRT2023.Dataset import DataUnit

class PredictionAggregator:
    def __init__(self, models:dict, region:str):
        self.models = models
        self.region = region

    def fit_all(self, training_data:DataUnit):
        self.fitted_models = {}
        for key in self.models.keys():
            model = self.models[key][self.region]
            X, y = training_data.X.values, training_data.y.values.ravel()
            model.fit(X, y)
            self.fitted_models[key] = model

    def predict_all(self, predicting_X:pd.DataFrame):
        predictions = {}
        for key in self.models.keys():
            pred_X = predicting_X.values
            predictions[key] = self.fitted_models[key].predict(pred_X)
        return pd.DataFrame(predictions, index=predicting_X.index)

    def fit_predict(
        self,
        training_data:DataUnit,
        predicting_X:pd.DataFrame,
        n_bootstrap:int=0,
        bootstrap_fraction:float=0,
        seed:int=42
        ):
        if n_bootstrap == 0:
            self.fit_all(training_data)
            self.full_predictions = self.predict_all(predicting_X)
        else:
            np.random.seed(seed)
            data_idx = training_data.X.index.to_list()
            bootstap_idxs = np.random.choice(
                data_idx,
                (n_bootstrap, int(len(data_idx)*bootstrap_fraction))
                )
            bootstrap_preds = []
            for subset_idx in bootstap_idxs:
                subset = DataUnit(
                    dunit=training_data.dunit.loc[subset_idx],
                    X_cols=training_data.X_cols,
                    y_col=training_data.y_col
                    )
                self.fit_all(subset)
                bootstrap_preds.append(self.predict_all(predicting_X))
            self.full_predictions = sum(bootstrap_preds) / n_bootstrap
        return self.full_predictions


"""https://anzeljg.github.io/rin2/book2/2405/docs/tkinter/message.html"""

import tkinter as tk

# Retriving the string for message.
# __file__ returns path of this script
path = __file__.replace('Message 2.py', 'sample.txt')
with open(path, mode='r') as text_file:
    lyrics = text_file.read()
    # print(lyrics)

root = tk.Tk()
root.geometry('300x620')
root.title('Message widget')

msg = tk.Message(master=root, text=lyrics)
msg.pack()

msg.config(bg='#fffdd0')        # setting background colour

root.mainloop()

GUI designing/widgets/Message widget/Message 2.py


########################################################################################
#                     Script author: Michał Tomaszewski                                #
########################################################################################

import numpy as np
import seaborn as sns
import sklearn.metrics as sklm
import matplotlib.pyplot as plt

import tensorflow as tf
import keras
from keras.preprocessing.image import ImageDataGenerator
from keras.layers import Dense, Conv2D, BatchNormalization, MaxPooling2D, Flatten, Dropout
from keras.models import Sequential, load_model, Model
from keras.applications import VGG16
from keras.optimizers import Adam, SGD, RMSprop
from keras.callbacks import ModelCheckpoint, EarlyStopping
from keras.applications.vgg16 import preprocess_input as vgg16_preprocessing

from glaucpack import run_tests, smooth_curve, visualize_training

class MetricsPack(keras.callbacks.Callback):
    def on_train_begin(self, logs={}):

        self.validation_data = val_generator
        self.confusion = []
        self.precision = []
        self.recall = []
        self.f1s = []
        self.kappa = []
        self.auc = []
        self.lowest_FN = np.sum(val_generator.classes)
        self.acceptable_FP = np.round(0.5*(len(val_generator.classes) - self.lowest_FN))

    def on_epoch_end(self, epoch, logs={}):

        val_generator = self.validation_data

        batch_size = val_batch_size
        breakpoint = len(val_generator) - 1

        if type(self.model.layers[-1]) is keras.engine.sequential.Sequential:
            class_layer_config = self.model.layers[-1].layers[-1].get_config()
        else:
            class_layer_config = self.model.layers[-1].get_config()

        # different handlers for different functions in the last node
        if class_layer_config['activation'] == 'sigmoid':

            val_data = np.zeros((len(val_generator)*batch_size, 256, 256, 3))
            val_labels = np.zeros((len(val_generator)*batch_size))

            for batch_id, (x, y) in enumerate(val_generator):
                val_data[batch_id*batch_size:(batch_id+1)*batch_size] = x
                val_labels[batch_id*batch_size:(batch_id+1)*batch_size] = y

                if batch_id == breakpoint:
                    break

            score = np.asarray(self.model.predict(val_data))
            predict = np.round(np.asarray(self.model.predict(val_data)))
            targ = val_labels

            go_further = True

        elif class_layer_config['activation'] == 'softmax' and class_layer_config['units'] == 2:
            print('got here')
            val_data = np.zeros((len(val_generator)*batch_size, 256, 256, 3))
            val_labels = np.zeros((len(val_generator)*batch_size, 2))

            for batch_id, (x, y) in enumerate(val_generator):
                val_data[batch_id*batch_size:(batch_id+1)*batch_size] = x
                val_labels[batch_id*batch_size:(batch_id+1)*batch_size, :] = y

                if batch_id == breakpoint:
                    break

            score = np.asarray(self.model.predict(val_data))
            predict = np.argmax(score, axis = 1)
            targ = np.argmax(val_labels, axis = 1)

            score = np.max(score, axis = 1)

            go_further = True

        else:
            print('Only binary classification is supported.')
            go_further = False

        # calculating metrics
        if go_further:
            epoch_auc = sklm.roc_auc_score(targ, score)
            epoch_confusion = sklm.confusion_matrix(targ, predict)
            epoch_precision = sklm.precision_score(targ, predict)
            epoch_recall = sklm.recall_score(targ, predict)
            epoch_f1 = sklm.f1_score(targ, predict)

            self.auc.append(epoch_auc)
            self.confusion.append(epoch_confusion)
            self.precision.append(epoch_precision)
            self.recall.append(epoch_recall)
            self.f1s.append(epoch_f1)

            epoch_FNs = epoch_confusion.ravel()[2]
            epoch_FPs = epoch_confusion.ravel()[1]

            # saving the model with lowest FN yet
            if epoch_FNs < self.lowest_FN and epoch_FPs <= self.acceptable_FP:
                filename = self.model.layers[0].name + '_low_FN_checkpoint.hdf5'
                self.model.save(filename)

                self.lowest_FN = epoch_FNs

                print('Found new low for FN. Saving this model to {}'.format(filename))
            else:
                pass

            print('Metrics on epoch end')
            print('AUC = {}'.format(epoch_auc))
            print('Precision = {}'.format(epoch_precision))
            print('Recall = {}'.format(epoch_recall))
            print('F1 = {}'.format(epoch_f1))
            print('False negativess = {}'.format(epoch_FNs))

            # If in JupyterNotebbok this would show a heatmap every interation
            # sns.heatmap(epoch_confusion, cmap = 'coolwarm', annot = True)
            # plt.xlabel('Predicted Labels')
            # plt.ylabel('True Labels')
            # plt.show()

train_dir = r''
val_dir = r''
test_dir = r''

# datagens
aggresive_augmentation = True
if aggresive_augmentation:
    train_datagen = ImageDataGenerator(#rescale = 1/255, # not needed with vgg16_preprocessing
                                       rotation_range = 40,
                                       width_shift_range = 0.1,
                                       height_shift_range = 0.1,
                                       zoom_range = 0.1,
                                       horizontal_flip = True,
                                       preprocessing_function = vgg16_preprocessing,
                                       fill_mode = 'reflect')
else:
    train_datagen = ImageDataGenerator(#rescale = 1/255,
                                       horizontal_flip = True,
                                       vertical_flip = True,
                                       preprocessing_function = vgg16_preprocessing)

test_datagen = ImageDataGenerator(#rescale = 1/255,
                                  preprocessing_function = vgg16_preprocessing)

# MODEL DEFINITION
target_size = (256, 256)
input_shape = target_size + (3,)

transfer_learning = True
if transfer_learning:
    conv_base = VGG16(weights = 'imagenet',
                include_top = False,
                input_shape = input_shape)

    for layer in conv_base.layers:
        layer.trainable = False

else:
    conv_base = VGG16(weights = None,
                     include_top = False,
                     input_shape = input_shape)

model = Sequential()
model.add(conv_base)
model.add(Flatten())
model.add(Dropout(0.5))
model.add(Dense(1024, activation = 'relu'))
model.add(Dense(1024, activation = 'relu'))
model.add(Dense(1, activation = 'sigmoid'))

# prep generators
train_batch_size = 10
train_generator = train_datagen.flow_from_directory(train_dir,
                                                   target_size = target_size,
                                                   batch_size = train_batch_size,
                                                   class_mode = 'binary',
                                                   shuffle = True)

val_batch_size = 6
val_generator = test_datagen.flow_from_directory(val_dir,
                                                target_size = target_size,
                                                batch_size = val_batch_size,
                                                class_mode = 'binary')

test_generator = test_datagen.flow_from_directory(test_dir, target_size = target_size,
                                                 batch_size = 1, shuffle = False, class_mode = 'binary')

metrics = MetricsPack()

model.compile(optimizer = Adam(),
             loss = 'binary_crossentropy',
             metrics = ['acc'])

val_loss_callback = ModelCheckpoint('train_demo_test_chckpt.hdf5', monitor = 'val_loss', verbose = 1, save_best_only = True)
early_stopper = EarlyStopping(monitor = 'val_loss', patience = 10, min_delta = 0.1)

train_bool = True
if train_bool:
    history = model.fit_generator(train_generator, validation_data = val_generator,
                        epochs = 1,
                        verbose = 1,
                        callbacks = [metrics, val_loss_callback])

# UNFREEZING SOME LAYERS
for layer in model.layers[0].layers[:15]:
    layer.trainable = False
for layer in model.layers[0].layers[15:]:
    layer.trainable = True

for layer_id, layer in enumerate(model.layers[0].layers):
    print(layer_id, layer.name, layer.trainable)

# we need to recompile the model for these modifications to take effect
# we use SGD with a low learning rate
model.compile(optimizer=SGD(lr=0.0001, momentum=0.9), loss='binary_crossentropy', metrics = ['acc'])

val_loss_callback = ModelCheckpoint('train_demo_test_chckpt.hdf5', monitor = 'val_loss', verbose = 1, save_best_only = True)

train_bool = True
if train_bool:
    history = model.fit_generator(train_generator, validation_data = val_generator,
                                epochs = 2,
                                verbose = 1,
                                callbacks = [metrics, val_loss_callback])

# whether to use the last trained model or use a saved model that was trained earilier
model_in_memory = True
if not model_in_memory:
    model = load_model('equal_vgg16_tuning_checkpoint.hdf5')
    run_tests(model, test_generator)
else:
    run_tests(model, test_generator)

visualize_training(history.history, smoothed = True)


import os
import subprocess
import re

def decompile_apk(apk_path, output_dir):
    subprocess.run(['jadx', '-d', output_dir, apk_path], check=True)

def search_keywords(directory, keywords):
    keyword_matches = {}

    for root, dirs, files in os.walk(directory):
        for file in files:
            file_path = os.path.join(root, file)
            with open(file_path, 'r', errors='ignore') as f:
                lines = f.readlines()
                current_class = None

                for line_number, line in enumerate(lines, start=1):
                    if line.strip().startswith("public class") or line.strip().startswith("class"):
                        current_class = re.search(r'\bclass\s+(\w+)', line)
                        if current_class:
                            current_class = current_class.group(1)

                    for keyword in keywords:
                        if current_class and re.search(r'\b{}\b'.format(re.escape(keyword)), line):
                            if file_path not in keyword_matches:
                                keyword_matches[file_path] = []
                            keyword_matches[file_path].append((current_class, line_number, line.strip()))

    return keyword_matches

def main():
    apk_path = 'yourapk file'
    output_dir = '/tmp'
    keywords_to_find = [
       'busybox',
       'su',
       'com.noshufou.android.su',
       'com.noshufou.android.su.elite',
       'eu.chainfire.supersu',
       'com.koushikdutta.superuser',
       'com.thirdparty.superuser',
       'com.yellowes.su',
       'com.topjohnwu.magisk',
       'com.kingroot.kinguser',
       'com.kingo.root',
       'com.smedialink.oneclickroot',
       'com.zhiqupk.root.global',
       'com.alephzain.framaroot',
       'com.koushikdutta.rommanager',
       'com.koushikdutta.rommanager.license',
       'com.dimonvideo.luckypatcher',
       'com.chelpus.lackypatch',
       'com.ramdroid.appquarantine',
       'com.ramdroid.appquarantinepro',
       'com.android.vending.billing.InAppBillingService.COIN',
       'com.android.vending.billing.InAppBillingService.LUCK',
       'com.chelpus.luckypatcher',
       'com.blackmartalpha',
       'org.blackmart.market',
       'com.allinone.free',
       'com.repodroid.app',
       'org.creeplays.hack',
       'com.baseappfull.fwd',
       'com.zmapp',
       'com.dv.marketmod.installer',
       'org.mobilism.android',
       'com.android.wp.net.log',
       'com.android.camera.update',
       'cc.madkite.freedom',
       'com.solohsu.android.edxp.manager',
       'org.meowcat.edxposed.manager',
       'com.xmodgame',
       'com.cih.game_cih',
       'com.charles.lpoqasert',
       'catch_.me_.if_.you_.can_',
       'com.devadvance.rootcloak',
       'com.devadvance.rootcloakplus',
       'de.robv.android.xposed.installer',
       'com.saurik.substrate',
       'com.zachspong.temprootremovejb',
       'com.amphoras.hidemyroot',
       'com.amphoras.hidemyrootadfree',
       'com.formyhm.hiderootPremium',
       'com.formyhm.hideroot',
       '/data/local/',
       '/data/local/bin/',
       '/data/local/xbin/',
       '/sbin/',
       '/su/bin/',
       '/system/bin/',
       '/system/bin/.ext/',
       '/system/bin/failsafe/',
       '/system/sd/xbin/',
       '/system/usr/we-need-root/',
       '/system/xbin/',
       '/cache/',
       '/data/',
       '/dev/',
       '/system',
       '/system/bin',
       '/system/sbin',
       '/system/xbin',
       '/vendor/bin',
       '/sbin',
       '/etc'
   ]

    decompile_apk(apk_path, output_dir)
    keyword_matches = search_keywords(output_dir, keywords_to_find)

    for file_path, matches in keyword_matches.items():
        print(f"\nMatches found in file: {file_path}")
        for class_name, line_number, line in matches:
            print(f"Class: {class_name}")
            print(f"Line {line_number}: {line}")

if __name__ == "__main__":
    main()


#
# @lc app=leetcode.cn id=477 lang=python3
#
# [477] 汉明距离总和
#
# https://leetcode-cn.com/problems/total-hamming-distance/description/
#
# algorithms
# Medium (60.43%)
# Likes:    251
# Dislikes: 0
# Total Accepted:    41.4K
# Total Submissions: 68.5K
# Testcase Example:  '[4,14,2]'
#
# 两个整数的 汉明距离 指的是这两个数字的二进制数对应位不同的数量。
#
# 给你一个整数数组 nums，请你计算并返回 nums 中任意两个数之间 汉明距离的总和 。
#
#
#
# 示例 1：
#
#
# 输入：nums = [4,14,2]
# 输出：6
# 解释：在二进制表示中，4 表示为 0100 ，14 表示为 1110 ，2表示为 0010 。（这样表示是为了体现后四位之间关系）
# 所以答案为：
# HammingDistance(4, 14) + HammingDistance(4, 2) + HammingDistance(14, 2) = 2 +
# 2 + 2 = 6
#
#
# 示例 2：
#
#
# 输入：nums = [4,14,4]
# 输出：4
#
#
#
#
# 提示：
#
#
# 1 <= nums.length <= 10^4
# 0 <= nums[i] <= 10^9
# 给定输入的对应答案符合 32-bit 整数范围
#
#
#

from typing import List

# @lc code=start
class Solution:
    '''
    对于每一位考虑

    数组中的数有上限 不妨考虑每个数0-29位的情况

    假设数组中有n个数 其中有c个数在当前位是1 另外n-c个数在当前位就是0

    对于这c个数其和另外的位上为0的数的汉明距离之和为c*(n-c)

    而这个距离同样也是另外位上为0的数和这些位上为1的数的汉明距离之和

    事实上 这里把数分为两种 一种是当前位是0的 一种是当前位是1的

    计算这两种数的汉明距离之和

    遍历整个数组  其时间复杂度应该为O(30*n)

    比两重遍历的O(n^2)要好多了
    '''

    def totalHammingDistance(self, nums: List[int]) -> int:

        res = 0
        for i in range(30):
            c = 0
            for n in nums:
                c += (n >> i) & 1  # 将当前位的数目取出来，加到结果中去
            res += c * (len(nums) - c)

        return res

# @lc code=end


import sys, random, json
from gensim.models import Word2Vec
from argparse import ArgumentParser
from collections import Counter
from pathlib import Path

__dir__= Path(__file__).absolute().parent.parent

class DirSentences(object):
    def __init__(self, *args, **kwargs):
        self.dirs = args
        self.len = None
        self.sep = kwargs.get("sep", " ")
        self.shuffle = kwargs.get("shuffle", False)
        self.ext = kwargs.get("ext", "csv")

    def __len__(self):
        if self.len is not None:
            return self.len
        ret = 0
        for dirname in self.dirs:
            for p in Path(dirname).glob("*."+self.ext):
                with p.open("rb") as f:
                    content = f.read().decode("utf-8", "ignore")
                    ret += len(content.split("\n"))
        self.len = ret
        return ret

    def __iter__(self):
        total_len = 0
        for dirname in self.dirs:
            for p in Path(dirname).glob("*."+self.ext):
                with p.open("rb") as f:
                    content = f.read().decode("utf-8", "ignore")
                    for line in content.split("\n"):
                        ret = line.strip(' \r\n"\'\t\xa0`').split(self.sep)
                        if self.shuffle:
                            random.shuffle(ret)
                        yield ret
                        total_len += 1
        self.len = total_len

    def build_vocab(self, output=None):
        vocab = Counter()
        for dirname in self.dirs:
            for p in Path(dirname).glob("*."+self.ext):
                with p.open("rb") as f:
                    content = f.read().decode("utf-8", "ignore")
                    for line in content.split("\n"):
                        vocab+=Counter(line.strip().split(self.sep))
        vocab = dict(vocab)
        if output is not None:
            with open(output, "w") as f:
                json.dump(vocab, f)
        return vocab

def main(params):
    train_sentences = DirSentences(*params.input.split(','),
    sep=params.sep, shuffle=params.shuffle,ext=params.input_file_extension)
    if Path(params.model).exists():
        print ("Loading previous model " + params.model)
        model = Word2Vec.load(params.model)
    else:
        print("Creating new model and building vocab")
        model = Word2Vec(epochs=1, min_count=params.minTF, workers=params.workers, vector_size=params.size, window=params.window)
        if Path(params.vocab).exists():
            with open(params.vocab, 'r') as f:
                vocab_freq = json.load(f)
        else:
            print("Vocab file not found, building vocab")
            vocab_freq = train_sentences.build_vocab(output=params.vocab)
        model.build_vocab_from_freq(vocab_freq)
        print("Vocab generation done {c} words".format(c=len(model.wv.index_to_key)))
    print("Training model")
    model.train(train_sentences, total_examples=len(train_sentences), epochs=1)
    print("Saving model to " + params.model)
    model.save(params.model)
    return 0

if __name__ == "__main__":
    argparse = ArgumentParser()
    argparse.add_argument('--window', default=5, type=int, help='window size')
    argparse.add_argument('--size', default=50, type=int, help='vector size')
    argparse.add_argument('--shuffle', default=True, type=bool, help='shuffle order of tokens')
    argparse.add_argument('--sep', default=',', type=str, help='token seperator')
    argparse.add_argument('--input_file_extension', default='csv', type=str, help='input files to process')
    argparse.add_argument('--workers', default=4, type=int, help='number of processes')
    argparse.add_argument('--minTF', default=4, type=int, help='minimum term frequency')
    argparse.add_argument('--model', default=str(__dir__/'w2v.pickle'), type=str, help='model file')
    argparse.add_argument('--input', default=str(__dir__/'data'), type=str, help='dirs to learn from, comma separated')
    argparse.add_argument('--vocab', default=str(__dir__ / 'vocab.json'), type=str, help='vocab frequency file, in json format')
    params = argparse.parse_args()
    sys.exit(main(params))


import numpy as np
import cv2
import matplotlib.pyplot as plt
import matplotlib.image as mpimg

"""
If you want to continue to explore additional color channels,
I have seen that the L channel from LUV with lower and upper thresholds around 225 & 255
respectively works very well to pick out the white lines, even in the parts of the video
with heavy shadows.You can also try out the b channel from Lab which does a great job with
the yellow lines (you can play around with thresholds around 155 & 200).
"""

def get_binary_img(img, s_thresh=(120, 255), sx_thresh=(20, 255),l_thresh=(40,255)):
    img = np.copy(img)
    # Convert to HLS color space and separate the V channel
    hls = cv2.cvtColor(img, cv2.COLOR_BGR2HLS).astype(np.float)
    l_channel = hls[:,:,1]
    s_channel = hls[:,:,2]
    # Sobel x
    sobelx = cv2.Sobel(l_channel, cv2.CV_64F, 1, 0) # Take the derivative in x
    abs_sobelx = np.absolute(sobelx) # Absolute x derivative to accentuate lines away from horizontal
    scaled_sobel = np.uint8(255*abs_sobelx/np.max(abs_sobelx))

    # Threshold x gradient
    sxbinary = np.zeros_like(scaled_sobel)
    sxbinary[(scaled_sobel >= sx_thresh[0]) & (scaled_sobel <= sx_thresh[1])] = 1

    # Threshold color channel
    s_binary = np.zeros_like(s_channel)
    s_binary[(s_channel >= s_thresh[0]) & (s_channel <= s_thresh[1])] = 1

     # Threshold lightness
    l_binary = np.zeros_like(l_channel)
    l_binary[(l_channel >= l_thresh[0]) & (l_channel <= l_thresh[1])] = 1

    # Stack each channel
    channels = 255*np.dstack(( l_binary, sxbinary, s_binary)).astype('uint8')

    # Combine the three binary thresholds
    binary = np.zeros_like(sxbinary)
    binary[((l_binary == 1) & (s_binary == 1) | (sxbinary==1))] = 1
    binary = 255*np.dstack((binary,binary,binary)).astype('uint8')

    return binary, channels

if __name__ == '__main__':
    num_test_images = 6
    for test_image in range(1, num_test_images+1):
        print('processing image test_images/test{}.jpg'.format(test_image))
        image = cv2.imread('test_images/test{}.jpg'.format(test_image))
        combined_binary, color_binary = get_binary_img(image)
        cv2.imwrite('output_images/test{}_binary.png'.format(test_image), combined_binary)
        cv2.imwrite('output_images/test{}_color_binary.png'.format(test_image), color_binary)


import re
import os
import sys
import json
import spacy
import string
import logging
import argparse

from nltk.corpus import stopwords

ATTN_WINDOW_RANGE = 0

def _process_strings(line,
                     lang_nlp,
                     get_lemmas,
                     get_pos,
                     remove_stopwords,
                     replace_stopwords,
                     get_maps):

    """ Helper function for obtaining various word representations """

    # strip, replace special tokens
    orig_line = line
    line = line.strip()
    line = re.sub(r''', '\'', line.strip())
    line = re.sub(r'"', '\"', line.strip())
    # Tokenize etc.
    line_nlp = lang_nlp(line)
    spacy_tokens = [elem.text for elem in line_nlp]
    spacy_tokens_lower = [elem.text.lower() for elem in line_nlp]
    spacy_lemmas = None
    spacy_pos = None
    if get_lemmas:
        spacy_lemmas = list()
        for elem in line_nlp:
            if elem.lemma_ == '-PRON-' or elem.lemma_.isdigit():
                spacy_lemmas.append(elem.lower_)
            else:
                spacy_lemmas.append(elem.lemma_.lower().strip())
    if get_pos:
        spacy_pos = [elem.pos_ for elem in line_nlp]

    # Generate a mapping between whitespace tokens and SpaCy tokens
    ws_tokens = orig_line.strip().split()
    ws_tokens_lower = orig_line.strip().lower().split()
    ws_to_spacy_map = dict()
    spacy_to_ws_map = dict()
    if get_maps:
        ws_loc = 0
        ws_tok = ws_tokens[ws_loc]

        for spacy_loc, spacy_tok in enumerate(spacy_tokens):
            while True:
                # Map whitespace tokens to be identical to spacy tokens
                ws_tok = re.sub(r''', '\'', ws_tok)
                ws_tok = re.sub(r'"', '\"', ws_tok)

                if spacy_tok == ws_tok or spacy_tok in ws_tok:
                    # Terminate
                    if ws_loc >= len(ws_tokens):
                        break

                    # Extend maps
                    if not ws_to_spacy_map.get(ws_loc, None):
                        ws_to_spacy_map[ws_loc] = list()
                    ws_to_spacy_map[ws_loc].append(spacy_loc)
                    if not spacy_to_ws_map.get(spacy_loc, None):
                        spacy_to_ws_map[spacy_loc] = list()
                    spacy_to_ws_map[spacy_loc].append(ws_loc)

                    # Move pointer
                    if spacy_tok == ws_tok:
                        ws_loc += 1
                        if ws_loc < len(ws_tokens):
                            ws_tok = ws_tokens[ws_loc]
                    else:
                        ws_tok = ws_tok[len(spacy_tok):]
                    break
                else:
                    ws_loc += 1

        # Assert full coverage of whitespace and SpaCy token sequences by the mapping
        ws_covered = sorted(list(ws_to_spacy_map.keys()))
        spacy_covered = sorted(list(set(list([val for val_list in ws_to_spacy_map.values() for val in val_list]))))
        assert ws_covered == [n for n in range(len(ws_tokens))], \
            'WS-SpaCy mapping does not cover all whitespace tokens: {}; number of tokens: {}'\
            .format(ws_covered, len(ws_tokens))
        assert spacy_covered == [n for n in range(len(spacy_tokens))], \
            'WS-SpaCy mapping does not cover all SpaCy tokens: {}; number of tokens: {}' \
            .format(spacy_covered, len(spacy_tokens))

    if remove_stopwords:
        # Filter out stopwords
        nsw_spacy_tokens_lower = list()
        nsw_spacy_lemmas = list()
        for tok_id, tok in enumerate(spacy_tokens_lower):
            if tok not in STOP_WORDS:
                nsw_spacy_tokens_lower.append(tok)
                if get_lemmas:
                    nsw_spacy_lemmas.append(spacy_lemmas[tok_id])
            else:
                if replace_stopwords:
                    nsw_spacy_tokens_lower.append('<STPWRD>')
                    if get_lemmas:
                        nsw_spacy_lemmas.append('<STPWRD>')

        spacy_tokens_lower = nsw_spacy_tokens_lower
        if get_lemmas:
            spacy_lemmas = nsw_spacy_lemmas

    return line_nlp, spacy_tokens_lower, spacy_lemmas, spacy_pos, ws_tokens, ws_tokens_lower, ws_to_spacy_map, \
        spacy_to_ws_map

# TODO / NOTE: Only compatible with word-level attractors, for now
def _get_nmt_label(translation,
                   src,
                   ambiguous_token,
                   ambiguous_token_loc,
                   seed_cluster_id,
                   adv_cluster_id,
                   other_cluster_ids,
                   sense_lemmas_to_cluster,
                   cluster_to_sense_lemmas,
                   sense_tokens_to_cluster,
                   cluster_to_sense_tokens,
                   alignments,
                   tgt_sent):

    """ Helper function for evaluating whether the translations of true and adversarially perturbed source samples
    perform lexical WSD correctly. """

    # Lemmatize the translation for higher coverage of attack successes (preserve punctuation)
    _, spacy_tokens_lower, spacy_lemmas, _, ws_tokens, ws_tokens_lower, ws_to_spacy_map, spacy_to_ws_map = \
        _process_strings(translation,
                         tgt_nlp,
                         get_lemmas=True,
                         get_pos=False,
                         remove_stopwords=False,
                         replace_stopwords=False,
                         get_maps=True)

    # All sequences are Moses-tokenized and can be split at whitespace
    src_tokens = src.strip().split()
    ambiguous_token_loc = ambiguous_token_loc[0]

    # Check that the provided locations of the ambiguous token are correct
    ambiguous_token_in_src = src_tokens[ambiguous_token_loc].lower().strip(punctuation_plus_space)
    assert ambiguous_token.lower().strip(punctuation_plus_space) == \
        ambiguous_token_in_src or ambiguous_token[:-1] in ambiguous_token_in_src, \
        'Ambiguous token \'{:s}\' does not match the true source token \'{:s}\' at the token location'\
        .format(ambiguous_token, ambiguous_token_in_src)

    other_cluster_lemmas = list()
    for cluster_id in other_cluster_ids:
        other_cluster_lemmas += cluster_to_sense_lemmas[cluster_id]
    other_cluster_tokens = list()
    for cluster_id in other_cluster_ids:
        other_cluster_tokens += cluster_to_sense_tokens[cluster_id]

    target_hits = list()
    # If alignments are available, look-up target sense aligned with the ambiguous source term
    alignments = alignments.strip()
    # Convert alignments into a more practical format, first
    line_align_table = dict()
    for word_pair in alignments.strip().split():
        src_id, tgt_id = word_pair.split('-')
        src_id = int(src_id)
        tgt_id = int(tgt_id)
        if not line_align_table.get(src_id, None):
            line_align_table[src_id] = [tgt_id]
        else:
            line_align_table[src_id].append(tgt_id)

    if ambiguous_token_loc in line_align_table.keys():
        homograph_aligned = True
        # Extend alignment window
        line_align_table_entry = sorted(line_align_table[ambiguous_token_loc])
        min_tgt_idx = max(0, line_align_table_entry[0] - ATTN_WINDOW_RANGE)
        max_tgt_idx = min(len(spacy_lemmas) - 1, line_align_table_entry[-1] + ATTN_WINDOW_RANGE)
        # Map from target whitespace to target spacy tokens
        min_tgt_idx = ws_to_spacy_map[min_tgt_idx][0]
        max_tgt_idx = ws_to_spacy_map[max_tgt_idx][0]
        tgt_window = range(min_tgt_idx, max_tgt_idx)
        # Check if aligned translation lemmas include correct / flipped source term translations
        aligned_translation_lemmas = [spacy_lemmas[idx] for idx in tgt_window]
        aligned_tgt_tokens = [spacy_tokens_lower[idx] for idx in tgt_window]

        for atl_id, atl in enumerate(aligned_translation_lemmas):
            # Skip stopwords
            if atl == '<STPWRD>':
                continue
            # Strip leading and trailing punctuation
            atl = atl.strip(punctuation_plus_space)
            # Match lemmas
            if sense_lemmas_to_cluster is not None:
                if sense_lemmas_to_cluster.get(atl, None):
                    target_hits.append(sense_lemmas_to_cluster[atl])
                else:
                    maybe_lemma_matches = sorted(cluster_to_sense_lemmas[seed_cluster_id] +
                                                 cluster_to_sense_lemmas[adv_cluster_id] +
                                                 other_cluster_lemmas, reverse=True, key=lambda x: len(x))

                    for maybe_lemma_match in maybe_lemma_matches:
                        if atl.startswith(maybe_lemma_match) or atl.endswith(maybe_lemma_match) or \
                                atl[:-1].endswith(maybe_lemma_match):
                            target_hits.append(sense_lemmas_to_cluster[maybe_lemma_match])
                            break
            # Match tokens
            if len(target_hits) == 0 and sense_tokens_to_cluster is not None:
                att = aligned_tgt_tokens[atl_id].strip(punctuation_plus_space)
                if sense_tokens_to_cluster.get(att, None):
                    target_hits.append(sense_tokens_to_cluster[att])
                else:
                    maybe_token_matches = \
                        sorted(cluster_to_sense_tokens[seed_cluster_id] +
                               cluster_to_sense_tokens[adv_cluster_id] +
                               other_cluster_tokens, reverse=True, key=lambda x: len(x))

                    for maybe_token_match in maybe_token_matches:
                        if att.startswith(maybe_token_match) or att.endswith(maybe_token_match.lower()) or \
                                att[:-1].endswith(maybe_token_match.lower()):
                            target_hits.append(sense_tokens_to_cluster[maybe_token_match.lower()])
                            break
    else:
        homograph_aligned = False

    # TODO: Can / should this be improved?
    # Check if the attractor term(s) have been translated (assumes translation if alignment was found)
    # NOTE: Currently works ONLY FOR 1-WORD ATTRACTORS (e.g. adjectives)
    is_compound = False

    # If no alignments are available, match the known target sense
    if len(target_hits) == 0:
        for stl_id, stl in enumerate(spacy_lemmas):
            # if stl_id not in range(ambiguous_token_spacy_loc_seed - 3, ambiguous_token_spacy_loc_seed + 4):
            #     continue
            # Skip stopwords
            if stl == '<STPWRD>':
                continue
            # Strip leading and trailing punctuation
            stl = stl.strip(punctuation_plus_space)
            # Match lemmas
            if sense_lemmas_to_cluster is not None:
                if sense_lemmas_to_cluster.get(stl, None):
                    target_hits.append(sense_lemmas_to_cluster[stl])
                else:
                    maybe_lemma_matches = \
                        sorted(cluster_to_sense_lemmas[seed_cluster_id] +
                               cluster_to_sense_lemmas[adv_cluster_id] +
                               other_cluster_lemmas, reverse=True, key=lambda x: len(x))
                    for maybe_lemma_match in maybe_lemma_matches:
                        if stl.startswith(maybe_lemma_match) or stl.endswith(maybe_lemma_match) or \
                                stl[:-1].endswith(maybe_lemma_match):
                            target_hits.append(sense_lemmas_to_cluster[maybe_lemma_match])
                            break
            # Match tokens
            if len(target_hits) == 0 and sense_tokens_to_cluster is not None:
                stt = spacy_tokens_lower[stl_id].strip(punctuation_plus_space)
                if sense_tokens_to_cluster.get(stt, None):
                    target_hits.append(sense_tokens_to_cluster[stt])
                else:
                    maybe_token_matches = \
                        sorted(cluster_to_sense_tokens[seed_cluster_id] +
                               cluster_to_sense_tokens[adv_cluster_id] +
                               other_cluster_tokens, reverse=True, key=lambda x: len(x))
                    for maybe_token_match in maybe_token_matches:
                        if stt.startswith(maybe_token_match) or stt.endswith(maybe_token_match.lower()) or \
                                stt[:-1].endswith(maybe_token_match.lower()):
                            try:
                                target_hits.append(sense_tokens_to_cluster[maybe_token_match.lower()])
                                break
                            except KeyError:
                                pass

    # Source homograph is assumed to be translated if:
    # 1. Homograph is aligned
    # 2. Homograph is not aligned, but len(target_hits) > 0
    if homograph_aligned:
        homograph_translated = True
    else:
        if len(target_hits) > 0:
            homograph_translated = True
        else:
            homograph_translated = False

    # TODO / NOTE: Fixing a corpus bug
    if 'rendezvous' in cluster_to_sense_lemmas[seed_cluster_id]:
        temp = seed_cluster_id
        seed_cluster_id = adv_cluster_id
        adv_cluster_id = temp

    # Flatten target hits
    target_hits = [hit[1] for hit_list in target_hits for hit in hit_list]
    # If target term is ambiguous, assume the translation is correct
    if seed_cluster_id in target_hits:

        return 'not_flipped', target_hits

    elif adv_cluster_id in target_hits:

        # TODO: DEBUGGING
        logging.info('-' * 10)
        logging.info('FLIPPED TO ATTR')
        logging.info(src)
        logging.info(translation)
        logging.info(tgt_sent)
        logging.info('SEED SENSES')
        logging.info(cluster_to_sense_lemmas[seed_cluster_id])
        logging.info('ADVERSARIAL SENSES')
        logging.info(cluster_to_sense_lemmas[adv_cluster_id])

        return 'flipped_to_attr', target_hits

    elif len(set(other_cluster_ids) & set(target_hits)) >= 1:
        return 'flipped_to_other', target_hits

    # i.e. target_hits is empty
    else:
        if homograph_translated:
            return 'maybe_flipped', target_hits

        else:
            return 'deleted_homograph', target_hits

def _build_cluster_lookup(sense_clusters_table):

    """ Post-processes the scraped target sense cluster table by constructing a sense-to-cluster_id lookup table """

    # Initialize empty tables
    logging.info('Constructing the cluster lookup table ...')
    sense_to_cluster_table = dict()

    # Fill tables
    for src_term in sense_clusters_table.keys():
        logging.info('Looking-up the term \'{:s}\''.format(src_term))
        sense_to_cluster_table[src_term] = dict()
        for cluster_id in sense_clusters_table[src_term].keys():
            # Construct cluster-ID lookup table entry
            for tgt_sense in sense_clusters_table[src_term][cluster_id]['[SENSES]']:
                # Lemmatizing single words is not ideal, but is expected to improve attractor recall
                _, _, tgt_lemmas, _, _, _, _, _ = \
                    _process_strings(tgt_sense, tgt_nlp, True, False, False, False, False)
                # Multi-word targets are excluded for simplicity (as a result, some words are dropped)
                if len(tgt_lemmas) < 1:
                    continue
                tgt_lemma = tgt_lemmas[0]
                if len(tgt_lemma) > 0:
                    if not sense_to_cluster_table[src_term].get(tgt_lemma, None):
                        sense_to_cluster_table[src_term][tgt_lemma] = [(tgt_sense, cluster_id, True)]
                    else:
                        sense_to_cluster_table[src_term][tgt_lemma].append((tgt_sense, cluster_id, True))

            # Check for blacklisted and ambiguous senses
            senses_to_ignore = list()
            senses_to_ignore += sense_clusters_table[src_term][cluster_id].get('[BLACKLISTED SENSES]', [])
            senses_to_ignore += sense_clusters_table[src_term][cluster_id].get('[AMBIGUOUS SENSES]', [])
            for tgt_sense in senses_to_ignore:
                # Lemmatizing single words is not ideal, but is expected to improve attractor recall
                _, _, tgt_lemmas, _, _, _, _, _ = \
                    _process_strings(tgt_sense, tgt_nlp, True, False, False, False, False)
                # Multi-word targets are excluded for simplicity (as a result, some words are dropped)
                if len(tgt_lemmas) < 1:
                    continue
                tgt_lemma = tgt_lemmas[0]
                if len(tgt_lemma) > 0:
                    if not sense_to_cluster_table[src_term].get(tgt_lemma, None):
                        sense_to_cluster_table[src_term][tgt_lemma] = [(tgt_sense, cluster_id, False)]
                    else:
                        sense_to_cluster_table[src_term][tgt_lemma].append((tgt_sense, cluster_id, False))
    return sense_to_cluster_table

def evaluate_attack_success(wsd_samples_path,
                            sources_path,
                            translations_path,
                            alignments_path,
                            attractors_path,
                            sense_clusters_path,
                            output_tables_dir):

    """ Detects successful attacks and computes correlation between attack success and various metrics. """

    def _score_and_filter(combined_sample, ambiguous_term, attractors_entry):
        """ Helper function for filtering a single adversarial sample. """

        # Combined sample contents:

        # [translations[line_id],
        # alignments[line_id],
        # src_to_sample[src_line.strip()][0],
        # src_to_sample[src_line.strip()][1]]

        sense_lemmas_to_cluster = sense_lemmas_to_cluster_table.get(ambiguous_term, None)
        sense_tokens_to_cluster = sense_tokens_to_cluster_table.get(ambiguous_term, None)
        cluster_to_sense_lemmas = cluster_to_sense_lemmas_table.get(ambiguous_term, None)
        cluster_to_sense_tokens = cluster_to_sense_tokens_table.get(ambiguous_term, None)

        # Unpack
        translation = combined_sample[0]
        line_alignments = combined_sample[1]
        challenge_entry = combined_sample[2]

        src_sent = challenge_entry[1]
        tgt_sent = challenge_entry[2]
        ambiguous_token_loc_seed = challenge_entry[7]
        ambiguous_token_ws_loc_seed = challenge_entry[8]  # list
        seed_cluster_num = challenge_entry[12]
        adv_cluster_num = challenge_entry[13]

        other_cluster_ids = list(attractors_entry.keys())
        other_cluster_ids.pop(other_cluster_ids.index(seed_cluster_num))
        try:
            other_cluster_ids.pop(other_cluster_ids.index(adv_cluster_num))
        except ValueError:
            pass

        # Ignore samples containing multiple instances of the attractor term
        src_tokens = [tok.strip(punctuation_plus_space) for tok in src_sent.split()]
        src_tokens = [tok for tok in src_tokens if len(tok) > 0]
        # Ignore short sentences
        if len(src_tokens) < 10:
            return None

        # Get NMT labels
        true_nmt_label, true_translation_sense_clusters = \
            _get_nmt_label(translation,
                           src_sent,
                           ambiguous_term,
                           ambiguous_token_ws_loc_seed,
                           seed_cluster_num,
                           adv_cluster_num,
                           other_cluster_ids,
                           sense_lemmas_to_cluster,
                           cluster_to_sense_lemmas,
                           sense_tokens_to_cluster,
                           cluster_to_sense_tokens,
                           line_alignments,
                           tgt_sent)

        # Sort true samples into appropriate output tables, based on filtering outcome
        seed_table_to_expand = None
        if true_nmt_label == 'not_flipped':
            if not unique_samples.get((src_sent, ambiguous_term, ambiguous_token_loc_seed), None):
                stat_dict['num_true_samples_good_translations'] += 1
                unique_samples[(src_sent, ambiguous_term, ambiguous_token_loc_seed)] = True
                seed_table_to_expand = true_samples_good_translations
        elif true_nmt_label == 'flipped_to_attr':
            if not unique_samples.get((src_sent, ambiguous_term, ambiguous_token_loc_seed), None):
                stat_dict['num_true_samples_bad_translations'] += 1
                unique_samples[(src_sent, ambiguous_term, ambiguous_token_loc_seed)] = True
                seed_table_to_expand = true_samples_bad_translations
        elif true_nmt_label == 'flipped_to_other':
            if not unique_samples.get((src_sent, ambiguous_term, ambiguous_token_loc_seed), None):
                stat_dict['num_true_samples_bad_translations'] += 1
                unique_samples[(src_sent, ambiguous_term, ambiguous_token_loc_seed)] = True
                seed_table_to_expand = true_samples_bad_translations
        elif true_nmt_label == 'deleted_homograph':
            if not unique_samples.get((src_sent, ambiguous_term, ambiguous_token_loc_seed), None):
                stat_dict['num_true_samples_deleted_homograph'] += 1
                unique_samples[(src_sent, ambiguous_term, ambiguous_token_loc_seed)] = True
                seed_table_to_expand = true_samples_deleted_homograph
        else:
            if not unique_samples.get((src_sent, ambiguous_term, ambiguous_token_loc_seed), None):
                stat_dict['num_true_samples_maybe_bad_translations'] += 1
                unique_samples[(src_sent, ambiguous_term, ambiguous_token_loc_seed)] = True
                seed_table_to_expand = true_samples_maybe_bad_translations

        # Collect seed translations
        if seed_table_to_expand is not None:
            if not seed_table_to_expand.get(ambiguous_term, None):
                seed_table_to_expand[ambiguous_term] = dict()
            if not seed_table_to_expand[ambiguous_term].get(seed_cluster_num, None):
                seed_table_to_expand[ambiguous_term][seed_cluster_num] = dict()
            if not seed_table_to_expand[ambiguous_term][seed_cluster_num].get(adv_cluster_num, None):
                seed_table_to_expand[ambiguous_term][seed_cluster_num][adv_cluster_num] = dict()
            if not seed_table_to_expand[ambiguous_term][seed_cluster_num][adv_cluster_num].get(src_sent, None):
                seed_table_to_expand[ambiguous_term][seed_cluster_num][adv_cluster_num][src_sent] = list()
            seed_table_to_expand[ambiguous_term][seed_cluster_num][adv_cluster_num][src_sent]\
                .append([translation, tgt_sent, true_translation_sense_clusters, ambiguous_token_loc_seed])

    def _show_stats():
        """ Helper for reporting on the generation process. """

        def _pc(enum, denom):
            """ Helper function for computing percentage of the evaluated sample type """
            return (enum / denom) * 100 if denom > 0 else 0

        logging.info('-' * 20)

        num_all_seed_samples = \
            stat_dict['num_true_samples_good_translations'] + \
            stat_dict['num_true_samples_bad_translations'] + \
            stat_dict['num_true_samples_deleted_homograph'] + \
            stat_dict['num_true_samples_maybe_bad_translations']

        # ==============================================================================================================
        logging.info('Evaluated {:d} seed samples'.format(num_all_seed_samples))
        logging.info('{:d} ({:.4f}%) TRUE source sentences have been translated CORRECTLY'
                     .format(stat_dict['num_true_samples_good_translations'],
                             _pc(stat_dict['num_true_samples_good_translations'], num_all_seed_samples)))

        logging.info('{:d} ({:.4f}%) TRUE source sentences have been translated INCORRECTLY'
                     .format(stat_dict['num_true_samples_bad_translations'],
                             _pc(stat_dict['num_true_samples_bad_translations'], num_all_seed_samples)))

        logging.info('{:d} ({:.4f}%) TRUE source sentences have been translated with a DELETED HOMOGRAPH'
                     .format(stat_dict['num_true_samples_deleted_homograph'],
                             _pc(stat_dict['num_true_samples_deleted_homograph'], num_all_seed_samples)))

        logging.info('{:d} ({:.4f}%) TRUE source sentences have been translated MAYBE INCORRECTLY'
                     .format(stat_dict['num_true_samples_maybe_bad_translations'],
                             _pc(stat_dict['num_true_samples_maybe_bad_translations'], num_all_seed_samples)))

        logging.info('{:d} ({:.4f}%) ALL TRUE source sentences have been translated WRONG'
                     .format(stat_dict['num_true_samples_bad_translations'] +
                             stat_dict['num_true_samples_deleted_homograph'] +
                             stat_dict['num_true_samples_maybe_bad_translations'],
                             100 - _pc(stat_dict['num_true_samples_good_translations'], num_all_seed_samples)))

    # Read-in adversarial samples
    logging.info('Reading-in challenge samples table ...')
    with open(wsd_samples_path, 'r', encoding='utf8') as wsp:
        wsd_samples_table = json.load(wsp)

    # Read-in attractor table
    logging.info('Reading-in attractor table ...')
    with open(attractors_path, 'r', encoding='utf8') as ap:
        attractors_table = json.load(ap)

    # Load sense to cluster mappings
    logging.info('Reading-in sense clusters ...')
    with open(sense_clusters_path, 'r', encoding='utf8') as scp:
        sense_clusters_table = json.load(scp)

    # Add blacklisted, ambiguous terms to the set of acceptable translations
    for term in sense_clusters_table.keys():
        for cluster in sense_clusters_table[term].keys():
            if '[AMBIGUOUS SENSES]' in sense_clusters_table[term][cluster].keys():
                sense_clusters_table[term][cluster]['[SENSES]'] += \
                    sense_clusters_table[term][cluster]['[AMBIGUOUS SENSES]']
                sense_clusters_table[term][cluster].pop('[AMBIGUOUS SENSES]')
            if '[BLACKLISTED SENSES]' in sense_clusters_table[term][cluster].keys():
                sense_clusters_table[term][cluster]['[SENSES]'] += \
                    sense_clusters_table[term][cluster]['[BLACKLISTED SENSES]']
                sense_clusters_table[term][cluster].pop('[BLACKLISTED SENSES]')
            sense_clusters_table[term][cluster]['[SENSES]'] = \
                list(set(sense_clusters_table[term][cluster]['[SENSES]']))
    sense_to_cluster_table = _build_cluster_lookup(sense_clusters_table)

    # Post-process sense-to-cluster table
    sense_lemmas_to_cluster_table = sense_to_cluster_table
    sense_tokens_to_cluster_table = dict()
    for src_term in sense_lemmas_to_cluster_table.keys():
        sense_tokens_to_cluster_table[src_term] = dict()
        for cluster_tuple_list in sense_lemmas_to_cluster_table[src_term].values():
            for cluster_tuple in cluster_tuple_list:
                sense_tokens_to_cluster_table[src_term][cluster_tuple[0].lower()] = [cluster_tuple]

    cluster_to_sense_lemmas_table = dict()
    # Derive a cluster-to-senses table, used for compound analysis
    for src_term in sense_lemmas_to_cluster_table.keys():
        cluster_to_sense_lemmas_table[src_term] = dict()
        for sense, clusters in sense_lemmas_to_cluster_table[src_term].items():
            for cls_tpl in clusters:
                if not cluster_to_sense_lemmas_table[src_term].get(cls_tpl[1], None):
                    cluster_to_sense_lemmas_table[src_term][cls_tpl[1]] = [sense]
                else:
                    if sense not in cluster_to_sense_lemmas_table[src_term][cls_tpl[1]]:
                        cluster_to_sense_lemmas_table[src_term][cls_tpl[1]].append(sense)

    cluster_to_sense_tokens_table = dict()
    # Derive a cluster-to-senses table, used for compound analysis
    for src_term in sense_tokens_to_cluster_table.keys():
        cluster_to_sense_tokens_table[src_term] = dict()
        for sense, clusters in sense_lemmas_to_cluster_table[src_term].items():
            for cls_tpl in clusters:
                if not cluster_to_sense_tokens_table[src_term].get(cls_tpl[1], None):
                    cluster_to_sense_tokens_table[src_term][cls_tpl[1]] = [cls_tpl[0]]
                else:
                    if sense not in cluster_to_sense_tokens_table[src_term][cls_tpl[1]]:
                        cluster_to_sense_tokens_table[src_term][cls_tpl[1]].append(cls_tpl[0])

    # Restructure NMT translations for easier access
    logging.info('-' * 10)
    logging.info('Hashing challenge set samples ...')
    with open(sources_path, 'r', encoding='utf8') as sp:
        sources = [line for line_id, line in enumerate(sp)]
    with open(translations_path, 'r', encoding='utf8') as tp:
        translations = [line for line_id, line in enumerate(tp)]
    with open(alignments_path, 'r', encoding='utf8') as ap:
        alignments = [line for line_id, line in enumerate(ap)]
    logging.info('-' * 10)

    # Create a adversarial-to-original map
    src_to_sample = dict()
    for term in wsd_samples_table.keys():
        for true_cluster_id in wsd_samples_table[term].keys():
            for adv_cluster_id in wsd_samples_table[term][true_cluster_id].keys():
                for smp in wsd_samples_table[term][true_cluster_id][adv_cluster_id]:
                    src = smp[1].strip()
                    src_to_sample[src] = (smp, term)

    # Combine sources, samples, translations, and alignments
    logging.info('Mapping translations...')
    src_to_info = {src_line: [translations[line_id],
                              alignments[line_id],
                              src_to_sample[src_line.strip()][0],
                              src_to_sample[src_line.strip()][1]] for line_id, src_line
                   in enumerate(sources)}

    # For stats
    unique_samples = dict()

    # Seed translations
    true_samples_good_translations = dict()
    true_samples_bad_translations = dict()
    true_samples_deleted_homograph = dict()
    true_samples_maybe_bad_translations = dict()

    # Initialize variables for reporting
    stat_dict = {
        'num_true_samples_good_translations': 0,
        'num_true_samples_bad_translations': 0,
        'num_true_samples_deleted_homograph': 0,
        'num_true_samples_maybe_bad_translations': 0,
    }

    sent_id = 0
    for sent_id, sent in enumerate(src_to_info.keys()):

        sample = src_to_info[sent]
        term = sample[3]

        # TODO: DEBUGGING
        # if term not in ['anchor', 'clip']:
        #     continue

        # Apply adversarial filtering and compute LM-based fluency / acceptability scores
        _score_and_filter(sample, term, attractors_table[term])

        # Occasionally report statistics
        if sent_id > 0 and sent_id % 1000 == 0:
            logging.info('\nLooked up {:d} sentences; reporting intermediate statistics:'.format(sent_id + 1))
            _show_stats()

        # TODO: DEBUGGING
        # if term_id == 2:
        #     break

    # Final report
    logging.info('\nLooked up {:d} sentences; reporting FINAL statistics:'.format(sent_id + 1))
    _show_stats()

    # Construct output paths
    table_list = [
        true_samples_good_translations,
        true_samples_bad_translations,
        true_samples_deleted_homograph,
        true_samples_maybe_bad_translations,
    ]

    attack_success_path = os.path.join(output_tables_dir, 'attack_success')

    if not os.path.isdir(attack_success_path):
        os.mkdir(attack_success_path)

    path_list = [
        os.path.join(attack_success_path, 'wsd_samples_good_translations.{:s}'.format(src_lang)),
        os.path.join(attack_success_path, 'wsd_samples_bad_translations.{:s}'.format(src_lang)),
        os.path.join(attack_success_path, 'wsd_samples_deleted_homograph.{:s}'.format(src_lang)),
        os.path.join(attack_success_path, 'wsd_samples_maybe_bad_translations.{:s}'.format(src_lang)),
    ]

    # Save
    for table, path in zip(table_list, path_list):
        with open(path + '.json', 'w', encoding='utf8') as json_file:
            json.dump(table, json_file, indent=3, sort_keys=True, ensure_ascii=False)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--wsd_samples_path', type=str, required=True,
                        help='path to the JSON file containing selected challenge samples')
    parser.add_argument('--sources_path', type=str, default=None,
                        help='path to the text file containing the true source sentences')
    parser.add_argument('--translations_path', type=str, default=None,
                        help='path to the translations produced by the filter NMT model for the true source sentences')
    parser.add_argument('--alignments_path', type=str, default=None,
                        help='path to file containing the fastalign alignments for the translations of the original '
                             'challenge sentences')
    parser.add_argument('--sense_clusters_path', type=str, default=None,
                        help='path to the JSON file containing scraped BabelNet sense clusters')
    parser.add_argument('--attractors_path', type=str, required=True,
                        help='path to the JSON file containing the extracted attractor terms')
    parser.add_argument('--output_tables_dir', type=str, required=True,
                        help='path to the destination of the filtered adversarial samples tables')
    parser.add_argument('--lang_pair', type=str, default=None,
                        help='language pair of the bitext; expected format is src-tgt')
    args = parser.parse_args()

    # Logging to file
    base_dir = '/'.join(args.wsd_samples_path.split('/')[:-1])
    file_name = args.wsd_samples_path.split('/')[-1]
    file_name = '.'.join(file_name.split('.')[:-1])
    file_name = file_name if len(file_name) > 0 else 'log'
    log_dir = '{:s}/logs/'.format(base_dir)
    if not os.path.exists(log_dir):
        os.mkdir(log_dir)
    log_path = '{:s}{:s}.log'.format(log_dir, file_name)
    logging.basicConfig(filename=log_path, filemode='w', level=logging.INFO, format='%(levelname)s: %(message)s')
    # Logging to console
    console = logging.StreamHandler()
    console.setLevel(logging.INFO)
    logging.getLogger('').addHandler(console)

    # Instantiate processing pipeline
    src_lang, tgt_lang = args.lang_pair.strip().split('-')
    spacy_map = {'en': 'en_core_web_sm', 'de': 'de_core_news_sm'}
    try:
        src_nlp = spacy.load(spacy_map[src_lang], disable=['parser', 'ner', 'textcat'])
        tgt_nlp = spacy.load(spacy_map[tgt_lang], disable=['parser', 'ner', 'textcat'])
    except KeyError:
        logging.info('SpaCy does not support the language {:s} or {:s}. Exiting.'.format(src_lang, tgt_lang))
        sys.exit(0)
    pct_stripper = str.maketrans(string.punctuation, ' ' * len(string.punctuation))
    punctuation_plus_space = string.punctuation + ' ' + '\t' + '\n'
    # Import stopword list
    if src_lang == 'en':
        STOP_WORDS = stopwords.words('english')
    else:
        STOP_WORDS = []

    evaluate_attack_success(args.wsd_samples_path,
                            args.sources_path,
                            args.translations_path,
                            args.alignments_path,
                            args.attractors_path,
                            args.sense_clusters_path,
                            args.output_tables_dir)


import Utils

## Main program ##

matrix = Utils.loadMatrix()
if not Utils.isSymmetric(matrix):
	raise ValueError("Not a symmetric matrix.")
Q, R = Utils.QRdecompose(matrix)
eVectors, eValues = Utils.computeVV(Q, R)
Utils.outputResults(eVectors, eValues)


from typing import TYPE_CHECKING, List, Optional

from pydantic import root_validator

from vkbottle.tools.mini_types.base import BaseMessageMin

if TYPE_CHECKING:
    from vkbottle.api import ABCAPI

from .foreign_message import ForeignMessageMin  # noqa: TCH001

class MessageMin(BaseMessageMin):
    user_id: Optional[int] = None
    reply_message: Optional["ForeignMessageMin"] = None
    fwd_messages: Optional[List["ForeignMessageMin"]] = []

    @property
    def is_mentioned(self) -> bool:
        return self.mention.id == self.user_id if self.mention else False

    @root_validator(pre=True)
    def __foreign_messages(cls, values):
        foreign_messages = []
        if values.get("fwd_messages"):
            foreign_messages.extend(values["fwd_messages"])
        if values.get("reply_message"):
            foreign_messages.append(values["reply_message"])
        for foreign_message in foreign_messages:
            foreign_message["unprepared_ctx_api"] = values["unprepared_ctx_api"]
            foreign_message["replace_mention"] = values["replace_mention"]
            foreign_message["user_id"] = values.get("user_id")
        return values

MessageMin.update_forward_refs()

async def message_min(
    message_id: int,
    ctx_api: "ABCAPI",
    replace_mention: bool = True,
) -> "MessageMin":
    response = await ctx_api.messages.get_by_id(message_ids=[message_id])
    if not response.items:
        raise ValueError(f"Message with id {message_id} not found, perhaps it was deleted")
    message_object = response.items[0].dict()
    return MessageMin(
        **message_object,
        unprepared_ctx_api=ctx_api,
        replace_mention=replace_mention,
    )


import matplotlib.pyplot as plt
import matplotlib.colors as mcolors

color_list = ["Green", "Blue"]
color_map = mcolors.ListedColormap(["Green", "Blue"])

import seaborn as sns
sns.set()
sns.set_palette("tab10")

import random as rnd
import numpy as np
rnd.seed(0)
np.random.seed(0)

import scipy.stats as stats
import bayes_logistic
from pypolyagamma import PyPolyaGamma

#
from config import parse_args
from utils import print_config

def run_program(config=None):

    # get params
    args = parse_args()
    if config:
        args.update(config)
    print_config(args)

    # generate data_x
    data_x = generate_data_x(args)
    # print(data_x.shape)
    # stop

    # generate prior and posterior samples
    samples_a_weights_prior, samples_b_weights_prior, samples_a_weights_posterior = \
        generate_prior_and_posterior_samples(data_x, args)
    # print(samples_a_weights_prior.shape)
    # print(samples_a_weights_prior[:5])
    # print(samples_b_weights_prior[:5])
    # print(samples_a_weights_posterior[:5])
    # stop

    # Visualize the generated prior and posterior samples
    nrows = 2
    fig, axes = plt.subplots(nrows=nrows, ncols=1, sharex=True, sharey=True, figsize=(10,10))
    axes = axes.flatten()

    for i in range(nrows):
        sns.kdeplot(samples_a_weights_prior[:,0], fill=False, color="blue", label="sample_a_prior", ax=axes[i])
        sns.kdeplot(samples_b_weights_prior[:,0], fill=False, color="green", label="sample_b_prior", ax=axes[i])
        sns.kdeplot(samples_a_weights_posterior[:,0], fill=False, color="orange", label="sample_a_posterior", ax=axes[i])
        axes[i].legend()
    plt.savefig(f"./figures/{args['test_name']}_indi")
    plt.show()

    # Visualize the generated prior and posterior samples
    fig, axes = plt.subplots(
        nrows=3, ncols=1, sharex=True, sharey=True, figsize=(10,10))
    # axes = axes.flatten()

    sns.kdeplot(x=samples_a_weights_prior[:,0], y=samples_a_weights_prior[:,1],
                n_levels=20, cmap="inferno", fill=False, cbar=True, ax=axes[0])

    sns.kdeplot(x=samples_b_weights_prior[:,0], y=samples_b_weights_prior[:,1],
                n_levels=20, cmap="inferno", fill=False, cbar=True, ax=axes[1])

    sns.kdeplot(x=samples_a_weights_posterior[:,0], y=samples_a_weights_posterior[:,1],
                n_levels=20, cmap="inferno", fill=False, cbar=True, ax=axes[2])
    axes[0].set_title("sample_a_prior")
    axes[1].set_title("sample_b_prior")
    axes[2].set_title("sample_a_posterior")
    plt.savefig(f"./figures/{args['test_name']}_pair")
    plt.show()

def generate_data_x(args):
    # Generate data_x
    num_data = args["num_data"]
    num_feats = args["num_feats"]
    data_x_type = args["data_x_type"]

    if data_x_type == "twopoints":
        data_x = np.array([0, 1]*num_data//2)
    elif data_x_type == "uniform":
        data_x = np.random.randn(num_data, num_feats)
    elif data_x_type == "multigauss":
        data_x = np.random.rand(num_data, num_feats)
        mu = args["data_x_marginal_params"][0][0]
        sigma = args["data_x_marginal_params"][0][1]
        data_x = stats.multivariate_normal.rvs(mu, sigma, seed=12345)
    elif data_x_type == "mixgauss":
        data_x_marginal_dists = [
            stats.multivariate_normal(mu, sigma, seed=12345) \
            for mu, sigma in args["data_x_marginal_params"]]  # fixed random seed

        c0_x = data_x_marginal_dists[0].rvs(size=(num_data))
        c1_x = data_x_marginal_dists[1].rvs(size=(num_data))
        data_x = np.vstack((c0_x, c1_x))
    else:
        stop

    print(data_x.shape)

    return data_x

def generate_prior_and_posterior_samples(data_x, args):

    # np.random.seed(0)

    # params
    num_feats = args["num_feats"]
    num_samples = args["num_samples"]
    pglogistic_burnin_steps = args["pglogistic_burnin_steps"]

    # weights' prior distribution
    weights_prior_params = args["weights_prior_params"]

    weights_prior_dist_a = stats.multivariate_normal(
        weights_prior_params[0], weights_prior_params[1], seed=args['seed'])

    weights_prior_dist_b = stats.multivariate_normal(
        weights_prior_params[0], weights_prior_params[1], seed=args['seed']+10)

    samples_a_weights_prior = []
    samples_b_weights_prior = []
    samples_a_weights_posterior = []

    pg_dist = PyPolyaGamma(seed=0)
    for i in range(num_samples):
        # sample two set of weights' priors
        sample_a_weights_prior = weights_prior_dist_a.rvs(1)[None,:]
        sample_b_weights_prior = weights_prior_dist_b.rvs(1)[None,:]
        samples_a_weights_prior.append(sample_a_weights_prior)
        samples_b_weights_prior.append(sample_b_weights_prior)

        # generate sample y_i from theta_i in A
        sample_a_logit = 1.0 / (1 + np.exp(
            -np.matmul(data_x, sample_a_weights_prior.T)))
        sample_a_y = stats.bernoulli.rvs(sample_a_logit)
        # print(sample_a_y.shape)

        # sample weights' posterior from polyagamma inference
        sample_a_weights_posterior = pg_inference(
            data_x, sample_a_y.squeeze(-1), weights_prior_params, pg_dist,
            burnin_steps=pglogistic_burnin_steps)
        samples_a_weights_posterior.append(sample_a_weights_posterior)

    samples_a_weights_prior = np.vstack(samples_a_weights_prior)
    samples_b_weights_prior = np.vstack(samples_b_weights_prior)
    samples_a_weights_posterior = np.vstack(samples_a_weights_posterior)
    # print(samples_a_weights_prior.shape)
    # print(samples_b_weights_prior)
    # print(samples_a_weights_posterior)
    return samples_a_weights_prior, samples_b_weights_prior, \
        samples_a_weights_posterior

def pg_inference(X, y, weights_prior_params, pg_dist, burnin_steps=200):
    #
    num_data, num_feats = X.shape

    # Gibbs sampling with PG augmentation for burnin_steps
    # init states
    beta_mu = np.array(weights_prior_params[0])
    beta_cov = np.array(weights_prior_params[1])
    # beta_cov = np.diag(np.ones(num_feats))
    beta_hat = np.random.multivariate_normal(beta_mu, beta_cov)
    k = y - 1/2

    # pg = PyPolyaGamma(seed=0)
    # perform Gibbs sampling
    for bid in range(burnin_steps+1):
        # ω ~ PG(b, c) = PG(1, x*β).
        omega_b = np.ones(num_data)
        omega_c = X @ beta_hat
        omega_diag = np.array(
            [pg_dist.pgdraw(b, c) for b, c in zip(omega_b, omega_c)])

        # β ~ N(m, V).
        V = np.linalg.inv(X.T @ np.diag(omega_diag) @ X + np.linalg.inv(beta_cov))
        m = np.dot(V, X.T @ k + np.linalg.inv(beta_cov) @ beta_mu)
        beta_hat = np.random.multivariate_normal(m, V)

    return beta_hat

if __name__ == '__main__':
    run_program()


class Solution:
    def findCriticalAndPseudoCriticalEdges(self, n: int, edges: List[List[int]]) -> List[List[int]]:

        originalEdges = edges[:]
        edges = sorted(edges, key = lambda x: x[2])

        def kruskalUsingDSU(edges, includeEdge = None, excludeEdge = None):
            print(includeEdge, excludeEdge)
            parent = [i for i in range(n)]
            def find(x):
                if parent[x] != x:
                    parent[x] = find(parent[x])
                return parent[x]

            def union(x, y):
                p1 = find(x)
                p2 = find(y)
                if p1!=p2:
                    parent[p2] = p1
                    return True
                else:
                    False

            mst = []
            cost = 0
            if includeEdge:
                if union(includeEdge[0], includeEdge[1]):
                    cost += includeEdge[2]
            for u, v, w in edges:
                if [u, v, w] == includeEdge or [u, v, w] == excludeEdge:
                    continue
                if union(u, v):
                    cost += w
            return cost

        originalCost = kruskalUsingDSU(edges)
        print(originalCost)
        critical = []
        pseudoCritical = []
        for i in range(len(originalEdges)):
            exclude = kruskalUsingDSU(edges, None,  originalEdges[i])
            include = kruskalUsingDSU(edges, originalEdges[i], None)
            if exclude > originalCost or exclude < originalCost:
                critical.append(i)
            elif include == originalCost:
                pseudoCritical.append(i)

        return [critical, pseudoCritical]


import yaml

from testtools.matchers import Equals

import snapcraft.internal
from snapcraft.tests import unit

class BuildStateBaseTestCase(unit.TestCase):

    def setUp(self):
        super().setUp()

        class Project:
            def __init__(self):
                self.deb_arch = 'amd64'

        self.project = Project()
        self.property_names = ['foo']
        self.part_properties = {'foo': 'bar'}

        self.state = snapcraft.internal.states.BuildState(
            self.property_names, self.part_properties, self.project)

class BuildStateTestCase(BuildStateBaseTestCase):

    def test_yaml_conversion(self):
        state_from_yaml = yaml.load(yaml.dump(self.state))
        self.assertThat(state_from_yaml, Equals(self.state))

    def test_comparison(self):
        other = snapcraft.internal.states.BuildState(
            self.property_names, self.part_properties, self.project)

        self.assertTrue(self.state == other, 'Expected states to be identical')

    def test_properties_of_interest(self):
        self.part_properties.update({
            'after': 'test-after',
            'build-attributes': ['test-build-attribute'],
            'build-packages': 'test-build-packages',
            'disable-parallel': 'test-disable-parallel',
            'organize': {'baz': 'qux'},
            'prepare': 'touch prepare',
            'build': 'touch build',
            'install': 'touch install',
        })

        properties = self.state.properties_of_interest(self.part_properties)
        self.assertThat(len(properties), Equals(9))
        self.assertThat(properties['foo'], Equals('bar'))
        self.assertThat(properties['after'], Equals('test-after'))
        self.assertThat(
            properties['build-attributes'], Equals(['test-build-attribute']))
        self.assertThat(
            properties['build-packages'], Equals('test-build-packages'))
        self.assertThat(
            properties['disable-parallel'], Equals('test-disable-parallel'))
        self.assertThat(properties['organize'], Equals({'baz': 'qux'}))
        self.assertThat(properties['prepare'], Equals('touch prepare'))
        self.assertThat(properties['build'], Equals('touch build'))
        self.assertThat(properties['install'], Equals('touch install'))

    def test_project_options_of_interest(self):
        options = self.state.project_options_of_interest(self.project)

        self.assertThat(len(options), Equals(1))
        self.assertThat(options['deb_arch'], Equals('amd64'))

class BuildStateNotEqualTestCase(BuildStateBaseTestCase):

    scenarios = [
        ('no property names', dict(
            other_property='property_names', other_value=[])),
        ('no part properties', dict(
            other_property='part_properties', other_value=None)),
        ('no project', dict(
            other_property='project', other_value=None)),
    ]

    def test_comparison_not_equal(self):
        setattr(self, self.other_property, self.other_value)
        other_state = snapcraft.internal.states.BuildState(
            self.property_names, self.part_properties, self.project)
        self.assertFalse(
            self.state == other_state,
            'Expected states to be different')


__author__ = 'Tsoingkam'
__date__ = '2018/4/13 21:52'

from django.conf.urls import url

from .views import LoginView, RegisterView, ActiveView, ForgetPWDView, ChangePWDView

app_name = 'users'
urlpatterns = [
    url(r'^login/$', LoginView.as_view(), name='login'),
    url(r'^register/$', RegisterView.as_view(), name='register'),
    url(r'^active/(?P<active_code>.*)$', ActiveView.as_view(), name='active'),
    url(r'^forget/$', ForgetPWDView.as_view(), name='forget_pwd'),
    url(r'^changepwd/$', ChangePWDView.as_view(), name='change_pwd'),
]


from typing import FrozenSet, Optional, Set

from pipenv.patched.pip._vendor.packaging.utils import canonicalize_name

from pipenv.patched.pip._internal.exceptions import CommandError

class FormatControl:
    """Helper for managing formats from which a package can be installed."""

    __slots__ = ["no_binary", "only_binary"]

    def __init__(
        self,
        no_binary: Optional[Set[str]] = None,
        only_binary: Optional[Set[str]] = None,
    ) -> None:
        if no_binary is None:
            no_binary = set()
        if only_binary is None:
            only_binary = set()

        self.no_binary = no_binary
        self.only_binary = only_binary

    def __eq__(self, other: object) -> bool:
        if not isinstance(other, self.__class__):
            return NotImplemented

        if self.__slots__ != other.__slots__:
            return False

        return all(getattr(self, k) == getattr(other, k) for k in self.__slots__)

    def __repr__(self) -> str:
        return "{}({}, {})".format(
            self.__class__.__name__, self.no_binary, self.only_binary
        )

    @staticmethod
    def handle_mutual_excludes(value: str, target: Set[str], other: Set[str]) -> None:
        if value.startswith("-"):
            raise CommandError(
                "--no-binary / --only-binary option requires 1 argument."
            )
        new = value.split(",")
        while ":all:" in new:
            other.clear()
            target.clear()
            target.add(":all:")
            del new[: new.index(":all:") + 1]
            # Without a none, we want to discard everything as :all: covers it
            if ":none:" not in new:
                return
        for name in new:
            if name == ":none:":
                target.clear()
                continue
            name = canonicalize_name(name)
            other.discard(name)
            target.add(name)

    def get_allowed_formats(self, canonical_name: str) -> FrozenSet[str]:
        result = {"binary", "source"}
        if canonical_name in self.only_binary:
            result.discard("source")
        elif canonical_name in self.no_binary:
            result.discard("binary")
        elif ":all:" in self.only_binary:
            result.discard("source")
        elif ":all:" in self.no_binary:
            result.discard("binary")
        return frozenset(result)

    def disallow_binaries(self) -> None:
        self.handle_mutual_excludes(
            ":all:",
            self.no_binary,
            self.only_binary,
        )