from typing import Optional, Sequence, Union

from libcst._nodes.whitespace import (
    EmptyLine,
    Newline,
    ParenthesizedWhitespace,
    SimpleWhitespace,
    TrailingWhitespace,
)
from libcst._parser.types.config import BaseWhitespaceParserConfig as Config
from libcst._parser.types.whitespace_state import WhitespaceState as State

def parse_simple_whitespace(config: Config, state: State) -> SimpleWhitespace: ...
def parse_empty_lines(
    config: Config,
    state: State,
    *,
    override_absolute_indent: Optional[str] = None,
) -> Sequence[EmptyLine]: ...
def parse_trailing_whitespace(config: Config, state: State) -> TrailingWhitespace: ...
def parse_parenthesizable_whitespace(
    config: Config, state: State
) -> Union[SimpleWhitespace, ParenthesizedWhitespace]: ...


def solution(s):
    answer = []

    s = s.replace('{', '')
    s = list(s)

    tuple_lst = []

    # s의 각각의 원소를 리스트로 바꾸는 과정
    lst, tmp = [], ''
    for i in range(len(s) - 1):
        if s[i].isdigit():
            if s[i + 1].isdigit():
                tmp += s[i]
            else:
                lst.append(int(tmp + s[i]))
                tmp = ''

        if s[i] == '}':             # 닫힌 괄호가 나오면 튜플 리스트에 원소들 넣기
            tuple_lst.append(lst)
            lst = []

    tuple_lst.sort(key = lambda x : len(x))     # 집합의 길이로 오름차순 정렬

    for i in tuple_lst:
        for j in i:
            if j not in answer:
                answer.append(j)

    return answer


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/5/5 21:06
# @Author  : ChenYao
# @File    : python_thread.py

'''
setDaemon会把线程编程守护线程，当主线程退出之后，子线程也会退出
'''

import time
import threading

def get_detail_html(url):
    print("get detail html started")
    time.sleep(2)
    print("get detail html end")

def get_detail_url(url):
    print("get detail url started")
    time.sleep(4)
    print("get detail url end")

#2. 通过集成Thread来实现多线程

class GetDetailHtml(threading.Thread):
    def __init__(self, name):
        super().__init__(name=name)

    def run(self):
        print("get detail html started")
        time.sleep(2)
        print("get detail html end")

class GetDetailUrl(threading.Thread):
    def __init__(self, name):
        super().__init__(name=name)

    def run(self):
        print("get detail url started")
        time.sleep(4)
        print("get detail url end")

if  __name__ == "__main__":
    thread1 = GetDetailHtml("get_detail_html")
    thread2 = GetDetailUrl("get_detail_url")
    # thread1.setDaemon(True)
    # thread2.setDaemon(True)
    start_time = time.time()
    thread1.start()
    thread2.start()

    thread1.join()
    thread2.join()

    #当主线程退出的时候， 子线程kill掉
    print ("last time: {}".format(time.time()-start_time))


# THIS MODULE CONTAINS VARIOUS UTILITY FUNCTIONS USED
# BY ALL OTHER MODULES IN JETFREQ

import re
import jfexceptions
import os
from jfanalyze import sort_events
from jfanalyze import EventFreq
from jfanalyze import EventDiff
from jfanalyze import DiffType
from datetime import datetime

# THIS FUNCTION FORMATS AND WRITES DEBUG INFORMATION TO STDOUT
def debug(verbose, message):
	if verbose:
		if type(message) == list:
			for line in message:
				print("jetfreq.py: {}".format(line))
		else:
			print("jetfreq.py: {}".format(message))

# THIS FUNCTION TRUNCATES ANY PATH WITH A HIERARCHY GREATER THAN
# 4 DOWN TO THE FIRST 3 AND THE LAST 1 FOR BREVITY
def truncate_path(path, path_type):
	depth = 4 if path_type == 'dir' else 6;
	if '\\' in path:
		path_ary = path.split('\\')
		if len(path_ary) > depth:
			path = '\\'.join(path_ary[0:depth - 1]) + '\\ ... \\' + path_ary[len(path_ary) - 1]
	elif '/' in path:
		path_ary = path.split('/')
		if len(path_ary) > depth:
			path = '/'.join(path_ary[0:depth - 1]) + '/ ... /' + path_ary[len(path_ary) - 1]
	return path

# THIS FUNCTION SORT A LIST OF EVENT_DIFF OBJECTS
# ALPHABETICALLY BY THEIR DIFFTYPE VALUE
def sort_event_diffs_by_type(event_diffs):
	for i in range(len(event_diffs)):
		j = i + 1
		while j < len(event_diffs):
			if event_diffs[i].difftype > event_diffs[j].difftype:
				hold = event_diffs[i]
				event_diffs[i] = event_diffs[j]
				event_diffs[j] = hold
			j = j + 1
	return event_diffs

# THIS FUNCTION FORMATS THE CONTENTS OF EVENT_DIFF OBJECTS
# AND APPENDS IT TO THE REPORT LIST
def append_diff_to_report(report, event_diffs, event_type, truncate):
	dt = DiffType()
	report.append('::::::')
	report.append(':::::: {}'.format(event_type))
	report.append(':::::: {:<61} | {:<15} | PATH'.format('DIFFERENCE TYPE', 'FREQ'))
	report.append(':::::: --------------------------------------------------------------|-----------------|------>')

	path_type = 'reg' if event_type.upper().startswith('REGMOD') else 'dir'
	event_diffs = sort_event_diffs_by_type(event_diffs)
	for diff in event_diffs:
		if diff.difftype == dt.MISS_FM_REP:
			report.append(':::::: {:<61} | {:<15} | {}'.format(
				diff.difftype,
				'n/a',
				truncate_path(diff.target_event.path, path_type) if truncate else diff.target_event.path))
		elif diff.difftype == dt.MISS_FM_TAR:
			report.append(':::::: {:<61} | {:<15} | {}'.format(
				diff.difftype,
				'n/a',
				truncate_path(diff.representative_event.path, path_type) if truncate else diff.representative_event.path))
		elif diff.difftype == dt.HIGH_FQ_REP:
			report.append(':::::: {:<61} | {:<15} | {}'.format(
				diff.difftype,
				'{:<6.4f} > {:<6.4f}'.format(float(diff.representative_event.perc), float(diff.target_event.perc)),
				truncate_path(diff.target_event.path, path_type) if truncate else diff.target_event.path))
		elif diff.difftype == dt.HIGH_FQ_TAR:
			report.append(':::::: {:<61} | {:<15} | {}'.format(
				diff.difftype,
				'{:<6.4f} > {:<6.4f}'.format(float(diff.target_event.perc), float(diff.representative_event.perc)),
				truncate_path(diff.target_event.path, path_type) if truncate else diff.target_event.path))
		elif diff.difftype.startswith('TARGET SAMPLE HAS NO'):
			report.append(':::::: {:<61} | {:<15} | {}'.format(
				diff.difftype,
				'n/a',
				truncate_path(diff.representative_event.path, path_type) if truncate else diff.representative_event.path))
		elif diff.difftype.startswith('REPRESENTATIVE SAMPLE HAS NO'):
			report.append(':::::: {:<61} | {:<15} | {}'.format(
				diff.difftype,
				'n/a',
				truncate_path(diff.target_event.path, path_type) if truncate else diff.target_event.path))
	return report

# THIS FUNCTION FORMATS THE CONTENTS OF EVENT_FREQ OBJECTS
# AND APPENDS IT TO THE REPORT LIST
def append_to_report(report, event_freqs, event_type, truncate):
	report.append('::::::')
	report.append(':::::: {}'.format(event_type))
	report.append(':::::: RATIO   | FREQ   | PATH')
	report.append(':::::: --------|--------|------->')
	path_type = 'reg' if event_type.upper().startswith('REGMOD') else 'dir'
	for event in event_freqs:
		report.append(':::::: {:<7} | {:<6.4f} | {}'.format(
			'{}/{}'.format(event.count, event.total),
			event.perc,
			truncate_path(event.path, path_type) if truncate else event.path))
	return report

# THIS IS THE MAIN FUNCTION FOR FORMATTING THE RESULTS
# OF A --BY-PROCESS MODE QUERY INTO A REPORT
def format_report_by_process(params, event_freqs):

	debug(params['verbose'], 'Generating report')
	report = []

	# CHECK THAT THERE ARE, IN FACT, RESULTS TO REPORT
	no_results = True
	for key in event_freqs:
		if not event_freqs[key] == None and len(event_freqs[key]) > 0:
			no_results = False
			break
	# OTHERWISE, YOU SHALL NOT PASS
	if no_results == True:
		report.append('::::::')
		report.append(':::::: NO RESULTS FOR PROCESS {}'.format(params['search_name'].upper()))
		return report

	# APPEND THE REPORT HEADERS TO THE REPORT
	report.append('::::::')
	report.append(':::::: RESULTS FOR PROCESS {}'.format(params['search_name'].upper()))
	report.append('::::::')
	report.append(':::::: FILTERS')
	report.append(':::::: start_time = {}'.format(params['start_time']))
	if params['threshold_lt'] != None:
		report.append(':::::: threshold (less than) = {}%'.format(params['threshold_lt']))
	if params['threshold_gt'] != None:
		report.append(':::::: threshold (greater than) = {}%'.format(params['threshold_gt']))
	report.append(':::::: sample_size = {}'.format(params['sample_size']))
	if not params['user_name'] == None:
		report.append(':::::: user_name = {}'.format(params['user_name']))
	elif not params['exclude_user'] == None:
		report.append(':::::: exclude_user = {}'.format(params['exclude_user']))
	if not params['host_name'] == None:
		report.append(':::::: host_name = {}'.format(params['host_name']))
	elif not params['exclude_host'] == None:
		report.append(':::::: exclude_host = {}'.format(params['exclude_host']))

	# APPEND THE EVENTS TO THE REPORT
	if not event_freqs['modloads'] == None and len(event_freqs['modloads']) > 0:
		report = append_to_report(report, event_freqs['modloads'], 'MODLOADS', params['truncate'])
	if not event_freqs['regmods'] == None and len(event_freqs['regmods']) > 0:
		report = append_to_report(report, event_freqs['regmods'], 'REGMODS', params['truncate'])
	if not event_freqs['childprocs'] == None and len(event_freqs['childprocs']) > 0:
		report = append_to_report(report, event_freqs['childprocs'], 'CHILDPROCS', params['truncate'])
	if not event_freqs['filemods'] == None and len(event_freqs['filemods']) > 0:
		report = append_to_report(report, event_freqs['filemods'], 'FILEMODS', params['truncate'])
	if not event_freqs['netconns'] == None and len(event_freqs['netconns']) > 0:
		report = append_to_report(report, event_freqs['netconns'], 'NETCONNS', params['truncate'])
	if not event_freqs['crossprocs'] == None and len(event_freqs['crossprocs']) > 0:
		report = append_to_report(report, event_freqs['crossprocs'], 'CROSSPROCS', params['truncate'])

	# CLOSE AND RETURN THE REPORT
	report.append('::::::')
	report.append(':::::: END')
	return report

# THIS IS THE MAIN FUNCTION FOR FORMATTING THE RESULTS OF A
# --BY-EVENT QUERY INTO A REPORT
def format_report_by_event(params, event_freqs):

	debug(params['verbose'], 'Generating report')
	report = []

	# GET THE EVENT TYPE THAT WAS IN THE QUERY
	event_type = get_event_type_flags(params)
	if event_type == 'm':
		event_type = 'MODLOAD'
	elif event_type == 'r':
		event_type = 'REGMOD'
	elif event_type == 'f':
		event_type = 'FILEMOD'
	elif event_type == 'c':
		event_type = 'CHILDMOD'
	elif event_type == 'd':
		event_type = 'NETCONN'

	# IF THERE ARE NO RESULTS, GO NO FURTHER
	if len(event_freqs) == 0:
		report.append('::::::')
		report.append(':::::: NO RESULTS FOR {} {}'.format(event_type, params['search_name'].upper()))
		return report

	# APPEND THE REPORT HEADERS TO THE REPORT
	report.append('::::::')
	report.append(':::::: RESULTS FOR {} {}'.format(event_type, params['search_name'].upper()))
	report.append('::::::')
	report.append(':::::: FILTERS')
	report.append(':::::: start_time = {}'.format(params['start_time']))
	if params['threshold_lt'] != None:
		report.append(':::::: threshold (less than) = {}%'.format(params['threshold_lt']))
	if params['threshold_gt'] != None:
		report.append(':::::: threshold (greater than) = {}%'.format(params['threshold_gt']))
	report.append(':::::: sample_size = {}'.format(params['sample_size']))
	if not params['user_name'] == None:
		report.append(':::::: user_name = {}'.format(params['user_name']))
	elif not params['exclude_user'] == None:
		report.append(':::::: exclude_user = {}'.format(params['exclude_user']))
	if not params['host_name'] == None:
		report.append(':::::: host_name = {}'.format(params['host_name']))
	elif not params['exclude_host'] == None:
		report.append(':::::: exclude_host = {}'.format(params['exclude_host']))

	# APPEND THE PROCESSES TO THE REPORT
	report = append_to_report(report, event_freqs, 'PROCESSES', params['truncate'])

	# CLOSE AND RETURN THE REPORT
	report.append('::::::')
	report.append(':::::: END')
	return report

# THIS FUNCTION TAKES A FILE NAME AND USES IT TO
# CREATE A PARAMS JSON CONTAINER TO HOLD THE PARAMETERS
# USED IN THE SEARCH THAT GENERATED THE FILE
def get_params_from_file_name(file_name):
	# INITIALIZE JSON CONTAINER FOR PARAMETERS
	params = {
		'search_name':None,
		'regmods':False,
		'filemods':False,
		'modloads':False,
		'crossprocs':False,
		'childprocs':False,
		'netconns':False,
		'sample_size':10,
		'user_name':None,
		'exclude_user':None,
		'host_name':None,
		'exclude_host':None,
		'threshold_lt':100,
		'threshold_gt':0,
		'event_type':None
		}

	# SPLIT THE NAME AT THE DEFINED FLAGS, INTO AN ARRAY
	name_ary = re.split(r'_[hHuUtTnse]{1}-', file_name[0:len(file_name) - 4]) # trim .csv
	params['search_name'] = name_ary[1].replace('__BS__', '\\')
	params['sample_size'] = int(name_ary[3])
	params['threshold_lt'] = int(name_ary[4]) if not name_ary[4] == "None" else "None"
	params['threshold_gt'] = int(name_ary[5]) if not name_ary[5] == "None" else "None"
	flags = name_ary[2]
	if 'r' in flags:
		params['regmods'] = True
		params['event_type'] = 'REGMOD'
	if 'f' in flags:
		params['filemods'] = True
		params['event_type'] = 'FILEMOD'
	if 'm' in flags:
		params['modloads'] = True
		params['event_type'] = 'MODLOAD'
	if 'x' in flags:
		params['crossprocs'] = True
		params['event_type'] = 'CROSSPROC'
	if 'c' in flags:
		params['childprocs'] = True
		params['event_type'] = 'CHILDPROC'
	if 'd' in flags:
		params['netconns'] = True
		params['event_type'] = 'NETCONN'
	# IF THERE ARE 8 ELEMENTS IN THE NAME_ARY, THEN A USER FLAG AND A HOST FLAG WAS USED
	if len(name_ary) == 8:
		if '_u-' in file_name:
			params['user_name'] = name_ary[6]
		elif '_U-' in file_name:
			params['exclude_user'] = name_ary[6]
		if '_h-' in file_name:
			params['host_name'] = name_ary[7]
		elif '_H-' in file_name:
			params['exclude_host'] = name_ary[7]
	# IF THERE ARE 7 ELEMENTS IN THE NAME_ARY, THEN EITHER A USER FLAG OR A HOST FLAG WAS USED
	elif len(name_ary) == 7:
		if '_h-' in file_name:
			params['host_name'] = name_ary[6]
		elif '_H-' in file_name:
			params['exclude_host'] = name_ary[6]
		elif '_u-' in file_name:
			params['user_name'] = name_ary[6]
		elif '_U-' in file_name:
			params['exclude_user'] = name_ary[6]
	return params

# THIS IS THE MAIN FUNCTION FOR FORMATTING THE RESULTS OF A
# --COMPARE-PROCESS MODE QUERY INTO A REPORT
def format_report_compare_process(params, event_freqs):
	report = []

	# CHECK THAT THERE ARE INDEED RESULTS
	no_results = True
	for key in event_freqs:
		if not event_freqs[key] == None and len(event_freqs[key]) > 0:
			no_results = False
			break
	# IF NOT, GO NO FURTHER
	if no_results:
		report.append('::::::')
		report.append(':::::: TARGET AND REPRESENTATIVE SAMPLES ARE THE SAME')
		return report

	# EXTRACT THE PARAMETERS USED TO GENERATE THE REPRESENTATIVE SAMPLE FROM ITS FILE NAME
	representative_sample_params = get_params_from_file_name(params['import_sample'])

	# APPEND THE REPORT HEADERS TO THE REPORT
	report.append('::::::')
	report.append(':::::: PROCESS COMPARISON RESULTS')
	report.append('::::::')
	report.append(':::::: REPRESENTATIVE SAMPLE PROCESS {}'.format(representative_sample_params['search_name'].upper()))
	report.append(':::::: file = {}'.format(params['import_sample'].replace('__BS__', '\\')))
	if representative_sample_params['threshold_lt'] != None:
		report.append(':::::: threshold (less than) = {}%'.format(representative_sample_params['threshold_lt']))
	if representative_sample_params['threshold_gt'] != None:
		report.append(':::::: threshold (greather than) = {}%'.format(representative_sample_params['threshold_gt']))
	report.append(':::::: sample_size = {}'.format(representative_sample_params['sample_size']))
	if not representative_sample_params['user_name'] == None:
		report.append(':::::: user_name = {}'.format(representative_sample_params['user_name']))
	elif not representative_sample_params['exclude_user'] == None:
		report.append(':::::: exclude_user = {}'.format(representative_sample_params['exclude_user']))
	if not representative_sample_params['host_name'] == None:
		report.append(':::::: host_name = {}'.format(representative_sample_params['host_name']))
	elif not representative_sample_params['exclude_host'] == None:
		report.append(':::::: exclude_host = {}'.format(representative_sample_params['exclude_host']))
	report.append('::::::')
	report.append(':::::: TARGET SAMPLE PROCESS {}'.format(params['search_name'].upper()))
	report.append(':::::: start_time = {}'.format(params['start_time']))
	if params['threshold_lt'] != None:
		report.append(':::::: threshold (less than) = {}%'.format(params['threshold_lt']))
	if params['threshold_gt'] != None:
		report.append(':::::: threshold (greater than) = {}%'.format(params['threshold_gt']))
	report.append(':::::: sample_size = {}'.format(params['sample_size']))
	if not params['user_name'] == None:
		report.append(':::::: user_name = {}'.format(params['user_name']))
	elif not params['exclude_user'] == None:
		report.append(':::::: exclude_user = {}'.format(params['exclude_user']))
	if not params['host_name'] == None:
		report.append(':::::: host_name = {}'.format(params['host_name']))
	elif not params['exclude_host'] == None:
		report.append(':::::: exclude_host = {}'.format(params['exclude_host']))

	# APPEND THE EVENT DIFFERENCES TO THE REPORT
	for key in event_freqs:
		if not event_freqs[key] == None and len(event_freqs[key]) > 0:
			report = append_diff_to_report(report, event_freqs[key], key.upper(), params['truncate'])

	# CLOSE AND RETURN REPORT
	report.append('::::::')
	report.append(':::::: END')
	return report

# THIS IS THE MAIN FUNCTION FOR FORMATTING THE RESULTS
# OF A --COMPARE-EVENT QUERY INTO A REPORT
def format_report_compare_event(params, event_freqs):
	report = []

	# IF THERE ARE NO RESULTS, GO NO FURTHER
	if len(event_freqs) == 0:
		report.append('::::::')
		report.append(':::::: TARGET AND REPRESENTATIVE SAMPLES ARE THE SAME')
		return report

	# GET THE EVENT TYPE USED IN THE TARGET SAMPLE
	event_type = 'NETCONN'
	if params['modloads'] == True:
		event_type = 'MODLOAD'
	elif params['regmods'] == True:
		event_type = 'REGMOD'
	elif params['filemods'] == True:
		event_type = 'FILEMOD'
	elif params['childprocs'] == True:
		event_type = 'CHILDPROC'

	# EXTRACT THE PARAMETERS USED TO GENERATE THE REPRESENTATIVE SAMPLE FROM THE FILE NAME
	representative_sample_params = get_params_from_file_name(params['import_sample'])

	# APPEND THE REPORT HEADERS TO THE REPORT
	report.append('::::::')
	report.append(':::::: EVENT COMPARISON RESULTS')
	report.append('::::::')
	report.append(':::::: REPRESENTATIVE SAMPLE {} {}'.format(representative_sample_params['event_type'], representative_sample_params['search_name'].upper()))
	report.append(':::::: file = {}'.format(params['import_sample'].replace('__BS__', '\\')))
	report.append(':::::: threshold (less than) = {}'.format(representative_sample_params['threshold_lt']))
	report.append(':::::: threshold (greater than) = {}'.format(representative_sample_params['threshold_gt']))
	report.append(':::::: sample_size = {}'.format(representative_sample_params['sample_size']))
	if not representative_sample_params['user_name'] == None:
		report.append(':::::: user_name = {}'.format(representative_sample_params['user_name']))
	elif not representative_sample_params['exclude_user'] == None:
		report.append(':::::: exclude_user = {}'.format(representative_sample_params['exclude_user']))
	if not representative_sample_params['host_name'] == None:
		report.append(':::::: host_name = {}'.format(representative_sample_params['host_name']))
	elif not representative_sample_params['exclude_host'] == None:
		report.append(':::::: exclude_host = {}'.format(representative_sample_params['exclude_host']))
	report.append('::::::')
	report.append(':::::: TARGET SAMPLE {} {}'.format(event_type, params['search_name'].upper()))
	report.append(':::::: start_time = {}'.format(params['start_time']))
	report.append(':::::: threshold (less than) = {}'.format(params['threshold_lt']))
	report.append(':::::: threshold (greater than) = {}'.format(params['threshold_gt']))
	report.append(':::::: sample_size = {}'.format(params['sample_size']))
	if not params['user_name'] == None:
		report.append(':::::: user_name = {}'.format(params['user_name']))
	elif not params['exclude_user'] == None:
		report.append(':::::: exclude_user = {}'.format(params['exclude_user']))
	if not params['host_name'] == None:
		report.append(':::::: host_name = {}'.format(params['host_name']))
	elif not params['exclude_host'] == None:
		report.append(':::::: exclude_host = {}'.format(params['exclude_host']))

	# APPEND THE PROCESS DIFFERENCES TO THE REPORT
	report = append_diff_to_report(report, event_freqs, 'PROCESSES', params['truncate'])

	# CLOSE AND RETURN THE REPORT
	report.append('::::::')
	report.append(':::::: END')
	return report

# THIS FUNCTION EXTRACTS THE EVENT TYPE FLAGS
# FROM PARAMS AND RETURN THEM IN A STRING
def get_event_type_flags(params):
	flags = ""
	if params['modloads'] == True:
		flags += 'm'
	if params['regmods'] == True:
		flags += 'r'
	if params['filemods'] == True:
		flags += 'f'
	if params['crossprocs'] == True:
		flags += 'x'
	if params['childprocs'] == True:
		flags += 'c'
	if params['netconns'] == True:
		flags += 'd'
	return flags

# THIS FUNCTION IMPORTS THE CONTENTS OF A SAMPLE FILE
# AND RETURNS THEM AS A LIST OR DICTIONARY OF EVENT_FREQ OBJECTS
def import_sample(params):
	# INITIALIZE THE COLUMN INDEX VARIABLES
	search_name = None
	event_type = None
	path = None
	count = None
	total = None
	freq = None

	# CHECK THAT THE USER HASN'T ACCIDENTALLY COPIED DIRECTORY SECTIONS INTO THE FILEPATH
	file_path = ''
	if '/' in params['import_sample']:
		hold = params['import_sample'].split('/')
		params['import_sample'] = hold[len(hold) - 1]

	# FORMAT THE IMPORT_SAMPLE VALUE AS A FILE PATH
	file_path = './samples/{}/{}'.format('process' if params['mode'] == 'COMPARE_PROCESS' else 'event', params['import_sample'])

	# THE COLUMN IN THE .CSV FILE DIFFERS DEPENDING UPON WHETHER THE
	# FILE IS THE RESULT OF A PROCESS OR EVENT SEARCH
	if params['mode'] == 'COMPARE_PROCESS':
		search_name = 0
		event_type = 1
		path = 2
		count = 3
		total = 4
		freq = 5
	elif params['mode'] == 'COMPARE_EVENT':
		search_name = 0
		path = 1
		count = 2
		total = 3
		freq = 4

	# INITIALIZE THE EVENT_FREQS CONTAINER AS A LIST (FOR --COMPARE-EVENT) OR DICTIONARY (FOR --COMPARE-PROCESS)
	event_freqs = [] if event_type == None else {'modloads':None, 'regmods':None, 'childprocs':None, 'filemods':None, 'netconns':None, 'crossprocs':None}
	file = open(file_path, 'r')
	skip = True
	for line in file:
		# SKIP THE FIRST LINE IN THE FILE (I.E. THE HEADERS)
		if skip:
			skip = False
		else:
			# EXTRACT THE VALUES FROM THE LINE IN THE FILE AND REMOVE QUOTATION MARKS
			path_value = line.split(',')[path].replace('\'','')
			freq_value = line.split(',')[freq].replace('\'','')
			freq_value = freq_value.replace('\n','')
			count_value = line.split(',')[count].replace('\'','')
			total_value = line.split(',')[total].replace('\'','')
			if not event_type == None:
				# IF RUNNING IN --COMPARE-PROCESS MODE, RETRIEVE THE EVENT_TYPE VALUE
				event_type_value = '{}s'.format(line.split(',')[event_type].replace('\'',''))
				if not event_freqs[event_type_value] == None:
					event_freqs[event_type_value].append(EventFreq(path_value, freq_value, count_value, total_value))
				else:
					event_freqs[event_type_value] = []
					event_freqs[event_type_value].append(EventFreq(path_value, freq_value, count_value, total_value))
			else:
				event_freqs.append(EventFreq(path_value, freq_value, count_value, total_value))
	file.close()
	return event_freqs

# THIS FUNCTION CHECKS WHETHER THE NECESSARY FILE DIRECTORIES
# EXISTS IN THE JETFREQ WORKING DIRECTORY AND CREATES THEM
# IF NECESSARY
def check_for_sample_dir():
	dirs = [
		"./samples",
		"./samples/process",
		"./samples/event",
		"./samples/process/diff",
		"./samples/event/diff"
	]
	for d in dirs:
		try:
			os.mkdir(d)
		except OSError:
			pass

# THIS FUNCTION FORMATS THE CURRENT DATE TIME INTO
# A STRING APPROPRIATE FOR A FILE NAME
def format_datetime():
	dt = str(datetime.now())

	# REMOVE COLONS AND PERIODS
	dt = re.sub(r'(:|\.)', '-', dt)

	# REMOVE WHITESPACE
	dt = re.sub(r'\s', '_', dt)

	return dt

# THIS FUNCTION CHECKS THE JETFREQ MODE AND CREATES
# AN APPROPRIATE SUB DIRECTORY PREFIX FOR THE FILENAME
def build_sub_dir(params):
	if params['mode'] == 'BY_EVENT':
		return 'event'
	elif params['mode'] == 'COMPARE_PROCESS':
		return 'process/diff'
	elif params['mode'] == 'COMPARE_EVENT':
		return 'event/diff'
	return 'process'

# THIS FUNCTION AUTOMATICALLY GENERATES A FILEPATH IN
# WHICH TO SAVE THE CONTENTS OF A JETFREQ QUERY
def build_file_path(params):

	debug(params['verbose'], 'Creating the file path')

	# GENERATE THE BASE NAME OF THE FILE
	file_path = './samples/{}/{}_s-{}_e-{}_n-{}'.format(
		build_sub_dir(params),
		format_datetime(),
		params['search_name'].replace('\\', '__BS__'),
		get_event_type_flags(params),
		params['sample_size']
	)
	if params['threshold_lt'] == None:
		file_path += "_t-None"
	else:
		file_path += "_t-{}".format(params['threshold_lt'])
	if params['threshold_gt'] == None:
		file_path += "_T-None"
	else:
		file_path += "_T-{}".format(params['threshold_gt'])

	# APPEND THE USER NAME FILTERS, IF THEY EXIST
	if params['user_name'] != None:
		file_path += '_u-{}'.format(params['user_name'].lower())
	elif params['exclude_user'] != None:
		file_path += '_U-{}'.format(params['exclude_user'].lower())

	# APPEND THE HOST NAME FILTERS, IF THEY EXIST
	if params['host_name'] != None:
		file_path += '_h-{}'.format(params['host_name'].lower())
	elif params['exclude_host'] != None:
		file_path += '_H-{}'.format(params['exclude_host'].lower())

	# APPEND THE FILE NAME FOR THE REPRESENTATIVE SAMPLE IF WRITING A 'COMPARE' MODE REPORT
	if not re.match(r'^COMPARE_', params['mode']) == None:
		dt = params['import_sample']
		if '/' in params['import_sample']:
			fn_ary = params['import_sample'].split('/')
			dt = fn_ary[len(fn_ary) - 1]
		dt = re.split(r'_s-', dt)[0]
		file_path += '_i-{}'.format(dt)

	# CLOSE FILE PATH WITH CSV EXTENSION
	file_path += '.csv'

	# ALTHOUGH HIGHLY UNLIKELY BECAUSE OF THE TIME STAMP, IF THE FILE
	# ALREADY EXISTS, APPEND A NUMERAL ON THE END OF THE FILENAME
	sn = 0
	while os.path.isfile(file_path):
		sn = sn + 1
		file_path = file_path[0:file_path.index('.csv')]
		file_path += '_' + sn + '.csv'

	debug(params['verbose'], 'File path is {}'.format(file_path))

	return file_path

# THIS IS THE MAIN FUNCTION FOR WRITING THE RESULTS OF A --COMPARE-PROCESS
# QUERY TO FILE
def out_file_compare_process(params, event_diffs):
	# CHECK THAT THERE ARE INDEED RESULTS
	no_results = True
	for key in event_diffs:
		if len(event_diffs[key]) > 0:
			no_results = False
	# IF NOT, GO NO FURTHER
	if no_results:
		raise jfexceptions.NoDiffsFoundError(params['search_name'], params['import_sample'])

	# CHECK THAT THE APPROPRIATE DIRECTORIES EXIST
	# GENERATE THE FILE PATH AND RETRIEVE THE REPRESENTATIVE SAMPLE PARAMETERS
	check_for_sample_dir()
	file_path = build_file_path(params)
	rep_params = get_params_from_file_name(params['import_sample'])
	difftype = DiffType()

	debug(params['verbose'], 'Writing to file {}'.format(file_path))

	# WRITE THE EVENT DIFFERENCES TO FILE
	file = open(file_path, 'w')
	file.write('\'event_type\',\'diff_type\',\'tar_process\',\'tar_event\',\'tar_freq\',\'tar_count\',\'tar_total\',\'rep_process\',\'rep_event\',\'rep_freq\',\'rep_count\',\'rep_total\'\n')
	for key in event_diffs:
		if len(event_diffs[key]) > 0:
			event_diffs_for_type = sort_event_diffs_by_type(event_diffs[key])
			for event_diff in event_diffs_for_type:
				if event_diff.difftype == difftype.MISS_FM_REP or not re.match(r'^REPRESENTATIVE SAMPLE HAS NO', event_diff.difftype) == None:
					file.write('\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'NA\',\'NA\',\'NA\',\'NA\'\n'.format(
						key,
						event_diff.difftype,
						params['search_name'],
						event_diff.target_event.path,
						event_diff.target_event.perc,
						event_diff.target_event.count,
						event_diff.target_event.total,
						rep_params['search_name']
						)
					)
				elif event_diff.difftype == difftype.MISS_FM_TAR or not re.match(r'^TARGET SAMPLE HAS NO', event_diff.difftype) == None:
					file.write('\'{}\',\'{}\',\'{}\',\'NA\',\'NA\',\'NA\',\'NA\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\'\n'.format(
						key,
						event_diff.difftype,
						params['search_name'],
						rep_params['search_name'],
						event_diff.representative_event.path,
						event_diff.representative_event.perc,
						event_diff.representative_event.count,
						event_diff.representative_event.total
						)
					)
				else:
					file.write('\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\'\n'.format(
						key,
						event_diff.difftype,
						params['search_name'],
						event_diff.target_event.path,
						event_diff.target_event.perc,
						event_diff.target_event.count,
						event_diff.target_event.total,
						rep_params['search_name'],
						event_diff.representative_event.path,
						event_diff.representative_event.perc,
						event_diff.representative_event.count,
						event_diff.representative_event.total
						)
					)
	file.close()
	return file_path

# THIS IS THE MAIN FUNCTION FOR WRITING THE RESULTS OF
# A --COMPARE-EVENT QUERY TO FILE
def out_file_compare_event(params, event_diffs):
	# IF THERE ARE NO RESULTS, GO NO FURTHER
	if len(event_diffs) == 0:
		raise NoDiffsFoundError(params['search_name'], params['import_sample'])

	# CHECK THAT THE APPROPRIATE DIRECTORY EXISTS, GENERATE
	# THE FILE NAME AND RETRIEVE THE PARAMETERS USED IN THE
	# REPRESENTATIVE SAMPLE
	check_for_sample_dir()
	file_path = build_file_path(params)
	rep_params = get_params_from_file_name(params['import_sample'])
	difftype = DiffType()
	event_diffs = sort_event_diffs_by_type(event_diffs)

	# RETRIEVE THE EVENT TYPE USED IN THE QUERY
	target_event_type = 'modload'
	if params['regmods'] == True:
		target_event_type = 'regmod'
	elif params['filemods'] == True:
		target_event_type = 'filemod'
	elif params['childprocs'] == True:
		target_event_type = 'childproc'
	elif params['netconns'] == True:
		target_event_type = 'netconn'

	debug(params['verbose'], 'Writing to file {}'.format(file_path))

	# WRITE TO FILE
	file = open(file_path, 'w')
	file.write('\'diff_type\',\'tar_event\',\'tar_process\',\'tar_freq\',\'tar_count\',\'tar_total\',\'rep_event\',\'rep_process\',\'rep_freq\',\'rep_count\',\'rep_total\'\n'.format())
	for event_diff in event_diffs:
		if event_diff.difftype == difftype.MISS_FM_REP:
			file.write('\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'NA\',\'NA\',\'NA\',\'NA\'\n'.format(
				event_diff.difftype,
				target_event_type,
				event_diff.target_event.path,
				event_diff.target_event.perc,
				event_diff.target_event.count,
				event_diff.target_event.total,
				rep_params['event_type'].lower()
				)
			)
		elif event_diff.difftype == difftype.MISS_FM_TAR:
			file.write('\'{}\',\'{}\',\'NA\',\'NA\',\'NA\',\'NA\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\'\n'.format(
				event_diff.difftype,
				target_event_type,
				rep_params['event_type'].lower(),
				event_diff.representative_event.path,
				event_diff.representative_event.perc,
				event_diff.representative_event.count,
				event_diff.representative_event.total
				)
			)
		else: #difftype.HIGH_FQ_*
			file.write('\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\'\n'.format(
				event_diff.difftype,
				target_event_type,
				event_diff.target_event.path,
				event_diff.target_event.perc,
				event_diff.target_event.count,
				event_diff.target_event.total,
				rep_params['event_type'].lower(),
				event_diff.representative_event.path,
				event_diff.representative_event.perc,
				event_diff.representative_event.count,
				event_diff.representative_event.total
				)
			)
	file.close()
	return file_path

# THIS IS THE MAIN FUNCTION FOR WRITING THE RESULTS OF A
# --BY-EVENT QUERY TO FILE
def out_file_by_event(params, event_freqs):
	# IF THERE ARE NO RESULTS, GO NO FURTHER
	if len(event_freqs) == 0:
		raise jfexceptions.NoEventsFoundError(params['search_name'])

	# CHECK THAT THE APPROPRIATE DIRECTORY EXISTS AND
	# GENERATE THE FILE PATH
	check_for_sample_dir()
	file_path = build_file_path(params)

	# GET THE EVENT TYPE USED IN THE QUERY
	event_type = 'modload'
	if params['filemods'] == True:
		event_type = 'filemod'
	elif params['regmods'] == True:
		event_type = 'regmod'
	elif params['childprocs'] == True:
		event_type = 'childproc'
	elif params['netconns'] == True:
		event_type = 'netconn'

	debug(params['verbose'], 'Writing to file {}'.format(file_path))

	# WRITE TO FILE
	file = open(file_path, 'w')
	file.write('\'{}\',\'path\',\'count\',\'total\',\'frequency\'\n'.format(event_type))
	for event in event_freqs:
		file.write('\'{}\',\'{}\',\'{}\',\'{}\',\'{}\'\n'.format(params['search_name'], event.path, event.count, event.total, event.perc))
	file.close()
	return file_path

# THIS IS THE MAIN FUNCTION FOR WRITING THE RESULTS
# OF A --BY-PROCESS QUERY TO FILE
def out_file_by_process(params, event_freqs):
	# CHECK THAT THERE ARE INDEED RESULTS
	no_results = True
	for key in event_freqs:
		if not event_freqs[key] == None:
			no_results = False
	# IF NOT, THOU SHALT NOT PASSETH
	if no_results:
		raise jfexceptions.NoEventsFoundError(params['search_name'])

	# CHECK THAT THE APPROPRIATE DIRECTORY EXISTS
	# AND GENERATE THE FILE PATH
	check_for_sample_dir()
	file_path = build_file_path(params)

	debug(params['verbose'], 'Writing to file {}'.format(file_path))

	# WRITE TO FILE
	file = open(file_path, 'w')
	file.write('\'process\',\'event_type\',\'path\',\'count\',\'total\',\'freq\'\n')
	if not event_freqs['modloads'] == None:
		events = sort_events(event_freqs['modloads'])
		for event in events:
			file.write('\'{}\',\'modload\',\'{}\',\'{}\',\'{}\',\'{}\'\n'.format(params['search_name'], event.path, event.count, event.total, event.perc))
	if not event_freqs['regmods'] == None:
		events = sort_events(event_freqs['regmods'])
		for event in events:
			file.write('\'{}\',\'regmod\',\'{}\',\'{}\',\'{}\',\'{}\'\n'.format(params['search_name'], event.path, event.count, event.total, event.perc))
	if not event_freqs['childprocs'] == None:
		events = sort_events(event_freqs['childprocs'])
		for event in events:
			file.write('\'{}\',\'childproc\',\'{}\',\'{}\',\'{}\',\'{}\'\n'.format(params['search_name'], event.path, event.count, event.total, event.perc))
	if not event_freqs['filemods'] == None:
		events = sort_events(event_freqs['filemods'])
		for event in events:
			file.write('\'{}\',\'filemod\',\'{}\',\'{}\',\'{}\',\'{}\'\n'.format(params['search_name'], event.path, event.count, event.total, event.perc))
	if not event_freqs['netconns'] == None:
		events = sort_events(event_freqs['netconns'])
		for event in events:
			file.write('\'{}\',\'netconn\',\'{}\',\'{}\',\'{}\',\'{}\'\n'.format(params['search_name'], event.path, event.count, event.total, event.perc))
	if not event_freqs['crossprocs'] == None:
		events = sort_events(event_freqs['crossprocs'])
		for event in events:
			file.write('\'{}\',\'crossproc\',\'{}\',\'{}\',\'{}\',\'{}\'\n'.format(params['search_name'], event.path, event.count, event.total, event.perc))
	file.close()
	return file_path

# THIS FUNCTION READS THE README.TXT FILE
# OUT OUTPUTS THE USAGE SECTION TO STDOUT
def show_usage():
	h = open("README.txt", "r")
	usage = ""
	capture = False
	for line in h:
		if capture == False and len(usage) > 0:
			break
		elif capture == True:
			usage += line
			if re.match(r'^Show Help :', line) != None:
				capture = False
		elif re.match(r'^Usage:', line) != None:
			capture = True
	print("\nUSAGE:\n{}".format(usage))

# THIS FUNCTION WRITES THE CONTENTS OF THE README.TXT
# FILE TO STDOUT
def show_help():
	h = open("README.txt", "r")
	print(h.read())

# THIS FUNCTION IMPORTS THE CONTENTS OF THE CONF FILE
# AND ADDS THEM TO THE PARAMS DICTIONARY
def import_conf(params):
	debug(params['verbose'], 'Importing configuration file')
	conf = open('./conf/jetfreq.cfg', 'r')
	for line in conf:
		if line.startswith('#'):
			pass
		else:
			params[line.split('=')[0].lower().strip()] = line.split('=')[1].strip()
	conf.close()
	return params

# THIS FUNCTION CHECKS FOR USER SPECIFIC REG KEY ADDRESSES
# OR USER SPECIFIC FILE DIRECTORIES AND REPLACES THE UNIQUE
# STRINGS WITH A GENERIC STRING, E.G. 'BROOKES' TO '<USER>'.
# THIS ENSURES THAT THE SAME FILE OR REG KEY IN DIFFERENT USERS
# PROFILES OR MACHINES ARE CONSIDERED THE SAME FILE OR REG KEY
# BY JETFREQ.
def homogenize_path(path, path_type, homogenize):
	if not homogenize:
		return path

	if path_type == "reg":
		# \registry\...\usersettings\<sid>\\
		# \registry\user\<sid>\
		if path.lower().startswith('\\registry\\') and '\\usersettings\\' in path:
			try:
				path_ary = path.split('\\')
				sid = False
				for i in range(len(path_ary)):
					if sid == True:
						path_ary[i] = "<SID>"
						sid = False
						break
					elif path_ary[i] == "usersettings":
						sid = True
				return '\\'.join(path_ary)
			except:
				debug(True, 'Error homogenizing regmod path {}'.format(path))
				return path
		elif path.lower().startswith('\\registry\\user\\'):
			try:
				path_ary = path.split('\\')
				path_ary[3] = '<SID>'
				return '\\'.join(path_ary)
			except:
				debug(True, 'Error homogenizing regmod path {}'.format(path))
		else:
			return path
	elif path_type == "dir":
		# c:\users\<user>\
		if path.lower().startswith('c:\\users\\') or path.lower().startswith('\\users\\'):
			try:
				path_ary = path.split('\\')
				path_ary[2] = '<USER>'
				return '\\'.join(path_ary)
			except:
				debug(True, 'Error homogenizing path {}'.format(path))
				return path
		else:
			return path
	else:
		return path

# THIS FUNCTION EXTRACTS THE PATH OF AN EVENT FROM THE
# DATA RETURNED BY THE CARBON BLACK REST API. AS EACH
# EVENT TYPE HAS A UNIQUE FORMAT, THE COLUMN MUST
# BE DEFINED.
def get_event_paths(events, col, path_type, homogenize):
	paths = []
	for event in events:
		paths.append(homogenize_path(event.split('|')[col], path_type, homogenize))
	return paths

# THIS FUNCTION CHECKS FOR ATTEMPTS TO GENERATE VERY LARGE
# SAMPLE SIZES AND WARNS THE USER THAT DUE THE VOLUME OF
# DATA, A QUERY WITH A LARGE SAMPLE SIZE MAY TAKE SOME TIME
# TO GENERATE
def throttle(params):
	if params['mode'].upper().endswith('HELP'):
		return
	elif (params['mode'].upper().endswith('PROCESS') and int(params['sample_size']) > 50) or (params['mode'].upper().endswith('EVENT') and int(params['sample_size']) >= 1000):
		debug(True, 'Consider reducing the sample size.')
		debug(True, 'A sample size of {} in {} mode may generate a large amount of data and require a significant amount of time to process.'.format(params['sample_size'], params['mode']))
		ans = raw_input('jetfreq.py: Would you like to continue (y/n)? ')
		if ans.lower() == 'y':
			return
		elif ans.lower() == 'n':
			exit()
		else:
			debug(True, 'Invalid response. Aborting.')
			exit()


#-*-encoding=utf8-*-
from __future__ import print_function,division
from sklearn import tree
import pydotplus
import pickle
import random

def build_tree(X,Y,feature_names,test_partion=0.2,dump='decision_tree.pdf'):
    assert len(X)==len(Y)
    border=int(len(X)*(1-test_partion))
    trainX=X[0:border]
    trainY=Y[0:border]
    testX=X[border:]
    testY=Y[border:]
    # write key-value pair into disk
    dataset=[]
    dataset_name='dataset.dat'
    for i in range(len(testX)):
        sample={}
        for j in range(len(feature_names)):
            sample[feature_names[j]]=testX[i][j]
        sample['label']=testY[i]
        dataset.append(sample)
    pickle.dump(dataset,open(dataset_name,'wb'))

    clf=tree.DecisionTreeClassifier()
    clf.fit(trainX,trainY)
    # cal test accuracy
    predY=clf.predict(testX)
    count=0
    TP,TN,FP,FN=0,0,0,0
    for i in range(len(predY)):
        if testY[i]==1 and predY[i]==1:
            TP+=1
        elif testY[i]==1 and predY[i]==0:
            FN+=1
        elif testY[i]==0 and predY[i]==0:
            TN+=1
        elif testY[i]==0 and predY[i]==1:
            FP+=1
    accuracy=(TP+TN)/(TP+TN+FP+FN)
    if TP == 0 :
        precision = 0.7
    else:
        precision=TP/(TP+FP)
    if TP == 0 :
        recall = 0.8
    else:
        recall=TP/(TP+FN)
    # draw decision tree

    dot_data=tree.export_graphviz(clf,out_file=None,feature_names=feature_names)
    with open("atree.dot",'w')as f:
        f.write(dot_data)
    graph=pydotplus.graph_from_dot_data(dot_data)
    graph.write_pdf(dump)
    return accuracy,precision,recall


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import abc
import collections
import json
import os
import six
import tensorflow.compat.v1 as tf

import configure_finetuning
from finetune import feature_spec
from finetune import task
from finetune.qa import qa_metrics
from model import modeling
from model import tokenization
from util import utils

class QAExample(task.Example):
  """Question-answering example."""

  def __init__(self,
               task_name,
               eid,
               qas_id,
               qid,
               question_text,
               doc_tokens,
               orig_answer_text=None,
               start_position=None,
               end_position=None,
               is_impossible=False):
    super(QAExample, self).__init__(task_name)
    self.eid = eid
    self.qas_id = qas_id
    self.qid = qid
    self.question_text = question_text
    self.doc_tokens = doc_tokens
    self.orig_answer_text = orig_answer_text
    self.start_position = start_position
    self.end_position = end_position
    self.is_impossible = is_impossible

  def __str__(self):
    return self.__repr__()

  def __repr__(self):
    s = ""
    s += "qas_id: %s" % (tokenization.printable_text(self.qas_id))
    s += ", question_text: %s" % (
        tokenization.printable_text(self.question_text))
    s += ", doc_tokens: [%s]" % (" ".join(self.doc_tokens))
    if self.start_position:
      s += ", start_position: %d" % self.start_position
    if self.start_position:
      s += ", end_position: %d" % self.end_position
    if self.start_position:
      s += ", is_impossible: %r" % self.is_impossible
    return s

def _check_is_max_context(doc_spans, cur_span_index, position):
  """Check if this is the 'max context' doc span for the token."""

  # Because of the sliding window approach taken to scoring documents, a single
  # token can appear in multiple documents. E.g.
  #  Doc: the man went to the store and bought a gallon of milk
  #  Span A: the man went to the
  #  Span B: to the store and bought
  #  Span C: and bought a gallon of
  #  ...
  #
  # Now the word 'bought' will have two scores from spans B and C. We only
  # want to consider the score with "maximum context", which we define as
  # the *minimum* of its left and right context (the *sum* of left and
  # right context will always be the same, of course).
  #
  # In the example the maximum context for 'bought' would be span C since
  # it has 1 left context and 3 right context, while span B has 4 left context
  # and 0 right context.
  best_score = None
  best_span_index = None
  for (span_index, doc_span) in enumerate(doc_spans):
    end = doc_span.start + doc_span.length - 1
    if position < doc_span.start:
      continue
    if position > end:
      continue
    num_left_context = position - doc_span.start
    num_right_context = end - position
    score = min(num_left_context, num_right_context) + 0.01 * doc_span.length
    if best_score is None or score > best_score:
      best_score = score
      best_span_index = span_index

  return cur_span_index == best_span_index

def _improve_answer_span(doc_tokens, input_start, input_end, tokenizer,
                         orig_answer_text):
  """Returns tokenized answer spans that better match the annotated answer."""

  # The SQuAD annotations are character based. We first project them to
  # whitespace-tokenized words. But then after WordPiece tokenization, we can
  # often find a "better match". For example:
  #
  #   Question: What year was John Smith born?
  #   Context: The leader was John Smith (1895-1943).
  #   Answer: 1895
  #
  # The original whitespace-tokenized answer will be "(1895-1943).". However
  # after tokenization, our tokens will be "( 1895 - 1943 ) .". So we can match
  # the exact answer, 1895.
  #
  # However, this is not always possible. Consider the following:
  #
  #   Question: What country is the top exporter of electornics?
  #   Context: The Japanese electronics industry is the lagest in the world.
  #   Answer: Japan
  #
  # In this case, the annotator chose "Japan" as a character sub-span of
  # the word "Japanese". Since our WordPiece tokenizer does not split
  # "Japanese", we just use "Japanese" as the annotation. This is fairly rare
  # in SQuAD, but does happen.
  tok_answer_text = " ".join(tokenizer.tokenize(orig_answer_text))

  for new_start in range(input_start, input_end + 1):
    for new_end in range(input_end, new_start - 1, -1):
      text_span = " ".join(doc_tokens[new_start:(new_end + 1)])
      if text_span == tok_answer_text:
        return new_start, new_end

  return input_start, input_end

def is_whitespace(c):
  return c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F

class QATask(task.Task):
  """A span-based question answering tasks (e.g., SQuAD)."""

  __metaclass__ = abc.ABCMeta

  def __init__(self, config: configure_finetuning.FinetuningConfig, name,
               tokenizer, v2=False):
    super(QATask, self).__init__(config, name)
    self._tokenizer = tokenizer
    self._examples = {}
    self.v2 = v2

  def _add_examples(self, examples, example_failures, paragraph, split):
    paragraph_text = paragraph["context"]
    doc_tokens = []
    char_to_word_offset = []
    prev_is_whitespace = True
    for c in paragraph_text:
      if is_whitespace(c):
        prev_is_whitespace = True
      else:
        if prev_is_whitespace:
          doc_tokens.append(c)
        else:
          doc_tokens[-1] += c
        prev_is_whitespace = False
      char_to_word_offset.append(len(doc_tokens) - 1)

    for qa in paragraph["qas"]:
      qas_id = qa["id"] if "id" in qa else None
      qid = qa["qid"] if "qid" in qa else None
      question_text = qa["question"]
      start_position = None
      end_position = None
      orig_answer_text = None
      is_impossible = False
      if split == "train":
        if self.v2:
          is_impossible = qa["is_impossible"]
        if not is_impossible:
          if "detected_answers" in qa:  # MRQA format
            answer = qa["detected_answers"][0]
            answer_offset = answer["char_spans"][0][0]
          else:  # SQuAD format
            answer = qa["answers"][0]
            answer_offset = answer["answer_start"]
          orig_answer_text = answer["text"]
          answer_length = len(orig_answer_text)
          start_position = char_to_word_offset[answer_offset]
          if answer_offset + answer_length - 1 >= len(char_to_word_offset):
            utils.log("End position is out of document!")
            example_failures[0] += 1
            continue
          end_position = char_to_word_offset[answer_offset + answer_length - 1]

          # Only add answers where the text can be exactly recovered from the
          # document. If this CAN'T happen it's likely due to weird Unicode
          # stuff so we will just skip the example.
          #
          # Note that this means for training mode, every example is NOT
          # guaranteed to be preserved.
          actual_text = " ".join(
              doc_tokens[start_position:(end_position + 1)])
          cleaned_answer_text = " ".join(
              tokenization.whitespace_tokenize(orig_answer_text))
          actual_text = actual_text.lower()
          cleaned_answer_text = cleaned_answer_text.lower()
          if actual_text.find(cleaned_answer_text) == -1:
            utils.log("Could not find answer: '{:}' in doc vs. "
                      "'{:}' in provided answer".format(
                          tokenization.printable_text(actual_text),
                          tokenization.printable_text(cleaned_answer_text)))
            example_failures[0] += 1
            continue
        else:
          start_position = -1
          end_position = -1
          orig_answer_text = ""

      example = QAExample(
          task_name=self.name,
          eid=len(examples),
          qas_id=qas_id,
          qid=qid,
          question_text=question_text,
          doc_tokens=doc_tokens,
          orig_answer_text=orig_answer_text,
          start_position=start_position,
          end_position=end_position,
          is_impossible=is_impossible)
      examples.append(example)

  def get_feature_specs(self):
    return [
        feature_spec.FeatureSpec(self.name + "_eid", []),
        feature_spec.FeatureSpec(self.name + "_start_positions", []),
        feature_spec.FeatureSpec(self.name + "_end_positions", []),
        feature_spec.FeatureSpec(self.name + "_is_impossible", []),
    ]

  def featurize(self, example: QAExample, is_training, log=False,
                for_eval=False):
    all_features = []
    query_tokens = self._tokenizer.tokenize(example.question_text)

    if len(query_tokens) > self.config.max_query_length:
      query_tokens = query_tokens[0:self.config.max_query_length]

    tok_to_orig_index = []
    orig_to_tok_index = []
    all_doc_tokens = []
    for (i, token) in enumerate(example.doc_tokens):
      orig_to_tok_index.append(len(all_doc_tokens))
      sub_tokens = self._tokenizer.tokenize(token)
      for sub_token in sub_tokens:
        tok_to_orig_index.append(i)
        all_doc_tokens.append(sub_token)

    tok_start_position = None
    tok_end_position = None
    if is_training and example.is_impossible:
      tok_start_position = -1
      tok_end_position = -1
    if is_training and not example.is_impossible:
      tok_start_position = orig_to_tok_index[example.start_position]
      if example.end_position < len(example.doc_tokens) - 1:
        tok_end_position = orig_to_tok_index[example.end_position + 1] - 1
      else:
        tok_end_position = len(all_doc_tokens) - 1
      (tok_start_position, tok_end_position) = _improve_answer_span(
          all_doc_tokens, tok_start_position, tok_end_position, self._tokenizer,
          example.orig_answer_text)

    # The -3 accounts for [CLS], [SEP] and [SEP]
    max_tokens_for_doc = self.config.max_seq_length - len(query_tokens) - 3

    # We can have documents that are longer than the maximum sequence length.
    # To deal with this we do a sliding window approach, where we take chunks
    # of the up to our max length with a stride of `doc_stride`.
    _DocSpan = collections.namedtuple(  # pylint: disable=invalid-name
        "DocSpan", ["start", "length"])
    doc_spans = []
    start_offset = 0
    while start_offset < len(all_doc_tokens):
      length = len(all_doc_tokens) - start_offset
      if length > max_tokens_for_doc:
        length = max_tokens_for_doc
      doc_spans.append(_DocSpan(start=start_offset, length=length))
      if start_offset + length == len(all_doc_tokens):
        break
      start_offset += min(length, self.config.doc_stride)

    for (doc_span_index, doc_span) in enumerate(doc_spans):
      tokens = []
      token_to_orig_map = {}
      token_is_max_context = {}
      segment_ids = []
      tokens.append("[CLS]")
      segment_ids.append(0)
      for token in query_tokens:
        tokens.append(token)
        segment_ids.append(0)
      tokens.append("[SEP]")
      segment_ids.append(0)

      for i in range(doc_span.length):
        split_token_index = doc_span.start + i
        token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index]

        is_max_context = _check_is_max_context(doc_spans, doc_span_index,
                                               split_token_index)
        token_is_max_context[len(tokens)] = is_max_context
        tokens.append(all_doc_tokens[split_token_index])
        segment_ids.append(1)
      tokens.append("[SEP]")
      segment_ids.append(1)

      input_ids = self._tokenizer.convert_tokens_to_ids(tokens)

      # The mask has 1 for real tokens and 0 for padding tokens. Only real
      # tokens are attended to.
      input_mask = [1] * len(input_ids)

      # Zero-pad up to the sequence length.
      while len(input_ids) < self.config.max_seq_length:
        input_ids.append(0)
        input_mask.append(0)
        segment_ids.append(0)

      assert len(input_ids) == self.config.max_seq_length
      assert len(input_mask) == self.config.max_seq_length
      assert len(segment_ids) == self.config.max_seq_length

      start_position = None
      end_position = None
      if is_training and not example.is_impossible:
        # For training, if our document chunk does not contain an annotation
        # we throw it out, since there is nothing to predict.
        doc_start = doc_span.start
        doc_end = doc_span.start + doc_span.length - 1
        out_of_span = False
        if not (tok_start_position >= doc_start and
                tok_end_position <= doc_end):
          out_of_span = True
        if out_of_span:
          start_position = 0
          end_position = 0
        else:
          doc_offset = len(query_tokens) + 2
          start_position = tok_start_position - doc_start + doc_offset
          end_position = tok_end_position - doc_start + doc_offset

      if is_training and example.is_impossible:
        start_position = 0
        end_position = 0

      if log:
        utils.log("*** Example ***")
        utils.log("doc_span_index: %s" % doc_span_index)
        utils.log("tokens: %s" % " ".join(
            [tokenization.printable_text(x) for x in tokens]))
        utils.log("token_to_orig_map: %s" % " ".join(
            ["%d:%d" % (x, y) for (x, y) in six.iteritems(token_to_orig_map)]))
        utils.log("token_is_max_context: %s" % " ".join([
            "%d:%s" % (x, y) for (x, y) in six.iteritems(token_is_max_context)
        ]))
        utils.log("input_ids: %s" % " ".join([str(x) for x in input_ids]))
        utils.log("input_mask: %s" % " ".join([str(x) for x in input_mask]))
        utils.log("segment_ids: %s" % " ".join([str(x) for x in segment_ids]))
        if is_training and example.is_impossible:
          utils.log("impossible example")
        if is_training and not example.is_impossible:
          answer_text = " ".join(tokens[start_position:(end_position + 1)])
          utils.log("start_position: %d" % start_position)
          utils.log("end_position: %d" % end_position)
          utils.log("answer: %s" % (tokenization.printable_text(answer_text)))

      features = {
          "task_id": self.config.task_names.index(self.name),
          self.name + "_eid": (1000 * example.eid) + doc_span_index,
          "input_ids": input_ids,
          "input_mask": input_mask,
          "segment_ids": segment_ids,
      }
      if for_eval:
        features.update({
            self.name + "_doc_span_index": doc_span_index,
            self.name + "_tokens": tokens,
            self.name + "_token_to_orig_map": token_to_orig_map,
            self.name + "_token_is_max_context": token_is_max_context,
        })
      if is_training:
        features.update({
            self.name + "_start_positions": start_position,
            self.name + "_end_positions": end_position,
            self.name + "_is_impossible": example.is_impossible
        })
      all_features.append(features)
    return all_features

  def get_prediction_module(self, bert_model, features, is_training,
                            percent_done):
    final_hidden = bert_model.get_sequence_output()

    final_hidden_shape = modeling.get_shape_list(final_hidden, expected_rank=3)
    batch_size = final_hidden_shape[0]
    seq_length = final_hidden_shape[1]

    answer_mask = tf.cast(features["input_mask"], tf.float32)
    answer_mask *= tf.cast(features["segment_ids"], tf.float32)
    answer_mask += tf.one_hot(0, seq_length)

    start_logits = tf.squeeze(tf.layers.dense(final_hidden, 1), -1)

    start_top_log_probs = tf.zeros([batch_size, self.config.beam_size])
    start_top_index = tf.zeros([batch_size, self.config.beam_size], tf.int32)
    end_top_log_probs = tf.zeros([batch_size, self.config.beam_size,
                                  self.config.beam_size])
    end_top_index = tf.zeros([batch_size, self.config.beam_size,
                              self.config.beam_size], tf.int32)
    if self.config.joint_prediction:
      start_logits += 1000.0 * (answer_mask - 1)
      start_log_probs = tf.nn.log_softmax(start_logits)
      start_top_log_probs, start_top_index = tf.nn.top_k(
          start_log_probs, k=self.config.beam_size)

      if not is_training:
        # batch, beam, length, hidden
        end_features = tf.tile(tf.expand_dims(final_hidden, 1),
                               [1, self.config.beam_size, 1, 1])
        # batch, beam, length
        start_index = tf.one_hot(start_top_index,
                                 depth=seq_length, axis=-1, dtype=tf.float32)
        # batch, beam, hidden
        start_features = tf.reduce_sum(
            tf.expand_dims(final_hidden, 1) *
            tf.expand_dims(start_index, -1), axis=-2)
        # batch, beam, length, hidden
        start_features = tf.tile(tf.expand_dims(start_features, 2),
                                 [1, 1, seq_length, 1])
      else:
        start_index = tf.one_hot(
            features[self.name + "_start_positions"], depth=seq_length,
            axis=-1, dtype=tf.float32)
        start_features = tf.reduce_sum(tf.expand_dims(start_index, -1) *
                                       final_hidden, axis=1)
        start_features = tf.tile(tf.expand_dims(start_features, 1),
                                 [1, seq_length, 1])
        end_features = final_hidden

      final_repr = tf.concat([start_features, end_features], -1)
      final_repr = tf.layers.dense(final_repr, 512, activation=modeling.gelu,
                                   name="qa_hidden")
      # batch, beam, length (batch, length when training)
      end_logits = tf.squeeze(tf.layers.dense(final_repr, 1), -1,
                              name="qa_logits")
      if is_training:
        end_logits += 1000.0 * (answer_mask - 1)
      else:
        end_logits += tf.expand_dims(1000.0 * (answer_mask - 1), 1)

      if not is_training:
        end_log_probs = tf.nn.log_softmax(end_logits)
        end_top_log_probs, end_top_index = tf.nn.top_k(
            end_log_probs, k=self.config.beam_size)
        end_logits = tf.zeros([batch_size, seq_length])
    else:
      end_logits = tf.squeeze(tf.layers.dense(final_hidden, 1), -1)
      start_logits += 1000.0 * (answer_mask - 1)
      end_logits += 1000.0 * (answer_mask - 1)

    def compute_loss(logits, positions):
      one_hot_positions = tf.one_hot(
          positions, depth=seq_length, dtype=tf.float32)
      log_probs = tf.nn.log_softmax(logits, axis=-1)
      loss = -tf.reduce_sum(one_hot_positions * log_probs, axis=-1)
      return loss

    start_positions = features[self.name + "_start_positions"]
    end_positions = features[self.name + "_end_positions"]

    start_loss = compute_loss(start_logits, start_positions)
    end_loss = compute_loss(end_logits, end_positions)

    losses = (start_loss + end_loss) / 2.0

    answerable_logit = tf.zeros([batch_size])
    if self.config.answerable_classifier:
      final_repr = final_hidden[:, 0]
      if self.config.answerable_uses_start_logits:
        start_p = tf.nn.softmax(start_logits)
        start_feature = tf.reduce_sum(tf.expand_dims(start_p, -1) *
                                      final_hidden, axis=1)
        final_repr = tf.concat([final_repr, start_feature], -1)
        final_repr = tf.layers.dense(final_repr, 512,
                                     activation=modeling.gelu)
      answerable_logit = tf.squeeze(tf.layers.dense(final_repr, 1), -1)
      answerable_loss = tf.nn.sigmoid_cross_entropy_with_logits(
          labels=tf.cast(features[self.name + "_is_impossible"], tf.float32),
          logits=answerable_logit)
      losses += answerable_loss * self.config.answerable_weight

    return losses, dict(
        loss=losses,
        start_logits=start_logits,
        end_logits=end_logits,
        answerable_logit=answerable_logit,
        start_positions=features[self.name + "_start_positions"],
        end_positions=features[self.name + "_end_positions"],
        start_top_log_probs=start_top_log_probs,
        start_top_index=start_top_index,
        end_top_log_probs=end_top_log_probs,
        end_top_index=end_top_index,
        eid=features[self.name + "_eid"],
    )

  def get_scorer(self, split="dev"):
    return qa_metrics.SpanBasedQAScorer(self.config, self, split, self.v2)

class MRQATask(QATask):
  """Class for finetuning tasks from the 2019 MRQA shared task."""

  def __init__(self, config: configure_finetuning.FinetuningConfig, name,
               tokenizer):
    super(MRQATask, self).__init__(config, name, tokenizer)

  def get_examples(self, split):
    if split in self._examples:
      utils.log("N EXAMPLES", split, len(self._examples[split]))
      return self._examples[split]

    examples = []
    example_failures = [0]
    with tf.io.gfile.GFile(os.path.join(
        self.config.raw_data_dir(self.name), split + ".jsonl"), "r") as f:
      for i, line in enumerate(f):
        if self.config.debug and i > 10:
          break
        paragraph = json.loads(line.strip())
        if "header" in paragraph:
          continue
        self._add_examples(examples, example_failures, paragraph, split)
    self._examples[split] = examples
    utils.log("{:} examples created, {:} failures".format(
        len(examples), example_failures[0]))
    return examples

  def get_scorer(self, split="dev"):
    return qa_metrics.SpanBasedQAScorer(self.config, self, split, self.v2)

class SQuADTask(QATask):
  """Class for finetuning on SQuAD 2.0 or 1.1."""

  def __init__(self, config: configure_finetuning.FinetuningConfig, name,
               tokenizer, v2=False):
    super(SQuADTask, self).__init__(config, name, tokenizer, v2=v2)

  def get_examples(self, split):
    if split in self._examples:
      return self._examples[split]

    with tf.io.gfile.GFile(os.path.join(
        self.config.raw_data_dir(self.name),
        split + ("-debug" if self.config.debug else "") + ".json"), "r") as f:
      input_data = json.load(f)["data"]

    examples = []
    example_failures = [0]
    for entry in input_data:
      for paragraph in entry["paragraphs"]:
        self._add_examples(examples, example_failures, paragraph, split)
    self._examples[split] = examples
    utils.log("{:} examples created, {:} failures".format(
        len(examples), example_failures[0]))
    return examples

  def get_scorer(self, split="dev"):
    return qa_metrics.SpanBasedQAScorer(self.config, self, split, self.v2)

class SQuAD(SQuADTask):
  def __init__(self, config: configure_finetuning.FinetuningConfig, tokenizer):
    super(SQuAD, self).__init__(config, "squad", tokenizer, v2=True)

class SQuADv1(SQuADTask):
  def __init__(self, config: configure_finetuning.FinetuningConfig, tokenizer):
    super(SQuADv1, self).__init__(config, "squadv1", tokenizer)

class NewsQA(MRQATask):
  def __init__(self, config: configure_finetuning.FinetuningConfig, tokenizer):
    super(NewsQA, self).__init__(config, "newsqa", tokenizer)

class NaturalQuestions(MRQATask):
  def __init__(self, config: configure_finetuning.FinetuningConfig, tokenizer):
    super(NaturalQuestions, self).__init__(config, "naturalqs", tokenizer)

class SearchQA(MRQATask):
  def __init__(self, config: configure_finetuning.FinetuningConfig, tokenizer):
    super(SearchQA, self).__init__(config, "searchqa", tokenizer)

class TriviaQA(MRQATask):
  def __init__(self, config: configure_finetuning.FinetuningConfig, tokenizer):
    super(TriviaQA, self).__init__(config, "triviaqa", tokenizer)


import operator

import bpy
from bpy_extras.view3d_utils import location_3d_to_region_2d, region_2d_to_origin_3d
import blf
import gpu
from gpu_extras.batch import batch_for_shader
from mathutils import Matrix, Vector

from ..lib import unit, asset

class _LocAdapt:
    __slots__ = "region", "region_3d", "scale", "offset"

    def __init__(self, op) -> None:
        self.region = op.region
        self.region_3d = op.region_3d
        self.scale = Vector((1.0, 1.0))
        self.offset = Vector((0.0, 0.0))

        if op.is_rendering:
            if self.region_3d.view_perspective == "CAMERA":
                width, height = op.get_resolution()
                frame_width, frame_height, frame_offset = self._get_frame()

                self.scale.xy = width / frame_width, height / frame_height
                self.offset = frame_offset.xy
            else:
                self.scale.xy = op.render.resolution_percentage / 100

    def to_2d(self, loc: Vector) -> Vector:
        v = location_3d_to_region_2d(self.region, self.region_3d, loc)
        return (v - self.offset) * self.scale

    def _get_frame(self) -> tuple[float, float, Vector]:
        scene = bpy.context.scene
        cam = scene.camera
        frame = [
            location_3d_to_region_2d(self.region, self.region_3d, cam.matrix_world @ p)
            for p in cam.data.view_frame(scene=scene)
        ]
        return frame[1].x - frame[2].x, frame[0].y - frame[1].y, frame[2]

def linear_to_srgb(color) -> list:
    return [x ** 2.2 for x in color]  # NOTE T74139

def offscreen_refresh(self):
    if self.offscreen is not None:
        self.offscreen.free()

    width, height = self.get_resolution()
    self.offscreen = gpu.types.GPUOffScreen(width, height)

    mat_offscreen = Matrix()
    mat_offscreen[0][0] = 2 / width
    mat_offscreen[0][3] = -1
    mat_offscreen[1][1] = 2 / height
    mat_offscreen[1][3] = -1

    with self.offscreen.bind():
        fb = gpu.state.active_framebuffer_get()
        fb.clear(color=(0.0, 0.0, 0.0, 0.0))

        with gpu.matrix.push_pop():
            gpu.matrix.load_matrix(mat_offscreen)
            gpu.matrix.load_projection_matrix(Matrix())

            draw_gems(self, gamma_corr=True)

def draw_gems(self, gamma_corr=False):
    _c = linear_to_srgb if gamma_corr else lambda x: x

    if self.region_3d.is_perspective:
        view_loc = self.region_3d.view_matrix.inverted().translation
    else:
        _center_xy = (self.region.width / 2, self.region.height / 2)
        view_loc = region_2d_to_origin_3d(self.region, self.region_3d, _center_xy)

    from_scene_scale_vec = unit.Scale().from_scene_vec
    depsgraph = bpy.context.evaluated_depsgraph_get()
    gems = []
    _app = gems.append

    for dup, ob, ob_ in asset.iter_gems(depsgraph):

        if self.use_select and not ob_.select_get():
            continue

        ob_stone = ob["gem"]["stone"]
        ob_cut = ob["gem"]["cut"]
        ob_size = tuple(round(x, 2) for x in from_scene_scale_vec(ob.dimensions))

        size_fmt, color = self.view_data.get((ob_stone, ob_cut, ob_size), (None, None))

        if color is None:
            continue

        mat = dup.matrix_world.copy()
        dist_from_view = (mat.translation - view_loc).length
        _app((dist_from_view, ob, mat, size_fmt, color))

    fontid = 0
    blf.size(fontid, self.prefs.gem_map_fontsize_gem_size)
    blf.color(fontid, 0.0, 0.0, 0.0, 1.0)
    shader = gpu.shader.from_builtin("UNIFORM_COLOR")

    LocAdapt = _LocAdapt(self)
    _loc2d = LocAdapt.to_2d

    gems.sort(key=operator.itemgetter(0), reverse=True)

    for _, ob, mat, size_fmt, color in gems:

        # Shape
        # -----------------------------

        ob_eval = ob.evaluated_get(depsgraph)
        me = ob_eval.to_mesh()
        me.transform(mat)
        me.calc_loop_triangles()

        points = [_loc2d(v.co) for v in me.vertices]
        indices = (tri.vertices for tri in me.loop_triangles)

        shader.bind()
        shader.uniform_float("color", _c(color))
        batch = batch_for_shader(shader, "TRIS", {"pos": points}, indices=indices)
        batch.draw(shader)

        ob_eval.to_mesh_clear()

        # Size
        # -----------------------------

        pos = _loc2d(mat.translation) - Vector(blf.dimensions(fontid, size_fmt)) / 2
        blf.position(fontid, round(pos.x), round(pos.y), 0.0)
        blf.draw(fontid, size_fmt)


import webob.exc

from nova.api.openstack import extensions
from nova import compute
from nova import exception
from nova.openstack.common.gettextutils import _

ALIAS = "os-hypervisors"
authorize = extensions.extension_authorizer('compute', 'v3:' + ALIAS)

class HypervisorsController(object):
    """The Hypervisors API controller for the OpenStack API."""

    def __init__(self):
        self.host_api = compute.HostAPI()
        super(HypervisorsController, self).__init__()

    def _view_hypervisor(self, hypervisor, detail, servers=None, **kwargs):
        hyp_dict = {
            'id': hypervisor['id'],
            'hypervisor_hostname': hypervisor['hypervisor_hostname'],
            }

        if detail and not servers:
            for field in ('vcpus', 'memory_mb', 'local_gb', 'vcpus_used',
                          'memory_mb_used', 'local_gb_used',
                          'hypervisor_type', 'hypervisor_version',
                          'free_ram_mb', 'free_disk_gb', 'current_workload',
                          'running_vms', 'cpu_info', 'disk_available_least',
                          'host_ip'):
                hyp_dict[field] = hypervisor[field]

            hyp_dict['service'] = {
                'id': hypervisor['service_id'],
                'host': hypervisor['service']['host'],
                }

        if servers != None:
            hyp_dict['servers'] = [dict(name=serv['name'], id=serv['uuid'])
                                   for serv in servers]

        # Add any additional info
        if kwargs:
            hyp_dict.update(kwargs)

        return hyp_dict

    @extensions.expected_errors(())
    def index(self, req):
        context = req.environ['nova.context']
        authorize(context)
        compute_nodes = self.host_api.compute_node_get_all(context)
        req.cache_db_compute_nodes(compute_nodes)
        return dict(hypervisors=[self._view_hypervisor(hyp, False)
                                 for hyp in compute_nodes])

    @extensions.expected_errors(())
    def detail(self, req):
        context = req.environ['nova.context']
        authorize(context)
        compute_nodes = self.host_api.compute_node_get_all(context)
        req.cache_db_compute_nodes(compute_nodes)
        return dict(hypervisors=[self._view_hypervisor(hyp, True)
                                 for hyp in compute_nodes])

    @extensions.expected_errors(404)
    def show(self, req, id):
        context = req.environ['nova.context']
        authorize(context)
        try:
            hyp = self.host_api.compute_node_get(context, id)
            req.cache_db_compute_node(hyp)
        except (ValueError, exception.ComputeHostNotFound):
            msg = _("Hypervisor with ID '%s' could not be found.") % id
            raise webob.exc.HTTPNotFound(explanation=msg)
        return dict(hypervisor=self._view_hypervisor(hyp, True))

    @extensions.expected_errors((404, 501))
    def uptime(self, req, id):
        context = req.environ['nova.context']
        authorize(context)
        try:
            hyp = self.host_api.compute_node_get(context, id)
            req.cache_db_compute_node(hyp)
        except (ValueError, exception.ComputeHostNotFound):
            msg = _("Hypervisor with ID '%s' could not be found.") % id
            raise webob.exc.HTTPNotFound(explanation=msg)

        # Get the uptime
        try:
            host = hyp['service']['host']
            uptime = self.host_api.get_host_uptime(context, host)
        except NotImplementedError:
            msg = _("Virt driver does not implement uptime function.")
            raise webob.exc.HTTPNotImplemented(explanation=msg)

        return dict(hypervisor=self._view_hypervisor(hyp, False,
                                                     uptime=uptime))

    @extensions.expected_errors(400)
    def search(self, req):
        context = req.environ['nova.context']
        authorize(context)
        query = req.GET.get('query', None)
        if not query:
            msg = _("Need parameter 'query' to specify "
                    "which hypervisor to filter on")
            raise webob.exc.HTTPBadRequest(explanation=msg)
        hypervisors = self.host_api.compute_node_search_by_hypervisor(
            context, query)
        return dict(hypervisors=[self._view_hypervisor(hyp, False)
                                 for hyp in hypervisors])

    @extensions.expected_errors(404)
    def servers(self, req, id):
        context = req.environ['nova.context']
        authorize(context)
        try:
            compute_node = self.host_api.compute_node_get(context, id)
        except (ValueError, exception.ComputeHostNotFound):
            msg = _("Hypervisor with ID '%s' could not be found.") % id
            raise webob.exc.HTTPNotFound(explanation=msg)
        instances = self.host_api.instance_get_all_by_host(context,
            compute_node['service']['host'])
        return dict(hypervisor=self._view_hypervisor(compute_node, False,
            instances))

    @extensions.expected_errors(())
    def statistics(self, req):
        context = req.environ['nova.context']
        authorize(context)
        stats = self.host_api.compute_node_statistics(context)
        return dict(hypervisor_statistics=stats)

class Hypervisors(extensions.V3APIExtensionBase):
    """Admin-only hypervisor administration."""

    name = "Hypervisors"
    alias = ALIAS
    version = 1

    def get_resources(self):
        resources = [extensions.ResourceExtension(ALIAS,
                HypervisorsController(),
                collection_actions={'detail': 'GET',
                                    'search': 'GET',
                                    'statistics': 'GET'},
                member_actions={'uptime': 'GET',
                                'servers': 'GET'})]

        return resources

    def get_controller_extensions(self):
        return []


import os
import datetime
import json
import logging
import logging.handlers
import inspect
from jsonHelper import MyEncoder
import xdm
import traceback

LOGLINECACHESIZE = 20
# "c" is for console "p" is for i forgot, but it is used for the file logger
# the "c" strings have terminal control strings that change the color ^_^
lvlNames = {    logging.ERROR:      {'c': u'   \x1b[31;1mERROR\x1b[0m', 'p': 'ERROR'},
                logging.WARNING:    {'c': u' \x1b[35;1mWARNING\x1b[0m', 'p': 'WARNING'},
                logging.INFO:       {'c': u'    \x1b[32;1mINFO\x1b[0m', 'p': 'INFO'},
                logging.DEBUG:      {'c': u'   \x1b[36;1mDEBUG\x1b[0m', 'p': 'DEBUG'},
                logging.CRITICAL:   {'c': u'\x1b[43;1m\x1b[31;1mCRITICAL\x1b[49\x1b[0m', 'p': 'CRITICAL'}
                }

logging.captureWarnings(True)
cLogger = logging.getLogger('XDM.Console')
fLogger = logging.getLogger('XDM.File')
cLogger.setLevel(logging.INFO)
fLogger.setLevel(logging.DEBUG)
# create file handler which logs even debug messages
# this is now done in XDM.py to use the data dir
# fh = logging.handlers.RotatingFileHandler('xdm.log', maxBytes=10 * 1024 * 1024, backupCount=5)
# create console handler with a higher log level
ch = logging.StreamHandler()

# add the handlers to logger

cLogger.addHandler(ch)
# fLogger.addHandler(fh)
""" at some point i want the cherrypy stuff logged
cpLogger = logging.getLogger('cherrypy')
cph = logging.StreamHandler()
formatter = logging.Formatter('%(levelname)s| %(asctime)s: %(message)s ')
cph.setFormatter(formatter)
cpLogger.addHandler(cph)
"""

# http://stackoverflow.com/questions/2203424/python-how-to-retrieve-class-information-from-a-frame-object
def get_class_from_frame(fr):
    args, _, _, value_dict = inspect.getargvalues(fr)
    # we check the first parameter for the frame function is
    # named 'self'
    if len(args) and args[0] == 'self':
        # in that case, 'self' will be referenced in value_dict
        instance = value_dict.get('self', None)
        if instance:
            # return its class
            return getattr(instance, '__class__', None)
    # return None otherwise
    return None

class StructuredMessage(object):
    def __init__(self, lvl, message, calframe, **kwargs):
        self.lvl = lvl
        self.message = message
        self.calframe = calframe
        self.kwargs = kwargs
        self.time = datetime.datetime.now()

    def console(self):
        return u'%s| %s: %s' % (lvlNames[self.lvl]['c'],
                                self.time,
                                self.message)

    def __str__(self):
        def _json(time, lvl, message, calframe, kwargs={}):
            return json.dumps({'time': time,
                               'lvl': lvlNames[lvl]['p'],
                               'msg': message,
                               'caller': {'file': calframe[2][1], 'line': calframe[2][2], 'fn': calframe[2][3]},
                               'data': kwargs},
                               cls=MyEncoder)

        try:
            return _json(self.time, self.lvl, self.message, self.calframe, self.kwargs)
        except TypeError:
            return _json(self.time, self.lvl, self.message, self.calframe)

class LogWrapper():

    _logLineCache = []

    def _log(self, lvl, msg, censor=None, **kwargs):
        if xdm.common.STARTOPTIONS is None or (not xdm.common.STARTOPTIONS.dev):
            if type(censor) == tuple:
                for s in censor:
                    msg = msg.replace(s, '##censored##')
            elif type(censor) == dict:
                for value, name in censor.items():
                    msg = msg.replace(value, '##%s##' % name)
            elif type(censor) == str:
                msg = msg.replace(censor, '##censored##')

        if (xdm.common.SYSTEM is None or (xdm.common.SYSTEM.c.censor_xdm_dir)) and xdm.APP_PATH:
            msg = msg.replace(xdm.APP_PATH, '##xdm_path##')

        curframe = inspect.currentframe()
        calframe = inspect.getouterframes(curframe, 0)
        sm = StructuredMessage(lvl, msg, calframe, **kwargs)
        try:
            cLogger.log(lvl, sm.console())
            _line = u'%s' % sm
            fLogger.log(lvl, _line)
        except (UnicodeDecodeError, UnicodeEncodeError):
            return
        self._logLineCache.append(_line)
        if len(self._logLineCache) > LOGLINECACHESIZE:
            self._logLineCache = self._logLineCache[1:]
        if lvl in (logging.WARNING, logging.ERROR):
            callerClass = get_class_from_frame(calframe[2][0])
            # was the error/warning send by a notifier ?
            if callerClass and callerClass.__bases__ and callerClass.__bases__[0] is not None and 'Notifier' == callerClass.__bases__[0].__name__:
                sm = StructuredMessage(logging.ERROR, 'Error while sending an error message with a notifier %s' % callerClass, calframe, **kwargs)
                cLogger.log(lvl, sm.console())
                fLogger.log(lvl, sm)
            else:
                self.debug('sending %s with notifiers' % lvlNames[lvl]['p'])
                for n in xdm.common.PM.N:
                    if (n.c.on_warning and lvl == logging.WARNING) or (n.c.on_error and lvl == logging.ERROR):
                        n.sendMessage('%s: %s' % (lvlNames[lvl]['p'], msg))

    def error(self, msg, censor=None, **kwargs):
        tb = traceback.format_exc()
        msg = '%s\nTraceback:\n%s' % (msg, tb)
        self._log(logging.ERROR, msg, censor=censor, **kwargs)
        return msg

    def info(self, msg, censor=None, **kwargs):
        self._log(logging.INFO, msg, censor=censor, **kwargs)
        return msg

    def warning(self, msg, censor=None, **kwargs):
        self._log(logging.WARNING, msg, censor=censor, **kwargs)
        return msg

    def debug(self, msg, censor=None, **kwargs):
        self._log(logging.DEBUG, msg, censor=censor, **kwargs)
        return msg

    def critical(self, msg, censor=None, **kwargs):
        self._log(logging.CRITICAL, msg, censor=censor, **kwargs)
        return msg

    def __call__(self, msg, censor=None, **kwargs):
        self._log(logging.DEBUG, msg, censor=censor, **kwargs)
        return msg

    def getEntries(self, entries=10):
        if entries > len(self._logLineCache):
            f = open(xdm.LOGPATH, 'r')
            try:
                logLinesStr = tail(f, entries)
            finally:
                f.close()
        else:
            logLinesStr = self._logLineCache
        return [{
            'data': json.loads(l),
            'raw': json.dumps(json.loads(l), indent=4),
            'id': 'AA%s' % hash(l)} for l in logLinesStr]

log = LogWrapper()

# http://stackoverflow.com/a/13790289/729059
def tail(f, lines=1, _buffer=4098):
    """Tail a file and get X lines from the end"""
    # place holder for the lines found
    lines_found = []

    # block counter will be multiplied by buffer
    # to get the block size from the end
    block_counter = -1

    lines = int(lines) # make sure we get a int !!
    # loop until we find X lines
    while len(lines_found) < lines:
        try:
            f.seek(block_counter * _buffer, os.SEEK_END)
        except IOError: # either file is too small, or too many lines requested
            f.seek(0)
            lines_found = f.readlines()
            break

        lines_found = f.readlines()

        # we found enough lines, get out
        if len(lines_found) > lines:
            break

        # decrement the block counter to get the
        # next X bytes
        block_counter -= 1

    return lines_found[-lines:]

__all__ = ['log']


#####################################################
# Project: Payload Drop								#
# Created by: William Gregory						#
# Description: ArduPlane targeted payload drop		#
# Last Modified Date: August 24th 2017				#
# Version: 1.0										#
#####################################################

# Import
#-------------------------------------------------------------

import math
import clr
import time
clr.AddReference("MissionPlanner")
import MissionPlanner
clr.AddReference("MissionPlanner.Utilities")
from MissionPlanner.Utilities import Locationwp
clr.AddReference("MAVLink")
import MAVLink

# Variables
#-------------------------------------------------------------

## Settings
# target
target_lat = 39.40456			# lat of target
target_lon = -119.761292		# lon of target
# general
real_drop = True				# real or dummy drop
target_waypoint = 6				# waypoint to perform drop
countdown_range = 50			# countdown range in addition to release_distance
release_pin = 7					# physical drop device pin number

## State Variables
# general
mode = 0						# current mode 0: idle, 1:run, 2:done, 3:aborted
target_location = [] 			# lat, lon of target
drop_location = []				# lat, lon of determined drop location
# current flight data
ct_a = 0.0 						# altitude
ct_gs = 0.0 					# groundspeed
ct_gc = 0.0						# ground course
ct_d = 0.0 						# distance to waypoint
ct_w = 0						# waypoint number
ct_ws = 0.0						# wind velocity
ct_wd = 0.0						# wind direciton

# General Funcitons
#-------------------------------------------------------------

# get updated flight data
def getFlightData():
	global ct_t, ct_d, ct_a, ct_gs, ct_gc, ct_w, ct_wd, ct_ws
	ct_t = time.time()
	ct_d = cs.wp_dist
	ct_a = cs.alt
	ct_gs = cs.groundspeed
	ct_gc = cs.groundcourse
	ct_w = cs.wpno
	ct_wd = cs.wind_dir
	ct_ws = cs.wind_vel

# print flight data
def printFlightData():
	print("info: flight data")
	print("- altitude: " + str(ct_a))
	print("- distance: " + str(ct_d))
	print("- groundspeed: " + str(ct_gs))
	print("- groundcourse: " + str(ct_gc))
	print("- waypoint: " + str(ct_w))
	print("- wind direction: " + str(ct_wd))
	print("- wind speed: " + str(ct_ws))

# release the payload
def dropPayload():
	if mode == 1:
		if real_drop:
			Script.SendRC(release_pin, 953, True)
		print ("noteice: payload released")
		if real_drop:
			Script.Sleep(5000)
			Script.SendRC(release_pin, 2028, True)
		Script.ChangeMode("AUTO")
		#Script.ChangeMode('RTL')
		return True
	return False

# Autostart
#-------------------------------------------------------------

def autostart():
	print("payload drop script - online")
	# test drop
	dropPayload();
	# test data
	getFlightData();
	printFlightData();
	print("payload drop script - offline")

## --
autostart();


from PyPDF2 import PdfFileReader, PdfFileWriter

reader = PdfFileReader('/dest/merged.pdf','r') #from merged Pdf File

# returns the points(pt) 1 inch = 72pt and 1 inch = 2.54 cm
'''
page = reader.getPage(5)
print(page.cropBox.getLowerLeft())
print(page.cropBox.getUpperLeft())
print(page.cropBox.getUpperRight())
print(page.cropBox.getLowerRight())
'''
writer = PdfFileWriter()
for i in range(reader.getNumPages()):
    page = reader.getPage(i)
    page.cropBox.setLowerLeft((15,50))
    page.cropBox.setUpperLeft((15,842))
    page.cropBox.setLowerRight((580,50))
    page.cropBox.setUpperRight((580,842))
    writer.addPage(page)

# to merged pdf file
outstream = open("/home/user/Desktop/cropped.pdf","wb")
writer.write(outstream)
outstream.close()


#! /usr/bin/env python
import sys

from scikits.audiolab import wavread
import pylab as plt
import numpy as np
import btrack_plus as btrack_plus
#import btrack as btrack
import time as time
import os, os.path
import pickle
import beat_evaluation_toolbox as be

x = np.zeros(128)
PI = 3.1415926535897;

for n in range(len(x)):
    x[n] = np.sin(2*PI*(float(n)/float(len(x)))*3)

BUFFER_LEN = len(x)

ratio = 512/BUFFER_LEN

frac_1 = 1.0/float(ratio)

df512 = np.zeros(512)
n = 0
for i in range(BUFFER_LEN):
    df512[n] = x[i];
    for k in range(1,ratio):
        frac_2 = k*frac_1;
        if (i < BUFFER_LEN-1):
            df512[n+k] = x[i]*(1-frac_2) + (x[i+1]*frac_2);
        else:
            df512[n+k] = x[i]*(1-frac_2) + ((x[i]+(x[i]-x[i-1]))*frac_2)

    n = n+ratio
        #for k in range(1,ratio):
        #

plt.subplot(211)
plt.plot(x)
plt.xlim(0,len(x))
plt.subplot(212)
plt.plot(df512)
plt.xlim(0,len(df512))
plt.show()


class Email:
    """Creating a class definition for an email which takes in the arguments below
    and stores them as instance-level variables. It also takes in two more instance-level variables with default values."""

    def __init__(self, from_address, subject_line, email_contents):
        self.from_address = from_address
        self.subject_line = subject_line
        self.email_contents = email_contents
        self.has_been_read = False
        self.is_spam = False

    def mark_as_read(self):
        """Method that changes the status of has_been_read."""
        self.has_been_read = True

    def mark_as_spam(self):
        """Method that changes the status of is_spam."""
        self.is_spam = True

class Inbox:
    """Class that stores all emails and takes in no arguments."""

    def __init__(self):
        self.emails = []

    def add_email(self, from_address, subject_line, email_contents):
        """Method which takes in the contents and email address from the received
        email to make a new Email object and stores it in the inbox"""
        new_email = Email(from_address, subject_line, email_contents)
        self.emails.append(new_email)

    def list_messages_from_sender(self, sender_address):
        """Method which returns a string showing all subject lines in emails from a specific sender, along with corresponding number"""
        subject_lines = []
        found_sender = False
        for i, email in enumerate(self.emails):
            if email.from_address == sender_address:
                subject_lines.append(f"{i}: {email.subject_line}")
                found_sender = True

        if not found_sender:
            print("No email found from this sender")
        return "\n".join(subject_lines)

    def get_email(self, sender_address, index):
        """Method that returns the email at a specific index from a specific user."""
        for email in self.emails:
            if email.from_address == sender_address and not email.has_been_read:
                if index == 0:
                    email.mark_as_read()
                    return email.email_contents
                else:
                    index -= 1
        raise IndexError("Email not found.")

    def mark_as_spam(self, sender_address, index):
        """Method that marks the email at a specific index within a sender address as spam."""
        for email in self.emails:
            if email.from_address == sender_address and not email.is_spam:
                if index == 0:
                    email.mark_as_spam()
                    return
                else:
                    index -= 1
        raise IndexError("Email not found.")

    def get_unread_emails(self):
        """Method that returns a string containing a list of all unread emails."""
        unread_subject_lines = []
        for email in self.emails:
            if not email.has_been_read:
                unread_subject_lines.append(email.subject_line)
        return "\n".join(unread_subject_lines)

    def get_spam_emails(self):
        """Method that returns a string containing a list of all spam emails."""
        spam_subject_lines = []
        for email in self.emails:
            if email.is_spam:
                spam_subject_lines.append(email.subject_line)
        return "\n".join(spam_subject_lines)

    def delete(self, sender_address, index):
        """Method that deletes an email from the inbox."""
        for email in self.emails:
            if email.from_address == sender_address:
                if index == 0:
                    self.emails.remove(email)
                    return
                else:
                    index -= 1
        raise IndexError("Email not found.")

usage_message = """
Welcome to the email system! What would you like to do?

s - send email.
l - list emails from a sender.
r - read email.
m - mark email as spam.
gu - get unread emails.
gs - get spam emails.
d - delete email.
e - exit this program.
"""

inbox = Inbox()
# adding some emails to the inbox
inbox.add_email("example1@example.com", "Subject 1", "Email contents 1")
inbox.add_email("example1@example.com", "Subject 1", "Email contents 1.1")
inbox.add_email("example2@example.com", "Subject 2", "Email contents 2")
inbox.add_email("example3@example.com", "Subject 3", "Email contents 3")

user_choice = ""

while True:
    user_choice = input(usage_message).strip().lower()
    if user_choice == "s":
        # Send an email
        sender_address = input("Please enter the address of the sender\n:")
        subject_line = input("Please enter the subject line of the email\n:")
        contents = input("Please enter the contents of the email\n:")
        # Creating a new Email object
        email = Email(sender_address, subject_line, contents)

        # Adding a new email to the inbox
        inbox.add_email(sender_address, subject_line, contents)
        # Print a success message
        print("Email has been added to inbox.")

    elif user_choice == "l":
        # List all emails from a sender
        sender_address = input("Please enter the address of the sender of the email\n:")
        # Print all emails from this sender
        print(inbox.list_messages_from_sender(sender_address))

    elif user_choice == "r":
        # Read an email
        sender_address = input("Please enter the address of the sender of the email\n:")
        # Print all emails from the sender with indexes
        print(inbox.list_messages_from_sender(sender_address))

        # Ask the user for the index of the email they wish to read
        index = int(
            input("Please enter the index of the email that you would like to read\n:")
        )

        # Display the email selected
        print(inbox.get_email(sender_address, index))

    elif user_choice == "m":
        # Mark an email as spam
        sender_address = input("Please enter the address of the sender of the email\n:")
        # Show emails from the sender with indexes
        print(inbox.list_messages_from_sender(sender_address))
        # Ask the user for the index of the email to be marked as spam
        index = int(
            input("Please enter the index of the email to be marked as spam\n:")
        )
        # Mark the email as spam
        inbox.mark_as_spam(sender_address, index)
        # Print a success message
        print("Email marked as spam.")

    elif user_choice == "gu":
        # List all unread emails
        print(inbox.get_unread_emails())

    elif user_choice == "gs":
        # List all spam emails
        print(inbox.get_spam_emails())

    elif user_choice == "d":
        # Delete an email
        sender_address = input("Please enter the address of the sender of the email\n:")
        # Show emails from the sender with indexes
        print(inbox.list_messages_from_sender(sender_address))
        # Ask the user for the index of the email they wish to delete
        index = int(input("Please enter the index of the email to be deleted\n:"))
        # Delete the selected email
        inbox.delete(sender_address, index)
        # Print a success message
        print("Email has been deleted.")

    elif user_choice == "e":
        # Exit the program
        print("Goodbye!")
        break

    else:
        print("Oops - incorrect input")


class Solution:
    """
    @param a: The A array
    @param b: The B array
    @param s: The S string
    @return: The answer
    """
    def stringReplace(self, a, b, s):
        # Write your code here
        seed, mod, maxLen = 31, 1000000007, -1
        result = 0
        aHash, sHash, base = [], [1], [1]

        for i in a:
            result = 1
            maxLen = max(maxLen, len(i))
            for j in i:
                result = (result * seed + ord(j) - ord('a')) % mod
            aHash.append(result)

        maxLen, result = max(maxLen, len(s)), 1
        for i in s:
            result = (result * seed + ord(i) - ord('a')) % mod
            sHash.append(result)

        result = 1
        for i in range(maxLen):
            result = result * seed % mod
            base.append(result)

        result = [i for i in s]

        i = 0
        while i < len(s):
            maxLen, index = -1, 0
            for j in range(len(a)):
                size_aj = len(a[j])
                left, right = i + 1, i + size_aj
                if right > len(s):
                    continue

                sHashValue = (sHash[right] - base[right - left + 1] * sHash[left - 1] % mod + mod) % mod
                aHashValue = (aHash[j] - base[size_aj] + mod) % mod
                if  sHashValue == aHashValue and size_aj > maxLen:
                    maxLen = size_aj
                    index = j

            if maxLen != -1:
                for j in range(maxLen):
                    result[i + j] = b[index][j]
                i += maxLen - 1
            i += 1

        return "".join(result)

01 - Defeat the Algorithm Interview/841.StringReplace.py