#!/usr/bin/python3

"""
This module contains a function that writes an Object to a
text file, using a JSON representation
"""

import json

def save_to_json_file(my_obj, filename):
    """
    This function saves a json object to a file
    """
    if type(filename) is str and filename is not None:
        with open(filename, mode="w", encoding="UTF-8") as doc:
            doc.write(json.dumps(my_obj))


from pathlib import Path
from os.path import join
from spacy.util import minibatch, compounding

import plac
import random
import warnings
import spacy
import json
import os

CURRENT_PATH = Path(__file__).parent.absolute()
TRAIN_DATA = []

@plac.annotations(
    model=("Model name. Defaults to blank 'en' model.", "option", "m", str),
    model_path=("Model path", "option", "mp", Path),
    output_dir=("Optional output directory", "option", "o", Path),
    n_iter=("Number of training iterations", "option", "n", int),
    train_data_file_path=("Raw train data in json format", "option", "t", Path)
)
def train(model=None, model_path=None, output_dir=None, n_iter=100, train_data_file_path=join(CURRENT_PATH, "./train_data.json")):
    if model or model_path is not None:
        if model is not None:
            nlp = spacy.load(model)
            print("Loaded model '%s'" % model)
        elif model_path is not None:
            nlp = spacy.load(Path(model_path))
            print("Loaded model '%s'" % model_path)
    else:
        nlp = spacy.load('en_core_web_sm')
        print("Created blank 'en' model")

    if "ner" not in nlp.pipe_names:
        ner = nlp.create_pipe("ner")
        nlp.add_pip(ner, last=True)
    else:
        ner = nlp.get_pipe("ner")

    if train_data_file_path is not None:
        train_data_file_path = Path(train_data_file_path)
        if not train_data_file_path.exists():
            train_data_file_path.touch()

        with open(train_data_file_path) as train_data_file:
            train_data_json = json.load(train_data_file)

            for training_record in train_data_json:
                train_text = training_record["train_text"]
                entities = []

                for entity in training_record["entities"]:
                    entities.append(
                        (entity[0], entity[1], entity[2])
                    )

                print(entities)

                TRAIN_DATA.append(
                    (train_text, {"entities": entities})
                )

    for _, annotations in TRAIN_DATA:
        for ent in annotations.get("entities"):
            ner.add_label(ent[2])

    pipe_exceptions = ["ner", "trf_wordpiecer", "trf_tok2vec"]
    other_pipes = [pipe for pipe in nlp.pipe_names if pipe not in pipe_exceptions]

    with nlp.disable_pipes(*other_pipes), warnings.catch_warnings():
        warnings.filterwarnings("once", category=UserWarning, module='spacy')

        print(model, model_path)

        if model and model_path is None:
            optimizer = nlp.begin_training()
        else:
            optimizer = nlp.entity.create_optimizer()

        for itn in range(n_iter):
            random.shuffle(TRAIN_DATA)
            losses = {}
            batches = minibatch(TRAIN_DATA, size=compounding(4.0, 32.0, 1.001))
            for batch in batches:
                texts, annotations = zip(*batch)
                nlp.update(
                    texts,
                    annotations,
                    sgd=optimizer,
                    drop=0.5,
                    losses=losses
                )

                print("Losses", losses)

    if output_dir is not None:
        output_dir = Path(output_dir)
        if not output_dir.exists():
            output_dir.mkdir()
        nlp.to_disk(output_dir)
        print("Saved model to", output_dir)

    for text, _ in TRAIN_DATA:
        doc = nlp(text)
        print("Entities", [(ent.text, ent.label_) for ent in doc.ents])

    os.remove(train_data_file_path)

if __name__ == "__main__":
    plac.call(train)


# 1047. 删除字符串中的所有相邻重复项
# 给出由小写字母组成的字符串S，重复项删除操作会选择两个相邻且相同的字母，并删除它们。
# 在 S 上反复执行重复项删除操作，直到无法继续删除。
# 在完成所有重复项删除操作后返回最终的字符串。答案保证唯一。
import datetime
import random
import string

def removeDuplicates_first(s: str):
    if len(s) < 2:
        return s
    while True:
        changed = False
        for i in range(len(s) - 1):
            if s[i] == s[i + 1]:
                s = s[:i] + s[i + 2:]
                changed = True
                break
        if not changed:
            break
    return s

s = "abbaca"
removeDuplicates_first(s)

def removeDuplicates_second(s: str):
    if len(s) < 2:
        return s
    stack = list()
    for i in range(len(s)):
        if not stack or s[i] != stack[-1]:
            stack.append(s[i])
        else:
            stack.pop()
    return ''.join(stack)

# s = "abbaca"
letters = string.ascii_letters[5:10]
s = ''
for i in range(50000):
    s += random.choice(letters)
start_time = datetime.datetime.now()
result1 = removeDuplicates_first(s)
end_time = datetime.datetime.now()
print(end_time - start_time)
start_time = datetime.datetime.now()
result2 = removeDuplicates_second(s)
end_time = datetime.datetime.now()
print(end_time - start_time)
print(result1 == result2)

1047. 删除字符串中的所有相邻重复项.py


__author__ = "Jianfeng Sun"
__version__ = "v1.0"
__copyright__ = "Copyright 2020"
__license__ = "GPL v3.0"
__email__ = "jianfeng.sunmt@gmail.com"
__maintainer__ = "Jianfeng Sun"

import os
import sys
import getopt
import subprocess
from src.FS_dprs import fs_dprs
from src.Stacking_dprs import stacking_dprs

def usage():
    return """
    DeepTMInter: improving prediction of interaction sites in transmembrane protein.

    Usage: run_deeptminter.py [-n|--name, [sequence name']] [-c|--chain, [sequence chain name]] [-i|--input, [input path]] [-o|--output, [output path]] [-r|--region, [region]] [-h|--help] [-v|--version]

    Description
                -n, --name      Sequence name. For example, '3jcu'.
                -c, --chain     Sequence chain name. For example, 'H'. This can be empty if you prefer a sequnce name like '3jcuH' or '0868'.
                -i, --input     Input path.
                -o, --output    Output path.
                -r, --region    region of transmembrane protein. It can take 'transmembrane', 'cytoplasmic', 'extracellular', 'combined'.

    for example:
    python run_deeptminter.py -n 3jcu -c H -i ./input/ -o ./output/ -r transmembrane

    """

def parser():
    opts, args = getopt.getopt(sys.argv[1:], '-h-n:-c:-i:-o:-r:-v',
                               ['help', 'name=', 'chain=', 'input=', 'output=', 'region=', 'version'])
    # print(opts)
    for opt_name, opt_value in opts:
        if opt_name in ('-h', '--help'):
            print(usage())
            sys.exit(0)
        if opt_name in ('-v', '--version'):
            print("*Version: 1.0")
        if opt_name in ('-n', '--name'):
            seq_name = opt_value
        if opt_name in ('-c', '--chain'):
            seq_chain = opt_value
        if opt_name in ('-i', '--input'):
            input = opt_value
        if opt_name in ('-o', '--output'):
            output = opt_value
        if opt_name in ('-r', '--region'):
            region = opt_value
        # else:
        #     print('wrong usage.')
        #     sys.exit(0)
    return seq_name, seq_chain, input, output, region

seq_name, seq_chain, input_path, output_path, region = parser()

# seq_name = '3jcu'
# seq_chain = 'H'
# input_path = './input/'
# output_path = './output/'
# region = 'extracellular'

fs_in = {
    'prot_name': seq_name,
    'file_chain': seq_chain,
    'window_size_1': 1,
    'window_size_2': 4,
    'fasta_path': input_path,
    'msa_path': input_path,
    'phobius_path': input_path,
    'mi_path': input_path,
    'fc_path': input_path,
    'cp_path': input_path,
    'plmc_path': input_path,
    'gdca_path': input_path,
    'sv_suffix': '.fs',
    'sv_path': input_path,
}
fs_dprs(fs_in).generate()

m1_in = [
    "python",
    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'deeptminter_call.py'),
    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'model/m1/m1.meta'),
    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'model/m1/m1'),
    seq_name,
    seq_chain,
    input_path,
    output_path,
    '.fs',
    '.m1',
]
subprocess.Popen(m1_in, shell=True).communicate()

m2_in = [
    "python",
    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'deeptminter_call.py'),
    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'model/m2/m2.meta'),
    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'model/m2/m2'),
    seq_name,
    seq_chain,
    input_path,
    output_path,
    '.fs',
    '.m2',
]
subprocess.Popen(m2_in, shell=True).communicate()

m3_in = [
    "python",
    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'deeptminter_call.py'),
    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'model/m3/m3.meta'),
    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'model/m3/m3'),
    seq_name,
    seq_chain,
    input_path,
    output_path,
    '.fs',
    '.m3',
]
subprocess.Popen(m3_in, shell=True).communicate()

m4_in = [
    "python",
    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'deeptminter_call.py'),
    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'model/m4/m4.meta'),
    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'model/m4/m4'),
    seq_name,
    seq_chain,
    input_path,
    output_path,
    '.fs',
    '.m4',
]
subprocess.Popen(m4_in, shell=True).communicate()

m5_in = [
    "python",
    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'deeptminter_call.py'),
    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'model/m5/m5.meta'),
    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'model/m5/m5'),
    seq_name,
    seq_chain,
    input_path,
    output_path,
    '.fs',
    '.m5',
]
subprocess.Popen(m5_in, shell=True).communicate()

mexpand1_in = [
    "python",
    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'deeptminter_call.py'),
    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'model/mexpand1/mexpand1.meta'),
    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'model/mexpand1/mexpand1'),
    seq_name,
    seq_chain,
    input_path,
    output_path,
    '.fs',
    '.mexpand1',
]
subprocess.Popen(mexpand1_in, shell=True).communicate()

mexpand2_in = [
    "python",
    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'deeptminter_call.py'),
    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'model/mexpand2/mexpand2.meta'),
    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'model/mexpand2/mexpand2'),
    seq_name,
    seq_chain,
    input_path,
    output_path,
    '.fs',
    '.mexpand2',
]
subprocess.Popen(mexpand2_in, shell=True).communicate()

mexpand3_in = [
    "python",
    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'deeptminter_call.py'),
    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'model/mexpand3/mexpand3.meta'),
    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'model/mexpand3/mexpand3'),
    seq_name,
    seq_chain,
    input_path,
    output_path,
    '.fs',
    '.mexpand3',
]
subprocess.Popen(mexpand3_in, shell=True).communicate()

stk_in = {
    'prot_name': seq_name,
    'file_chain': seq_chain,
    'region': region,
    'sv_test_set': output_path,
    'input_path': input_path,
    'phobius_path': input_path,
    'model_fpn': os.path.join(os.path.dirname(os.path.abspath(__file__)), 'model/ensemble.model'),
    'sv_pred': output_path,
    'sv_suffix': '.deeptminter'
}
stk = stacking_dprs(stk_in)
stk.stkg()
stk.stkr()


import hashlib

def sum_digits(digit):
    if digit < 10:
        return digit
    else:
        sum = (digit % 10) + (digit // 10)
        return sum

def validate(cc_num):
    # reverse the credit card number
    cc_num = cc_num[::-1]
    # convert to integer list
    cc_num = [int(x) for x in cc_num]
    # double every second digit
    doubled_second_digit_list = list()
    digits = list(enumerate(cc_num, start=1))
    for index, digit in digits:
        if index % 2 == 0:
            doubled_second_digit_list.append(digit * 2)
        else:
            doubled_second_digit_list.append(digit)

    # add the digits if any number is more than 9
    doubled_second_digit_list = [sum_digits(x) for x in doubled_second_digit_list]
    # sum all digits
    sum_of_digits = sum(doubled_second_digit_list)
    # return True or False
    return sum_of_digits % 10 == 0

while True:
    try:
        first6_str = input("Enter first 6 digits: ")
        if len(first6_str) != 6:
            print("Sorry, must be 6 digits")
            continue
        first6 = int(first6_str)
    except ValueError:
        print("Sorry, only integers")
        continue
    else:
        break
print("First 6 digits: " + first6_str)

while True:
    try:
        last4_str = input("Enter last 4 digits: ")
        if len(last4_str) != 4:
            print("Sorry, must be 4 digits")
            continue
        last4 = int(last4_str)
    except ValueError:
        print("Sorry, only integers")
        continue
    else:
        break
print("Last 4 digits: " + last4_str)

hash_str = input("Enter any string of MD5 hash: ")
print("Provided MD5 hash strings: " + hash_str)

initialPAN = first6_str + '000000' + last4_str
initialPAN = int(initialPAN)
tot_pan = 0
print("Constructed PAN(s): ")
for i in range(1000000):
	hash_value_pan = hashlib.md5(str(initialPAN).encode()).hexdigest()
	split_hash_pan = hash_value_pan[0:len(hash_str)]
	if split_hash_pan == hash_str:
		if validate(str(initialPAN)):
			print(initialPAN)
			tot_pan = tot_pan + 1
	initialPAN += 10000
print("Total constructed PAN(s): " + format(tot_pan))


import conect

def insert_type(id,name):
    query = f"select * from type where id={id} and name=\"{name}\""
    if not conect.select_query(query):
        query = f"insert into type (id,name) values ({id},\"{name}\")"
        conect.insert_query(query)

def delete_pokemon(id_pokemon,id_trainer):
    query = f"delete from owner_by where id_pokemon={id_pokemon} and id_trainer={id_trainer}"
    conect.insert_query(query)

def insert_pokemon(id, name, types, height, weight, ownedBy):
    query = f"select id from pokemon where name=\"{name}\""
    if conect.select_query(query):
        return "this pokemon name already exist"
    query = f"insert into pokemon (id,name,height,weight) values ({id},\"{name}\",{height},{weight})"
    conect.insert_query(query)
    if types!=0:
        for type in types:
            if(type(type)==str):
                name = type
            else:
                name = type["type"]["name"]
            query = f"select id from type where name=\"{name}\" and id=\"{id}\""
            result=conect.select_query(query)
            if len(result)==0:
                query = f"insert into type(id,name) values({id},\"{name}\")"
                conect.insert_query(query)
    for own in ownedBy:
        name = own["name"]
        town=own["town"]
        query = f"select id from trainer where name=\"{name}\" and town=\"{town}\""
        result=conect.select_query(query)
        if len(result)==0:
            query = f"INSERT into trainer (name,town) values (\"{name}\",\"{town}\")"
            conect.insert_query(query)
            query= f"select id from trainer where name=\"{name}\" and town=\"{town}\""
            result=conect.convert_array(conect.select_query(query))
            id_trainer=result[0]
            query = f"INSERT into owner_by (id_pokemon,id_trainer) values (\"{id}\",\"{id_trainer}\")"
            conect.insert_query(query)

def get_id_by_name(name,name_table):
    query=f"select id from {name_table} where name=\"{name}\""
    res = conect.select_query(query)
    if len(res)==0:
        return False
    return res[0]["id"]

def is_owner_by(id_pokemon,id_trainer):
    query = f"select * from owner_by where id_pokemon=\"{id_pokemon}\" and id_trainer=\"{id_trainer}\""
    res = conect.select_query(query)
    if(len(res)==0):
        return False
    return True

def insert_owner_by(id_pokemon,id_trainer):
    query = f"INSERT into owner_by (id_pokemon,id_trainer) values ({id_pokemon},{id_trainer})"
    conect.insert_query(query)


# -*- coding: utf-8 -*-

import gtk, pango

class BaseMessageDialog(gtk.MessageDialog):
    pass

class ConfigValidationErrorDialog(BaseMessageDialog):
    def __init__(self, parent, errors, **kws):
        BaseMessageDialog.__init__(self, parent=parent, type=gtk.MESSAGE_ERROR, buttons=gtk.BUTTONS_CLOSE, **kws)
        self.set_title('Item validation error')
        pre = 'The settings cannot be saved due to the following errors. \n\n%s\n\n Please correct the errors and try again.'
        self.set_markup(pre % ': '.join(errors))
        self.connect('response', self.on_close)

    def on_close(self, w, id, *args):
        self.destroy()

class ResendConfimDialog(BaseMessageDialog):
    def __init__(self, parent, msg, **kws):
        BaseMessageDialog.__init__(self, parent=parent, type=gtk.MESSAGE_QUESTION, buttons=gtk.BUTTONS_YES_NO, **kws)
        self.set_title('Message delivery error')
        pre = 'The following message is not deliverd. \n\n%s\n\n Would you like to give it a retry?'
        self.set_markup(pre % msg)

class AttachmentsErrorDialog(BaseMessageDialog):
    def __init__(self, parent, errors, **kws):
        BaseMessageDialog.__init__(self, parent=parent, type=gtk.MESSAGE_QUESTION, buttons=gtk.BUTTONS_YES_NO, **kws)
        self.set_title('File attach error')
        pre = 'The following file can not sent. \n\n%s\n\n Would you like to continue with other files?'
        self.set_markup(pre % '\n'.join([':'.join(e) for e in errors]))

class NetworkErrorDialog(BaseMessageDialog):
    def __init__(self, parent, **kws):
        BaseMessageDialog.__init__(self, parent=parent, type=gtk.MESSAGE_INFO, buttons=gtk.BUTTONS_CLOSE, **kws)
        self.set_title('Network error')
        pre = 'Opps! I can\'t find any avalible network connection. Going offline.'
        self.set_markup(pre)
        self.connect('response', self.on_close)

    def on_close(self, w, id, *args):
        self.destroy()

class AddressBindDialog(BaseMessageDialog):
    def __init__(self, parent, port, **kws):
        BaseMessageDialog.__init__(self, parent=parent, type=gtk.MESSAGE_INFO, buttons=gtk.BUTTONS_OK_CANCEL, **kws)
        self.set_title('Address binding error')
        pre = 'The port %s is already in use. ' % port
        self.set_markup(pre)
        adj = gtk.Adjustment(port+1, 1024, 65535, 1, 1, 0)
        self.new_port = gtk.SpinButton(adj, 0, 0)
        self.new_port.set_wrap(True)
        self.new_port.show()
        hbox = gtk.HBox()
        hbox.show()
        lbl = gtk.Label('Specify a new port:')
        lbl.show()
        hbox.pack_start(lbl)
        hbox.pack_start(self.new_port, True, True, 2)
        self.vbox.pack_end(hbox)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('meetups', '0001_initial'),
    ]

    operations = [
        migrations.AddField(
            model_name='meetup',
            name='online_event',
            field=models.URLField(blank=True),
        ),
    ]


import numpy as np
import matplotlib.pyplot as plt
from scipy import stats

def find_CV(name):
    d_data = (np.loadtxt(name,skiprows=1).T)
    dT = d_data[3]
    dQ = d_data[2]

    slope, intercept, r_value, p_value, std_err = stats.linregress(dT,dQ)
    return slope, std_err

d_data = (np.loadtxt("problem_d_lammps.dat",skiprows=1).T)
dT = d_data[3]
dQ = d_data[2]

slope, intercept, r_value, p_value, std_err = stats.linregress(dT,dQ)

dT_cont = np.linspace(np.min(dT),np.max(dT),1000)

##############
# Oppgave f) #
##############

Ts = [1,0.694,2,6,10,13]
T_names = ["0694","2","6","10","13"]
C_Vs = [slope]
uncertainties = [std_err]

for t,name in zip(Ts[1:],T_names):

    slope,std_err = find_CV("problem_d_lammbs_%s.dat" %name)

    C_Vs.append(slope)
    uncertainties.append(std_err)

plt.errorbar(Ts,C_Vs,uncertainties,fmt=".-.")
plt.title(r"Heat Capacity as Temperature increases. ($\rho = 0.01$)")
plt.xlabel("Temperature $T$")
plt.ylabel("Heat capacity $C_V$")
plt.show()


import gpt as g
import os
from gpt_qpdf_utils import pion_DA_measurement

parameters = {
    "zmax" : 24,
    "pzmin" : 0,
    "pzmax" : 5,
    "width" : 2.2,
    "pos_boost" : [0,0,0],
    "neg_boost" : [0,0,0],
    "save_propagators" : True
}

# loading gauge configuration
U = g.load(groups[group]["conf_fmt"] % conf)
g.message("finished loading gauge config")

# do gauge fixing

U_prime, trafo = g.gauge_fix(U, maxiter=500)
del U_prime

Measurement = pion_DA_measurement(parameters)

prop_exact, prop_sloppy, pin = Measurement.make_96I_inverter(U, groups[group]["evec_fmt"])

phases = Measurement.make_mom_phases(U[0].grid)

# set source

L = U[0].grid.fdimensions

source_positions_sloppy = [
    [rng.uniform_int(min=0, max=L[i] - 1) for i in range(4)]
    for j in range(jobs[job]["sloppy"])
]
source_positions_exact = [
    [rng.uniform_int(min=0, max=L[i] - 1) for i in range(4)]
    for j in range(jobs[job]["exact"])
]

g.message(f" positions_sloppy = {source_positions_sloppy}")
g.message(f" positions_exact = {source_positions_exact}")

# set output path and open files
root_job = f"{root_output}/{conf}/{job}"
Measurement.set_output_facilites(f"{root_job}/correlators",f"{root_job}/propagators")

# make the Wilson lines
g.message("Starting Wilson lines")
W = Measurement.create_WL(U)

# exact positions
for pos in source_positions_exact:

    g.message("STARTING EXACT MEASUREMENTS")

    g.message("Starting DA 2pt function")

    g.message("Generatring boosted src's")
    srcDp, srcDm = Measurement.create_src_2pt(pos, trafo, U[0].grid)

    g.message("Starting prop exact")
    prop_exact_f = g.eval(prop_exact * srcDp)
    g.message("forward prop done")
    prop_exact_b = g.eval(prop_exact * srcDm)
    g.message("backward prop done")

    tag = "%s/%s" % ("exact", str(pos))

    #construct the list of 'backwards propagators' for DA: (W * prop b)^dagger
    prop_b = Measurement.constr_backw_prop_for_DA(prop_exact_b,W)
    g.message("Start DA contractions")
    Measurement.contract_DA(prop_exact_f, prop_b, phases, tag)
    del prop_b
    g.message("DA done")

    g.message("Starting 2pt contraction (includes sink smearing)")
    Measurement.contract_2pt(prop_exact_f, prop_exact_b, phases, trafo, tag)
    g.message("2pt contraction done")

    if(parameters["save_propagators"]):
        Measurement.propagator_output(tag, prop_exact_f, prop_exact_b)

    del prop_exact_f
    del prop_exact_b

    g.message("STARTING SLOPPY MEASUREMENTS")

    g.message("Starting DA 2pt function")

    g.message("Starting prop sloppy")
    prop_sloppy_f = g.eval(prop_sloppy * srcDp)
    g.message("forward prop done")
    prop_sloppy_b = g.eval(prop_sloppy * srcDm)
    g.message("backward prop done")

    del srcDp
    del srcDm

    tag = "%s/%s" % ("sloppy", str(pos))

    prop_b = Measurement.constr_backw_prop_for_DA(prop_sloppy_b,W)

    g.message("Start DA contractions")
    Measurement.contract_DA(prop_sloppy_f, prop_b, phases, tag)
    del prop_b
    g.message("DA done")

    g.message("Starting 2pt contraction (includes sink smearing)")
    Measurement.contract_2pt(prop_sloppy_f, prop_sloppy_b, phases, trafo, tag)
    g.message("2pt contraction done")

    if(parameters["save_propagators"]):
        Measurement.propagator_output(tag, prop_sloppy_f, prop_sloppy_b)

    del prop_sloppy_f
    del prop_sloppy_b

g.message("exact positions done")

# sloppy positions
for pos in source_positions_sloppy:

    g.message("STARTING SLOPPY MEASUREMENTS")

    # do stuff for sloppy sources

g.message("sloppy positions done")

del pin


class Solution:
    def minAvailableDuration(self, slots1, slots2, duration):
      slots1 = sorted(filter(lambda x: x[1]-x[0] >= duration, slots1))
      slots2 = sorted(filter(lambda x: x[1]-x[0] >= duration, slots2))

      i = j = 0

      while i < len(slots1) and j < len(slots2):
        start1, end1 = slots1[i]
        start2, end2 = slots2[j]

        if start1 >= start2 and start1+duration <= end2:
          return [start1, start1+duration]
        if start2 >= start1 and start2+duration <= end1:
          return [start2, start2+duration]
        if end1 < end2:
          i += 1
        else:
          j += 1
      return []


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models
import datetime

class Migration(migrations.Migration):

    dependencies = [
        ('questions', '0015_auto_20160214_1519'),
    ]

    operations = [
        migrations.AddField(
            model_name='simplequestion',
            name='exam_board',
            field=models.CharField(max_length=100, default=''),
        ),
        migrations.AlterField(
            model_name='dayscore',
            name='date',
            field=models.DateField(default=datetime.date(2016, 3, 24)),
        ),
    ]


import gc
import json
import os
import random
import time
from itertools import chain

import cv2
import numpy as np
import pandas as pd
import torch
from torch import optim
from torch.nn import functional
from tqdm import tqdm

from rivagan.adversary import Adversary, Critic
from rivagan.attention import AttentiveDecoder, AttentiveEncoder
from rivagan.dataloader import load_train_val
from rivagan.dense import DenseDecoder, DenseEncoder
from rivagan.noise import Compression, Crop, Scale
from rivagan.utils import mjpeg, psnr, ssim

def get_acc(y_true, y_pred):
    assert y_true.size() == y_pred.size()
    return (y_pred >= 0.0).eq(y_true >= 0.5).sum().float().item() / y_pred.numel()

def quantize(frames):
    # [-1.0, 1.0] -> {0, 255} -> [-1.0, 1.0]
    return ((frames + 1.0) * 127.5).int().float() / 127.5 - 1.0

def make_pair(frames, data_dim, use_bit_inverse=True, multiplicity=1):
    # Add multiplicity to further stabilize training.
    frames = torch.cat([frames] * multiplicity, dim=0).cuda()
    data = torch.zeros((frames.size(0), data_dim)).random_(0, 2).cuda()

    # Add the bit-inverse to stabilize training.
    if use_bit_inverse:
        frames = torch.cat([frames, frames], dim=0).cuda()
        data = torch.cat([data, 1.0 - data], dim=0).cuda()

    return frames, data

class RivaGAN(object):

    def __init__(self, model="attention", data_dim=32):
        self.model = model
        self.data_dim = data_dim
        self.adversary = Adversary().cuda()
        self.critic = Critic().cuda()
        if model == "attention":
            self.encoder = AttentiveEncoder(data_dim=data_dim).cuda()
            self.decoder = AttentiveDecoder(self.encoder).cuda()
        elif model == "dense":
            self.encoder = DenseEncoder(data_dim=data_dim).cuda()
            self.decoder = DenseDecoder(data_dim=data_dim).cuda()
        else:
            raise ValueError("Unknown model: %s" % model)

    def fit(self, dataset, log_dir=False,
            seq_len=1, batch_size=12, lr=5e-4,
            use_critic=False, use_adversary=False,
            epochs=300, use_bit_inverse=True, use_noise=True):

        if not log_dir:
            log_dir = "experiments/%s-%s" % (self.model, str(int(time.time())))
        os.makedirs(log_dir, exist_ok=False)

        # Set up the noise layers
        crop = Crop()
        scale = Scale()
        compress = Compression()

        def noise(frames):
            if use_noise:
                if random.random() < 0.5:
                    frames = crop(frames)
                if random.random() < 0.5:
                    frames = scale(frames)
                if random.random() < 0.5:
                    frames = compress(frames)

            return frames

        # Set up the data and optimizers
        train, val = load_train_val(seq_len, batch_size, dataset)
        G_opt = optim.Adam(chain(self.encoder.parameters(), self.decoder.parameters()), lr=lr)
        G_scheduler = optim.lr_scheduler.ReduceLROnPlateau(G_opt)
        D_opt = optim.Adam(chain(self.adversary.parameters(), self.critic.parameters()), lr=lr)
        # D_scheduler = optim.lr_scheduler.ReduceLROnPlateau(D_opt)

        # Set up the log directory
        with open(os.path.join(log_dir, "config.json"), "wt") as fout:
            fout.write(json.dumps({
                "model": self.model,
                "data_dim": self.data_dim,
                "seq_len": seq_len,
                "batch_size": batch_size,
                "dataset": dataset,
                "lr": lr,
                "log_dir": log_dir,
            }, indent=2, default=lambda o: str(o)))

        # Optimize the model
        history = []
        for epoch in range(1, epochs + 1):
            metrics = {
                "train.loss": [],
                "train.raw_acc": [],
                "train.mjpeg_acc": [],
                "train.adv_loss": [],
                "val.ssim": [],
                "val.psnr": [],
                "val.crop_acc": [],
                "val.scale_acc": [],
                "val.mjpeg_acc": [],
            }

            gc.collect()
            self.encoder.train()
            self.decoder.train()

            # Optimize critic-adversary
            if use_critic or use_adversary:
                iterator = tqdm(train, ncols=0)
                for frames in iterator:
                    frames, data = make_pair(frames, self.data_dim,
                                             use_bit_inverse=use_bit_inverse)
                    wm_frames = self.encoder(frames, data)
                    adv_loss = 0.0
                    if use_critic:
                        adv_loss += torch.mean(self.critic(frames) - self.critic(wm_frames))
                    if use_adversary:
                        adv_loss -= functional.binary_cross_entropy_with_logits(
                            self.decoder(self.adversary(wm_frames)), data)
                    D_opt.zero_grad()
                    adv_loss.backward()
                    D_opt.step()
                    for p in self.critic.parameters():
                        p.data.clamp_(-0.1, 0.1)

                    metrics["train.adv_loss"].append(adv_loss.item())
                    iterator.set_description("Adversary | %s" % np.mean(metrics["train.adv_loss"]))

            # Optimize encoder-decoder using critic-adversary
            if use_critic or use_adversary:
                iterator = tqdm(train, ncols=0)
                for frames in iterator:
                    frames, data = make_pair(frames, self.data_dim,
                                             use_bit_inverse=use_bit_inverse)
                    wm_frames = self.encoder(frames, data)
                    loss = 0.0
                    if use_critic:
                        critic_loss = torch.mean(self.critic(wm_frames))
                        loss += 0.1 * critic_loss
                    if use_adversary:
                        adversary_loss = functional.binary_cross_entropy_with_logits(
                            self.decoder(self.adversary(wm_frames)), data)
                        loss += 0.1 * adversary_loss
                    G_opt.zero_grad()
                    loss.backward()
                    G_opt.step()

            # Optimize encoder-decoder
            iterator = tqdm(train, ncols=0)
            for frames in iterator:
                frames, data = make_pair(frames, self.data_dim, use_bit_inverse=use_bit_inverse)

                wm_frames = self.encoder(frames, data)
                wm_raw_data = self.decoder(noise(wm_frames))
                wm_mjpeg_data = self.decoder(mjpeg(wm_frames))

                loss = 0.0
                loss += functional.binary_cross_entropy_with_logits(wm_raw_data, data)
                loss += functional.binary_cross_entropy_with_logits(wm_mjpeg_data, data)
                G_opt.zero_grad()
                loss.backward()
                G_opt.step()

                metrics["train.loss"].append(loss.item())
                metrics["train.raw_acc"].append(get_acc(data, wm_raw_data))
                metrics["train.mjpeg_acc"].append(get_acc(data, wm_mjpeg_data))
                iterator.set_description("%s | Loss %.3f | Raw %.3f | MJPEG %.3f" % (
                    epoch,
                    np.mean(metrics["train.loss"]),
                    np.mean(metrics["train.raw_acc"]),
                    np.mean(metrics["train.mjpeg_acc"]),
                ))

            # Validate
            gc.collect()
            self.encoder.eval()
            self.decoder.eval()
            iterator = tqdm(val, ncols=0)
            with torch.no_grad():
                for frames in iterator:
                    frames = frames.cuda()
                    data = torch.zeros((frames.size(0), self.data_dim)).random_(0, 2).cuda()

                    wm_frames = self.encoder(frames, data)
                    wm_crop_data = self.decoder(mjpeg(crop(wm_frames)))
                    wm_scale_data = self.decoder(mjpeg(scale(wm_frames)))
                    wm_mjpeg_data = self.decoder(mjpeg(wm_frames))

                    metrics["val.ssim"].append(
                        ssim(frames[:, :, 0, :, :], wm_frames[:, :, 0, :, :]).item())
                    metrics["val.psnr"].append(
                        psnr(frames[:, :, 0, :, :], wm_frames[:, :, 0, :, :]).item())
                    metrics["val.crop_acc"].append(get_acc(data, wm_crop_data))
                    metrics["val.scale_acc"].append(get_acc(data, wm_scale_data))
                    metrics["val.mjpeg_acc"].append(get_acc(data, wm_mjpeg_data))

                    iterator.set_description(
                        "%s | SSIM %.3f | PSNR %.3f | Crop %.3f | Scale %.3f | MJPEG %.3f" % (
                            epoch,
                            np.mean(metrics["val.ssim"]),
                            np.mean(metrics["val.psnr"]),
                            np.mean(metrics["val.crop_acc"]),
                            np.mean(metrics["val.scale_acc"]),
                            np.mean(metrics["val.mjpeg_acc"]),
                        )
                    )

            metrics = {
                k: round(np.mean(v), 3) if len(v) > 0 else "NaN"
                for k, v in metrics.items()
            }
            metrics["epoch"] = epoch
            history.append(metrics)
            pd.DataFrame(history).to_csv(
                os.path.join(log_dir, "metrics.tsv"), index=False, sep="\t")
            with open(os.path.join(log_dir, "metrics.json"), "wt") as fout:
                fout.write(json.dumps(metrics, indent=2, default=lambda o: str(o)))

            torch.save(self, os.path.join(log_dir, "model.pt"))
            G_scheduler.step(metrics["train.loss"])

        return history

    def save(self, path_to_model):
        torch.save(self, path_to_model)

    def load(path_to_model):
        return torch.load(path_to_model)

    def encode(self, video_in, data, video_out):
        assert len(data) == self.data_dim

        video_in = cv2.VideoCapture(video_in)
        width = int(video_in.get(cv2.CAP_PROP_FRAME_WIDTH))
        height = int(video_in.get(cv2.CAP_PROP_FRAME_HEIGHT))
        length = int(video_in.get(cv2.CAP_PROP_FRAME_COUNT))

        data = torch.FloatTensor([data]).cuda()
        video_out = cv2.VideoWriter(
            video_out, cv2.VideoWriter_fourcc(*'mp4v'), 20.0, (width, height))

        for i in tqdm(range(length)):
            ok, frame = video_in.read()
            frame = torch.FloatTensor([frame]) / 127.5 - 1.0      # (L, H, W, 3)
            frame = frame.permute(3, 0, 1, 2).unsqueeze(0).cuda()  # (1, 3, L, H, W)
            wm_frame = self.encoder(frame, data)                       # (1, 3, L, H, W)
            wm_frame = torch.clamp(wm_frame, min=-1.0, max=1.0)
            wm_frame = (
                (wm_frame[0, :, 0, :, :].permute(1, 2, 0) + 1.0) * 127.5
            ).detach().cpu().numpy().astype("uint8")
            video_out.write(wm_frame)

        video_out.release()

    def decode(self, video_in):
        video_in = cv2.VideoCapture(video_in)
        # width = int(video_in.get(cv2.CAP_PROP_FRAME_WIDTH))
        # height = int(video_in.get(cv2.CAP_PROP_FRAME_HEIGHT))
        length = int(video_in.get(cv2.CAP_PROP_FRAME_COUNT))

        for i in tqdm(range(length)):
            ok, frame = video_in.read()
            frame = torch.FloatTensor([frame]) / 127.5 - 1.0      # (L, H, W, 3)
            frame = frame.permute(3, 0, 1, 2).unsqueeze(0).cuda()  # (1, 3, L, H, W)
            data = self.decoder(frame)[0].detach().cpu().numpy()
            yield data


import nltk
from bs4 import BeautifulSoup
from bs4 import Comment
import re
import json

html_file = ".\\data_processing\\html.json"
doc_file = ".\\data_processing\\doc.json"
pages = json.load(open(html_file))
documents = {}
def clean_text(text):
    clean = re.sub(r"""
               [-_#\t\n]+[ ]*
               """,
               " ", text, flags=re.VERBOSE)
    return clean
for page, html in pages.items():
    print(page)
    soup = BeautifulSoup(html)
    body = soup.find("div", {"class": "aui-w75 portlet-column portlet-column-last"})
    if body and body.find("p"):
        for s in body.select('script'):
            s.extract()
        for s in body.select('style'):
            s.extract()
        for s in body(text=lambda text: isinstance(text, Comment)):
            s.extract()
        #texts = body.findAll("p")#, text=True)
        texts = [tag.getText() for tag in body.find_all('p')]#["p", re.compile('^h[0-9]*$')]
        text = " ".join((" ".join([t for t in texts if t != '\n'])).split())
        documents[page] = clean_text(text)
        json.dump(documents, open(doc_file, 'w+'))
json.dump(documents, open(doc_file, 'w+'))


from django.db import models
from basedata.models import SoftDeletionModel
from django.utils import timezone

class CustomerReceipt(SoftDeletionModel):
    '''
        This Gentleman is the extension of the Payment Model
        It does not generate a Journal Entry for the Payment Model takes care of that
        There are no views to generate a receipt but it is just created automatcally when\
        the save method is called on the Payment Model
        Most of its field are determined by realted fields. The Views expose the Receipt only
        as Detail and List View and a receipt shall not be deleted hence it inherits the SoftDeletionModel
        This Model is blocked from deletion and putting active to false can only delete the Model and such
        privillege is extended to admin

    '''
    sales_rep = models.ForeignKey(
                            'invoicing.SalesRepresentative',
                            null=True,
                            blank=True,
                            db_index=True,
                            related_name= 'soldreceipts',
                            on_delete=models.SET_NULL
                        )
    customer = models.ForeignKey(
                            'invoicing.Customer',
                            null=True,
                            blank=True,
                            db_index=True,
                            related_name = 'receipts',
                            on_delete=models.SET_NULL,
                        )
    receipt_number = models.CharField(max_length=255, null=True, default=None)
    created_date = models.DateTimeField(auto_now_add=True, db_index=True)
    comment = models.CharField(max_length=511, null=True, blank=True, default='')
    payment_method = models.CharField(max_length=500)
    has_finished = models.BooleanField(default=False, db_index=True)
    has_error = models.BooleanField(default=False, db_index=True)
    amount_paid = models.DecimalField(max_digits=16, decimal_places=2)
    amount_tendered = models.DecimalField(max_digits=16, decimal_places=2)

    def __str__(self):
        return "Receipt #" + str(self.receipt_number)

    def save(self, *args, **kwargs):
        if not self.receipt_number:
           prefix = 'REC{}'.format(timezone.now().strftime('%y%m%d'))
           prev_instances = self.__class__.objects.filter(receipt_number__contains=prefix)
           if prev_instances.exists():
              last_instance_id = prev_instances.last().receipt_number[-4:]
              self.receipt_number = prefix+'{0:04d}'.format(int(last_instance_id)+1)
           else:
               self.receipt_number = prefix+'{0:04d}'.format(1)
        super(CustomerReceipt, self).save(*args, **kwargs)

    @property
    def paid_as_of_date(self):
        return self.payments.invoice.total_paid

    @property
    def balance_as_of_date(self):
        return self.payments.invoice.total_due

    @property
    def change(self):
        return self.payments.customer_change


import pandas as pd
from pandas import Series,DataFrame
import numpy as np
from datetime import datetime

start_stage = 8000
end_stage = 9000

#wadiz_title_url.csv   total
# 1001
data = pd.read_csv("/Users/shockingjmh/Dev/git/crawler/wadiz_title_url.csv")
data2 = data.iloc[start_stage:end_stage]['url']
#data2 = data['url']
one = data2.values.tolist()

from time import time
from selenium import webdriver
from selenium.webdriver.chrome.options import Options

for i in range(1) :
    chrome_options = Options()
    chrome_options.add_argument('--proxy-server=socks5://127.0.0.1:9150')
    prefs = {'profile.default_content_setting_values': {'cookies' : 2, 'images': 2, 'plugins' : 2, 'popups': 2, 'geolocation': 2, 'notifications' : 2, 'auto_select_certificate': 2, 'fullscreen' : 2, 'mouselock' : 2, 'mixed_script': 2, 'media_stream' : 2, 'media_stream_mic' : 2, 'media_stream_camera': 2, 'protocol_handlers' : 2, 'ppapi_broker' : 2, 'automatic_downloads': 2, 'midi_sysex' : 2, 'push_messaging' : 2, 'ssl_cert_decisions': 2, 'metro_switch_to_desktop' : 2, 'protected_media_identifier': 2, 'app_banner': 2, 'site_engagement' : 2, 'durable_storage' : 2}}
    chrome_options.add_experimental_option('prefs', prefs)

    driver = webdriver.Chrome(executable_path="/Users/shockingjmh/opt/anaconda3/envs/py8/bin/chromedriver") # 다운받은 크롬드라이버 위치

import time
import random
from multiprocessing import Pool, pool

def mul(url) :
    driver.get(url)

    wadiz_body=""
    wadiz_title=""
    wadiz_category=""
    wadiz_achievement_rate=""
    wadiz_total_amount=""
    wadiz_total_supporter=""
    wadiz_url=""

    try:
        #본문내용
        body = driver.find_elements_by_xpath(f'//*[@id="introdetails"]/div')
        for value in body:
            wadiz_body += value.text

        #제목
        title = driver.find_element_by_xpath(f'//*[@id="container"]/div[3]/h2/a')
        wadiz_title = title.text

        #카테고리
        category = driver.find_element_by_xpath(f'//*[@id="container"]/div[3]/p')
        wadiz_category = category.text

    #    achievement_rate = driver.find_element_by_xpath(f'//*[@id="container"]/div[6]/div/div[1]/div[1]/div[1]/div[1]/p[3]')
    #    wadiz_achievement_rate = achievement_rate.text

    #    total_amount = driver.find_element_by_xpath(f'//*[@id="container"]/div[6]/div/div[1]/div[1]/div[1]/div[1]/p[4]')
    #    wadiz_total_amount = total_amount.text

    #    total_supporter = driver.find_element_by_xpath(f'//*[@id="container"]/div[6]/div/div[1]/div[1]/div[1]/div[1]/p[5]')
    #    wadiz_total_supporter = total_supporter.text

        achievement_rate = driver.find_element_by_class_name("achievement-rate")
        wadiz_achievement_rate = achievement_rate.text

        total_amount = driver.find_element_by_class_name("total-amount")
        wadiz_total_amount = total_amount.text

        total_supporter = driver.find_element_by_class_name("total-supporter")
        wadiz_total_supporter = total_supporter.text

        wadiz_url = url

        step = int(url[url.rindex('_')+1:])

        if (step % 100) == 0 :
            time.sleep(600)
        elif (step % 50) == 0 :
            time.sleep(300)
        elif (step % 10) == 0 :
            time.sleep(60)
        elif (step % 5) == 0 :
            time.sleep(30)
        else :
            time.sleep(random.randint(1, 5) * 2)

    except:
        print("exception! : " + wadiz_url)

    return [str(wadiz_title), str(wadiz_category), str(wadiz_url), str(wadiz_achievement_rate), str(wadiz_total_amount), str(wadiz_total_supporter), str(wadiz_body)]

wadiz_list = []

import os
import pandas as pd
import numpy as np

if __name__ == '__main__':
    cpu_core = os.cpu_count()
    print("cpu core : " + str(cpu_core))

    pool = Pool(processes=cpu_core) #32 process use
    wadiz_list.append(pool.map(mul, one))
    pool.close()
    pool.join()
    driver.quit()

    df1 = pd.DataFrame(data=sum(wadiz_list,[]), columns=['title', 'category', 'url', 'achievement_rate', 'total_amount', 'total_supporter', 'body'])
    #df1.to_csv("wadiz_result_final"+str(datetime.now().strftime('%Y%m%d%H%M%S'))+".csv",mode='w',encoding='utf-8-sig')
    df1.to_csv("wadiz_result_final_"+str(start_stage)+"_"+str(end_stage)+"_"+str(datetime.now().strftime('%Y%m%d%H%M%S'))+".csv",mode='w',encoding='utf-8-sig')
    #df1.to_csv("wadiz_result_final_"+str(end_stage)+"_"+str(datetime.now().strftime('%Y%m%d%H%M%S'))+".csv",mode='w',encoding='utf-8-sig')


import pickle

from .head_transformation import HeadPlaneProjection
import numpy as np

"""
The following sklearn imports are definitely needed for successful compilation!
When the facial classifier model is unpickled, it requires the sklearn library to run.
"""

from sklearn import svm, metrics
import sklearn.metrics._pairwise_distances_reduction._datasets_pair
import sklearn.metrics._pairwise_distances_reduction._middle_term_computer

MODEL_PATH = "./data/ml_models/head/facial-gesture-classifier.bin"

GESTURE_CLASSES = ["resting", "open_mouth", "raise_eyebrows", "smiling", "fish_face"]

FACIAL_METRICS = [

    (("left-cheek-inner", "lip-left"), ("left-undereye", "mouth-top")),
    (("right-cheek-inner", "lip-right"),  ("right-undereye", "mouth-top")),

    (("lip-top", "lip-bottom"), ("mouth-top", "chin-centre")),

    (("lip-left", "lip-right"), ("left-undereye", "right-undereye")),

    (("lip-top-left", "lip-bottom-left"), ("mouth-top", "chin-centre")),
    (("lip-top-right", "lip-bottom-right"), ("mouth-top", "chin-centre")),

    (("left-eyebrow-lower", "left-eye-bottom"), ("nose-tip", "mouth-top")),
    (("right-eyebrow-lower", "right-eye-bottom"), ("nose-tip", "mouth-top")),

]

class _FacialGestureClassifier:

    def __init__(self):

        self.model = pickle.load(open(MODEL_PATH, "rb"))

        self.landmark_frame = None

    def get_metrics(self, projected_frame):

        use_z = True

        metric_count = len(FACIAL_METRICS)
        metrics = np.zeros(metric_count)

        for index in range(metric_count):

            metric, norm = FACIAL_METRICS[index]

            metric_val = projected_frame.get_distance_sq(metric[0], metric[1], use_z=use_z)
            norm_val = projected_frame.get_distance_sq(norm[0], norm[1], use_z=use_z)

            metrics[index] = metric_val / norm_val

        return metrics

    def consume(self, landmark_frame):

        self.landmark_frame = landmark_frame

    def evaluate(self):

        prediction_label = GESTURE_CLASSES[0]

        if self.landmark_frame is not None:

            head_transformation = HeadPlaneProjection(self.landmark_frame)

            plane_projected_frame = head_transformation.project_landmarks(self.landmark_frame)

            metrics = self.get_metrics(plane_projected_frame)

            class_prediction = self.model.predict(metrics.reshape(1, -1))[0]

            prediction_label = GESTURE_CLASSES[class_prediction]

        return prediction_label

FacialGestureClassifier = _FacialGestureClassifier()

"""
Create a singleton classifier that can consume new frames from HeadLandmarkDetector
but only perform classification if gestures are actually needed, in HeadPosition.

This is a workaround, since to optionally perform classification, we need to know
whether gesture primitives are needed, but we only have access to 'used_primitives'
in head_position class. This isn't the best imo, but it avoids modifications to
core code.
"""


#!/usr/bin/python3
# -*- coding: utf-8 -*-
# @IDE          :PyCharm
# @FileName     :DateEncoder
# @CreatTime    :2022/3/20 16:53
# @CreatUser    :DaneSun
# @Author       :孙智涵
import datetime
import json

class DateEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj,datetime.datetime):
            return obj.strftime("%Y-%m-%d %H:%M:%S")
        elif isinstance(obj,datetime.date):
            return obj.strftime("%Y-%m-%d")
        else:
            return json.JSONEncoder.default(self,obj)


#!/usr/bin/env python

from __future__ import print_function, division

import numpy as np
import h5py
from argparse import ArgumentParser
from glob import glob
from progressbar import ProgressBar

def PCA(data):
    # Calculate the eigendecomposition of the covariance matrix
    C = np.cov(data, rowvar=False)
    eival, eivec = np.linalg.eigh(C)

    # Normalize eigenvectors (unit length) and rescale eigenvalues
    norm = np.linalg.norm(eivec, axis=1)
    eivec = eivec[:,:] / norm[:,None]
    eival *= norm

    # Sort the eigenvalues/eigenvectors (largest to smallest)
    idx = np.argsort(eival)[::-1]
    eival = eival[idx]
    eivec = eivec[:,idx]

    # Transform the data to the new coordinate system
    d_transf = np.dot(data - np.mean(data, axis=0), eivec)

    # Returns the (eigenvalues, eigenvectors, transormed data)
    return eival, eivec, d_transf

def autocorr_1d(y, threshold=0.05):
    """
    Calculates the autocorrelation of a 1-dimensional
    signal, y.

    Inputs:
        y (array-like): 1-dimensional signal.

    Returns:
        Autocorrelation as a function of displacement,
        and an estimate of the autocorrelation time,
        based on the smallest displacement with a
        negative autocorrelation.

    From the StackOverflow answer by unutbu:
    <https://stackoverflow.com/a/14298647/1103939>
    """
    n = len(y)
    y0 = np.mean(y)
    sigma2 = np.var(y)
    dy = y - y0

    if sigma2 == 0:
        return np.ones(n), -1.0

    r = np.correlate(dy, dy, mode='full')[-n:]
    r /= (sigma2 * np.arange(n, 0, -1))

    idx = np.where(r < threshold)[0]

    if not len(idx):
        return r, -1.0

    tau = idx[0]
    return r, tau

def rel2abs_coords(ax, x, y):
    xlim = ax.get_xlim()
    ylim = ax.get_ylim()
    w = xlim[1] - xlim[0]
    h = ylim[1] - ylim[0]
    return (xlim[0] + x*w, ylim[0] + y*h)

def get_n_tau_min(dset):

    ln_like = dset[:,0]
    ln_prior = dset[:,1]
    theta = dset[:,2:]

    data = {'ln_like': ln_like, 'ln_prior': ln_prior, 'theta': theta}

    # Autocorrelation of log(prior) and log(likelihood)
    acorr_prior, tau_prior = autocorr_1d(
        data['ln_prior'],
        threshold=0.05)
    acorr_like, tau_like = autocorr_1d(
        data['ln_like'],
        threshold=0.05)

    # Autocorrelation of principal component coefficients
    eival, eivec, chain_transf = PCA(data['theta'])
    acorr, tau = autocorr_1d(
        chain_transf[:,0],
        threshold=0.05)

    tau_max = max([tau, tau_prior, tau_like])
    tau_max_idx = np.argmax([tau, tau_prior, tau_like])
    n_tau_min = acorr.size / tau_max

    return n_tau_min

def load_file(fname):
    with h5py.File(fname, 'r') as f:
        n_pix = len(f.keys())

        shape = f[[key for key in f.keys()][0]]['discrete-los'].shape[1:]

        shape = (n_pix,) + shape

        chains = np.empty(shape, dtype='f4')

        dtype = [
            ('nside', 'i4'),
            ('pix_idx', 'i4'),
            ('runtime_star', 'f4'),
            ('runtime_los', 'f4'),
            ('n_tau', 'f4'),
            ('reject_frac', 'f4')
        ]
        pix_info = np.empty(n_pix, dtype=dtype)

        for k,key in enumerate(f.keys()):
            # dset.shape = (chain, GR+best+sample, lnlike+lnprior+parameter)
            chains[k,:,:] = 0.01 * f[key]['discrete-los'][0,:,:]

            n,i = key.split()[1].split('-')
            pix_info[k]['nside'] = int(n)
            pix_info[k]['pix_idx'] = int(i)
            pix_info[k]['runtime_los'] = f[key]['discrete-los'].attrs['runtime']

            n_tau = get_n_tau_min(f[key]['discrete-los'][0,2:,:])
            pix_info[k]['n_tau'] = n_tau

    return chains, pix_info

def combine_files(in_fnames, out_fname):
    print('Loading bayestar output files ...')
    bar = ProgressBar(max_value=len(in_fnames))
    bar.update(0)

    chains, pix_info = [], []

    for k,fn in enumerate(in_fnames):
        c,p = load_file(fn)
        chains.append(c)
        pix_info.append(p)

        bar.update(k+1)

    print('Concatenating data ...')
    chains = np.concatenate(chains, axis=0)
    pix_info = np.hstack(pix_info)

    print('Writing combined output file ...')

    with h5py.File(out_fname, 'w') as f:
        f.create_dataset('discrete-los', data=chains, chunks=True, compression='gzip', compression_opts=3)
        f.create_dataset('pix_info', data=pix_info, chunks=True, compression='gzip', compression_opts=3)

def main():
    parser = ArgumentParser(description='Concatenate bayestar output files.',
                            add_help=True)
    parser.add_argument('-i', '--inputs', type=str, nargs='+',  help='Bayestar output files.')
    parser.add_argument('-o', '--output', type=str, help='Combined output filename.')
    args = parser.parse_args()

    fnames = []
    for fn in args.inputs:
        fnames += glob(fn)

    combine_files(fnames, args.output)

    return 0

if __name__ == '__main__':
    main()


import json
import os
from datetime import datetime, timedelta
import hashlib
import shutil
from logging import Logger
from glob import glob
import time

import pytz
import piexif
from pymediainfo import MediaInfo
import ffmpeg
from PIL import Image, UnidentifiedImageError
import pyheif

MEDIA_TYPES = {
    'image': ['.jpg', '.jpeg', '.png', '.heic'],
    'video': ['.mp4', '.mpg', '.mov', '.m4v', '.mts', '.mkv'],
}

class OrganizePictures:
    FILENAME_DATE_FORMAT = "%Y-%m-%d_%H'%M'%S"
    ENCODED_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
    PIEXIF_DATE_FORMAT = '%Y:%m:%d %H:%M:%S'
    VIDEO_DATE_FORMATS = {
        "default": "%Z %Y-%m-%d %H:%M:%S",
        ".mkv": "%Z %Y-%m-%dT%H:%M:%SZ",
    }
    IMG_CONVERT_EXTS = ['.heic']
    IMG_CHANGE_EXTS = ['.jpeg']
    VID_CONVERT_EXTS = ['.mpg', '.mov', '.m4v', '.mts', '.mkv']
    PREFERRED_IMAGE_EXT = '.jpg'
    PREFERRED_VIDEO_EXT = '.mp4'

    # pylint: disable=too-many-arguments
    def __init__(
            self,
            logger: Logger,
            source_directory: str,
            destination_directory: str,
            extensions: list = None,
            media_type: str = None,
            dry_run: bool = False,
            cleanup: bool = False,
            sub_dirs: bool = True,
            verbose: bool = False,
    ):
        self.logger = logger
        self.source_dir = source_directory
        self.dest_dir = destination_directory
        self.media_type = media_type
        self.dry_run = dry_run
        self.cleanup = cleanup
        self.sub_dirs = sub_dirs
        self.verbose = verbose

        self.results = {"moved": 0, "duplicate": 0, "failed": 0, "deleted": 0}

        self.extensions = extensions
        if self.extensions is None:
            if media_type is not None:
                self.extensions = MEDIA_TYPES.get(media_type)
            else:
                self.extensions = []
                for exts in MEDIA_TYPES.values():
                    self.extensions += exts

    @staticmethod
    def _get_file_ext(file):
        _, ext = os.path.splitext(os.path.basename(file))
        return ext

    def _get_json_file(self, _file):
        """Get the json file name for the given file"""
        if "(" in _file and ")" in _file:
            ext = self._get_file_ext(_file)
            start = _file.find("(")
            end = _file.find(")")
            base_file = _file[:start]
            file_num = _file[start + 1:end]
            _file = f"{base_file}{ext}({file_num})"
        return f"{_file}.json"

    @staticmethod
    def _load_json_file(json_file: str):
        parsed_json = None
        if os.path.isfile(json_file):
            with open(json_file, encoding="utf-8") as user_file:
                parsed_json = json.load(user_file)
        return parsed_json

    @staticmethod
    def _md5(fname):
        hash_md5 = hashlib.md5()
        with open(fname, "rb") as file_handle:
            for chunk in iter(lambda: file_handle.read(4096), b""):
                hash_md5.update(chunk)
        return hash_md5.hexdigest()

    def _get_files(self, path: str):
        files = []
        paths = glob(f"{path}/*")
        for file in paths:
            if os.path.isfile(file):
                ext = self._get_file_ext(file)
                ext_lower = ext.lower()
                if ext_lower in self.extensions:
                    add = True
                    if self.media_type is not None and ext_lower not in MEDIA_TYPES.get(self.media_type):
                        add = False
                    if add:
                        files.append(file)
            elif os.path.isdir(file):
                files += self._get_files(file)
        return sorted(files)

    def _get_date_taken(self, _file: str) -> datetime:
        date_time_obj = None
        ext = self._get_file_ext(_file).lower()
        if ext in MEDIA_TYPES.get('video'):
            media_info = MediaInfo.parse(_file)
            for track in media_info.tracks:
                if track.track_type in ['Video', 'General']:
                    if track.encoded_date is not None:
                        date_time_obj = datetime.strptime(
                            track.encoded_date, self.VIDEO_DATE_FORMATS.get(
                                ext,
                                self.VIDEO_DATE_FORMATS.get("default")
                            )
                        )
                        _fromtz = pytz.timezone(track.encoded_date[0:track.encoded_date.find(" ")])
                        _totz = pytz.timezone('US/Mountain')
                        date_time_obj = datetime.astimezone(date_time_obj.replace(tzinfo=_fromtz), _totz)
                        break
                    if track.recorded_date is not None:
                        date_time_obj = datetime.strptime(track.recorded_date, "%Y-%m-%d %H:%M:%S%z")
                        break
        elif ext in MEDIA_TYPES.get('image'):
            json_file = self._get_json_file(_file)
            if os.path.isfile(json_file):
                date_time_obj = datetime.fromtimestamp(
                    int(self._load_json_file(json_file).get('photoTakenTime', {}).get('timestamp')))
            else:
                try:
                    exif_dict = piexif.load(_file)
                    for tag, value in exif_dict['Exif'].items():
                        if "DateTimeDigitized" in piexif.TAGS['Exif'][tag]["name"]:
                            date_time_obj = datetime.strptime(value.decode(), self.PIEXIF_DATE_FORMAT)
                            break
                except piexif._exceptions.InvalidImageDataError:
                    self.logger.error(f'Unable to get exif data for file: {_file}')
        # if other dates are not found, use the file modified date
        if date_time_obj is None:
            date_time_obj = datetime.strptime(time.ctime(os.path.getmtime(_file)), '%c')

        return date_time_obj

    def _convert_video(self, _file: str, _new_file: str):
        converted = False
        if os.path.isfile(_new_file):
            self.logger.info(f"Skipping conversion of \"{_file}\" to \"{_new_file}\" as it already exists")
            converted = True
        self.logger.info(f"Converting \"{_file}\" to \"{_new_file}\"")
        stream = ffmpeg.input(_file)
        stream = ffmpeg.output(
            stream,
            _new_file,
            acodec="aac",
            vcodec="h264",
            map_metadata=0,
            metadata=f"comment=Converted {_file} to {_new_file}",
            loglevel="verbose" if self.verbose else "quiet"
        )
        _, err = ffmpeg.run(stream)
        if err is None:
            self.results['moved'] += 1
            self.logger.info(f"Successfully converted \"{_file}\" to \"{_new_file}\"")
            converted = True
        else:
            self.logger.error(f"Failed to convert \"{_file}\" to \"{_new_file}\"")
        return converted

    def _convert_image(self, source_file: str, dest_file: str):
        self.logger.debug(f"Converting file:\n\tSource: {source_file}\n\tDestination: {dest_file}")
        image_ext = self._get_file_ext(source_file).lower()
        method = "pillow"
        try:
            image = Image.open(source_file)
            image.convert('RGB').save(dest_file)
        except UnidentifiedImageError as pilexc:
            if image_ext != ".heic":
                return False
            method = "pyheif"
            self.logger.error(f"Failed first conversion attempt via {method}: {source_file}\n{pilexc}")
            heif_file = pyheif.read(source_file)
            image = Image.frombytes(
                heif_file.mode,
                heif_file.size,
                heif_file.data,
                "raw",
                heif_file.mode,
                heif_file.stride,
            )
            image.save(dest_file)
        except Exception as exc:
            self.logger.error(f"Failed second conversion attempt via {method}: {source_file}\n{exc}")
            return False
        self.results['moved'] += 1
        self.logger.debug(
            f"Successfully converted file via {method}:\n\tSource: {source_file}\n\tDestination: {dest_file}"
        )
        return True

    def _media_file_matches(self, source_file: str, dest_file: str):
        matches = False
        # assume source file exists, ensure dest file exists
        if os.path.isfile(dest_file):
            if self._md5(source_file) == self._md5(dest_file):
                self.results['duplicate'] += 1
                matches = True
                self.logger.debug(f"""Source file matches existing destination file:
                            Source: {source_file}
                            Destination: {dest_file}""")
            else:
                self.logger.debug(f"""Source file does not match existing destination file:
                            Source: {source_file}
                            Destination: {dest_file}""")
                source_file_ext = self._get_file_ext(source_file).lower()
                if source_file_ext in MEDIA_TYPES.get('video'):
                    self.logger.debug(f"Checking if video file has already been converted")
                    _media_info = MediaInfo.parse(dest_file)
                    for _track in _media_info.tracks:
                        if hasattr(_track, 'comment') and _track.comment is not None:
                            if "Converted" in _track.comment:
                                self.results['duplicate'] += 1
                                self.logger.info(f"Video file already converted: {source_file}")
                                self.logger.debug(f"\t{_track.comment}")
                                matches = True
                                break
        return matches

    @staticmethod
    def _find_image_animation(_file: str, _ext: str):
        for ext in MEDIA_TYPES.get('video'):
            image_animation = _file.replace(_ext, ext)
            if os.path.isfile(image_animation):
                return image_animation
            image_animation = _file.replace(_ext, ext.upper())
            if os.path.isfile(image_animation):
                return image_animation
        return None

    def _update_file_date(self, _file, _date: datetime):
        try:
            new_date = _date.strftime(self.PIEXIF_DATE_FORMAT)
            image = Image.open(_file)

            exif_dict = piexif.load(_file)
            if exif_dict["Exif"][piexif.ExifIFD.DateTimeDigitized] != new_date:
                self.logger.debug(f"Updating date digitized for {_file}")
                exif_dict["Exif"][piexif.ExifIFD.DateTimeDigitized] = new_date
                exif_bytes = piexif.dump(exif_dict)
                image.save(_file, exif=exif_bytes)
            else:
                self.logger.debug(f"Exif date already matches for {_file}")
        except Exception as exc:
            self.logger.error(f"Failed to update file date for {_file}: {exc}")

    def _get_new_fileinfo(self, _file: str, _date: datetime):
        _ext = self._get_file_ext(_file)
        _ext_lower = _ext.lower()
        _year = _date.strftime("%Y")
        _month = _date.strftime("%b")
        _dir = self.dest_dir
        if self.sub_dirs:
            _dir += f"/{_year}/{_month}"

        _filename = f"{_date.strftime(self.FILENAME_DATE_FORMAT)}{_ext_lower}"
        _new_file_info = {
            'ext': _ext_lower,
            'dir': _dir,
            'filename': _filename,
            'path': f"{_dir}/{_filename}",
            'date_encoded': _date
        }
        json_file = self._get_json_file(_file)
        if os.path.isfile(json_file):
            _new_file_info['json_file'] = json_file
            _new_file_info['new_json_file'] = f"{_dir}/{self._get_json_file(_filename)}"

        image_animation = self._find_image_animation(_file, _ext)
        if self.media_type == 'image' and image_animation is not None:
            _new_file_info['animation_source'] = image_animation
            _new_file_info['animation_dest'] = f"{_dir}/{_filename.replace(_ext_lower, self.PREFERRED_VIDEO_EXT)}"

        if _ext_lower in self.IMG_CONVERT_EXTS:
            _new_file_info['convert_path'] = f"{_dir}/{_filename.replace(_ext_lower, self.PREFERRED_IMAGE_EXT)}"
        if _ext_lower in self.VID_CONVERT_EXTS:
            _new_file_info['path'] = f"{_dir}/{_filename.replace(_ext_lower, self.PREFERRED_VIDEO_EXT)}"
        if _ext_lower in self.IMG_CHANGE_EXTS:
            _new_file_info['path'] = f"{_dir}/{_filename.replace(_ext_lower, self.PREFERRED_IMAGE_EXT)}"

        if not os.path.isdir(_new_file_info['dir']):
            self.logger.debug(f"Destination path does not exist, creating: {_new_file_info['dir']}")
            os.makedirs(_new_file_info['dir'])
        if not os.path.exists(_new_file_info['path']):
            return _new_file_info

        self.logger.debug(f"Destination file already exists: {_new_file_info['path']}")
        if self._media_file_matches(_file, _new_file_info['path']):
            _new_file_info['duplicate'] = True
            return _new_file_info
        else:
            # increment 1 second and try again
            new_dt = _date + timedelta(seconds=1)
            # Need to work out issues with this before enabling
            # self._update_file_date(_file, new_dt)
            return self._get_new_fileinfo(_file, new_dt)

    def run(self):
        files = self._get_files(self.source_dir)
        for index, media_file in enumerate(files, start=1):
            self.logger.info(
                f"Processing file {index} / {len(files)}:\n\t{media_file}"
            )
            cleanup_files = []
            date_taken = self._get_date_taken(media_file)
            if date_taken is not None:
                new_file_info = self._get_new_fileinfo(media_file, date_taken)
                if not new_file_info.get('duplicate'):
                    if new_file_info['ext'] in MEDIA_TYPES.get('video'):
                        if self._convert_video(media_file, new_file_info['path']):
                            cleanup_files.append(media_file)
                    else:
                        try:
                            self.logger.info(
                                f"Moving file:\n\tSource: {media_file}\n\tDestination: {new_file_info['path']}"
                            )
                            if new_file_info.get('convert_path') is not None:
                                if not self._convert_image(media_file, new_file_info['convert_path']):
                                    self.logger.error(f"Failed to convert image: {media_file}")
                                    continue
                            # Need to work out issues with this before enabling
                            # self._update_file_date(media_file, new_file_info['date_encoded'])
                            shutil.copyfile(media_file, new_file_info['path'])
                            self.results['moved'] += 1
                            cleanup_files.append(media_file)
                            if "animation_source" in new_file_info:
                                if self._media_file_matches(
                                        new_file_info['animation_source'],
                                        new_file_info['animation_dest']
                                ) or self._convert_video(
                                        new_file_info['animation_source'],
                                        new_file_info['animation_dest']
                                ):
                                    cleanup_files.append(new_file_info['animation_source'])
                        except shutil.Error as exc:
                            self.results['failed'] += 1
                            self.logger.error(f"Failed to move file: {media_file}\n{exc}")

                    if new_file_info.get('json_file') is not None:
                        self.logger.info(
                            f"Moving file:\n\tSource: {new_file_info.get('json_file')}\n\t"
                            f"Destination: {new_file_info.get('new_json_file')}")
                        shutil.copyfile(new_file_info['json_file'], new_file_info['new_json_file'])
                        self.results['moved'] += 1
                        cleanup_files.append(new_file_info['json_file'])
                else:
                    # file is already moved
                    self.logger.info(f"File already moved: {media_file}")
                    cleanup_files.append(media_file)
                    if new_file_info.get('json_file'):
                        cleanup_files.append(new_file_info.get('json_file'))
                    if new_file_info.get('animation_source'):
                        cleanup_files.append(new_file_info.get('animation_source'))

            if cleanup_files and self.cleanup:
                for cleanup_file in cleanup_files:
                    self.results['deleted'] += 1
                    self.logger.info(f"Deleting file: {cleanup_file}")
                    os.remove(cleanup_file)

        return self.results


"""
"""

import numpy as np
from itertools import izip, chain

from gnumpy import zeros as gzeros

from misc import idnty
from stack import Stack
import chopmunk as munk

class DAE(Stack):
    def __init__(self, ind, schedule):
        super(DAE, self).__init__(ind=ind, schedule=schedule)
        self.encoder = None

    def __repr__(self):
        if self.encoder is None:
            rep = super(DAE, self).__repr__()
        else:
            rep = "|".join([str(l) for l in self.encoder])
            rep += "|"
            rep += "|".join([str(l) for l in self.decoder])
        return rep

    def pretrain(self, schedule):
        super(DAE, self).pretrain(schedule=schedule)

        p = self.params.as_numpy_array()

        pretrained = schedule["pretrained"]

        # How many parameters in the unrolled model?
        _dec = []
        _enc = [0]
        self.psize = 0
        for layer in self:
            _enc.append(layer.shape[0]*layer.shape[1] + layer.shape[1])
            _dec.append(layer.shape[0]*layer.shape[1] + layer.shape[0])
            self.psize += _enc[-1] + _dec[-1]
        self.enc = np.cumsum(_enc)
        _dec.append(0)
        _dec.reverse()
        self.dec = np.cumsum(_dec) + self.enc[-1]

        # Build up encoder and decoder
        self.encoder = []
        self.params = gzeros(self.psize)
        for layer, (c1, c2) in izip(self, izip(self.enc[:-1], self.enc[1:])):
            self.encoder.append(layer)
            self.params[c1:c2] = p[c1:c2]
            layer.p = self.params[c1:c2]
        self.decoder = []
        for layer, (c1, c2) in izip(self[-1::-1], izip(self.dec[:-1], self.dec[1:])):
            l = layer.transpose(self.params[c1:c2])
            if pretrained:
                l.p[:l.m_end] = layer.p[:layer.m_end].reshape(layer.shape).T.ravel()
            self.decoder.append(l)

        # Fix missing activations of decoder
        for i, layer in enumerate(self[-2::-1]):
            self.decoder[i].activ = layer.activ
        self.decoder[-1].activ = idnty

        msg = {"msg": "DAE unrolled: %s"%self}
        munk.taggify(self.logging, "pretty").send(msg)

    def score(self, params, inputs, targets, **kwargs):
        data = inputs
        for layer, (c1, c2) in izip(self.encoder, izip(self.enc[:-1], self.enc[1:])):
            data = layer.fward(self.params[c1:c2], data)

        # possible spot for semi supervision?

        for layer, (c1, c2) in izip(self.decoder, izip(self.dec[:-1], self.dec[1:])):
            data = layer.fward(self.params[c1:c2], data)

        return self._score(data, inputs, **kwargs)

    def evaluate_score(self, params, inputs, targets, **kwargs):
        data = inputs
        for layer, (c1, c2) in izip(self.encoder, izip(self.enc[:-1], self.enc[1:])):
            data = layer.fward(self.params[c1:c2], data)

        # possible spot for semi supervision?

        for layer, (c1, c2) in izip(self.decoder, izip(self.dec[:-1], self.dec[1:])):
            data = layer.fward(self.params[c1:c2], data)

        return self._eval_score(data, inputs, **kwargs)

    def fward(self, inputs, **kwargs):
        data = inputs
        for layer, (c1, c2) in izip(self.encoder, izip(self.enc[:-1], self.enc[1:])):
            data = layer.fward(self.params[c1:c2], data)
        return data

    def grad(self, params, inputs, targets, **kwargs):
        data = inputs
        for layer, (c1, c2) in izip(self.encoder, izip(self.enc[:-1], self.enc[1:])):
            data = layer.fprop(self.params[c1:c2], data)

        # possible spot for semisupervision?

        for layer, (c1, c2) in izip(self.decoder, izip(self.dec[:-1], self.dec[1:])):
            data = layer.fprop(self.params[c1:c2], data)

        _, delta = self._score(data, inputs, error=True)

        g = gzeros(self.psize)
        for layer, (c1, c2) in izip(self.decoder[::-1], izip(self.dec[-2::-1], self.dec[:0:-1])):
            delta = layer.bprop(params=params[c1:c2], grad=g[c1:c2], delta=delta)

        # in case: fuse in gradient from semisupervision

        for layer, (c1, c2) in izip(self.encoder[::-1], izip(self.enc[-2::-1], self.enc[:0:-1])):
            delta = layer.bprop(params=params[c1:c2], grad=g[c1:c2], delta=delta)
        return g

    def _fward(self, data):
        for layer in self.encoder:
            data = layer._fward(data)
        return data


import torch
import MinkowskiEngine as ME
import math
from abc import *

class BaseMETask(metaclass=ABCMeta):
    def __init__(self, cfg, log):
        self.cfg = cfg
        self.log = log
        self.log.info('Building %s task ...' % cfg.task_type)
        self.model = self._init_model()
        self.log.info('Parameters in %s task : %d' %
                      (cfg.task_type, self._count_model_params(self.model)))
        self.task_state = 'train'
        self.current_epoch = 0
        self.num_devices = 0
        self.devices = 0
        self.master_device = None

    @abstractmethod
    def _init_model(self):
        raise NotImplementedError('You must implement _init_model method')

    def train(self):
        self.task_state = 'train'
        self.model.train()

    def eval(self):
        self.task_state = 'eval'
        self.model.eval()

    def cuda(self):
        self.num_devices = torch.cuda.device_count()
        self.devices = list(range(self.num_devices))
        self.master_device = self.devices[0]
        self.model = self.model.to(self.master_device)
        self.model = ME.MinkowskiSyncBatchNorm.convert_sync_batchnorm(
            self.model)

    def load(self, load_dir):
        self.log.info('Loading task state from %s ...' % load_dir)
        state = torch.load(load_dir)
        if 'model' in self.cfg.resume_items:
            self.model.load_state_dict(state['model_state_dict'], strict=False)
        if 'epoch' in self.cfg.resume_items:
            self.current_epoch = state['epoch']
        return state

    def save(self, save_dir, **kwargs):
        state = dict(kwargs)
        state['model_state_dict'] = self.model.state_dict()
        state['epoch'] = self.current_epoch
        if not self.cfg.debug:
            self.log.info('Saving task state to %s ...' % save_dir)
            torch.save(state, save_dir)

    def model_params(self):
        return self.model.parameters()

    def _count_model_params(self, model):
        return sum(p.numel() for p in model.parameters())

    @abstractmethod
    def _train_step(self, meta_data, batch_data, **kwargs):
        raise NotImplementedError('You must implement _train_step method')

    @abstractmethod
    def _eval_step(self, meta_data, batch_data, **kwargs):
        raise NotImplementedError('You must implement _eval_step method')

    def step(self, meta_data, batch_data, **kwargs):

        bs_per_device = math.ceil(
            len(batch_data['pcd_feats'])/self.num_devices)

        if self.task_state == 'eval':
            bs_per_device = len(batch_data['pcd_feats'])

        pcd_on_devices = []
        raw_pcd_on_devices = []

        for i in range(self.num_devices if self.task_state == 'train' else 1):
            coords = batch_data['pcd_coords'][i*bs_per_device:min(
                (i+1)*bs_per_device, len(batch_data['pcd_coords']))]
            coords = ME.utils.batched_coordinates(coords)
            feats = batch_data['pcd_feats'][i*bs_per_device:min(
                (i+1)*bs_per_device, len(batch_data['pcd_feats']))]
            feats = torch.cat(feats, dim=0)
            raw_pcd = torch.stack(batch_data['pcd'][i*bs_per_device:min(
                (i+1)*bs_per_device, len(batch_data['pcd']))], dim=0)

            with torch.cuda.device(self.devices[i]):
                pcd_on_devices.append(ME.SparseTensor(
                    feats.to(self.devices[i]), coordinates=coords.to(self.devices[i])))
                raw_pcd_on_devices.append(raw_pcd.to(self.devices[i]))

        batch_data['pcd'] = pcd_on_devices
        batch_data['raw_pcd'] = raw_pcd_on_devices
        del batch_data['pcd_feats']
        del batch_data['pcd_coords']

        if self.task_state == 'train':
            return self._train_step(meta_data, batch_data, **kwargs)
        elif self.task_state == 'eval':
            return self._eval_step(meta_data, batch_data, **kwargs)
        else:
            raise Exception('Unsupported task state: %s' % self.task_state)

    def before_epoch(self, epoch):
        self.current_epoch = epoch

    def after_epoch(self, epoch):
        self.current_epoch = epoch


PRODUCTS = {
    'iphone': {
        'name': 'iphone 5S',
        'category': 'Phones',
        'price': 699,

    },
    'galaxy':{
        'name': 'Samsung Galaxy 5',
        'category': 'Phones',
        'price': 649,
    },
    'ipad-air':{
        'name': 'iPad Air',
        'category': 'Tablets',
        'price': 649,
    },
    'ipad-mini':{
        'name': 'iPad Mini',
        'category': 'Tablets',
        'price': 549
    },
    'aaaa':{
        'name': 'aaaa-aaaa',
        'category': 'bbbbbbb',
        'price': 549
    }
}


class Solution:
    def matrixReshape(self, mat: List[List[int]], r: int, c: int) -> List[List[int]]:
        m = len(mat)
        n = len(mat[0])
        if m * n != r * c:
            return mat
        ans = [[] for i in range(r)]
        for i in range(m):
            for j in range(n):
                val = mat[i][j]
                idx = (i * n) + j
                ans[idx // c].append(val)
        return ans


# import summary
#
# file_name = "output/donasi/Oxygen for Indonesia_donor"
# summ = summary.Summary()
# summ.summary_donor(file_name)

import sys
import time

for i in range(10):
    sys.stdout.write(f"\r===== Scrolling count: {i} ({i+2} days) =====")
    sys.stdout.flush()
    if i == 9:
        sys.stdout.write(f"\nend")
        sys.stdout.flush()
        break
    if i == 2:
        print("\nbaris 2")
    time.sleep(1)
print("\nbaris terakhir")


import random
import time
import sys

no_plot=False

try:
    import matplotlib.pyplot as plt
    import matplotlib.colors

    c= []
    bad_colors = ['whitesmoke','floralwhite','lightgoldenrodyellow','aliceblue','plum','w','lightgray','seashell','navajowhite','ivory','lemonchiffon','ghostwhite','mediumturquiose','mediumaquamarine','deeppink','white','dimgray','lightgrey','beige','honeydew','lavender','snow','linen','cornsilk','lightyellow','mintcream','lightcyan']

    for name in matplotlib.colors.cnames.keys():
        if name in bad_colors:
            continue
        c.append(name)
except :
    no_plot=True
    print('No matplotlib is detected, won\'t able to plot')

#reading config file

filename='config'
config = {}
#params list will contain all the parameters info
params = []
with open(filename,'r') as file:
    for line in file:
        if line=='\n' or line=='' or line[0]=='#' :
            continue
        if not '=' in line:
            print(' \'=\' assignment operator is not found in config file\nplease format the config file properly')
            sys.exit(0)
        k,v = line.split('=')
        rm = ['\n','\t',' ']
        for i in rm:
            k = k.strip(i)
            v = v.strip(i)
        params.append(k)
        config[k] = v
file_vectors = config[params[0]]
file_candid_vectors = config[params[1]]
output_file = config[params[2]]
min_cluster_size = int(config[params[3]])
mx_num_of_sub_clusters = int(config[params[4]])

if len(config)>=7:
    mean_verbose = bool(config[params[5]].title()=='True')
    plot_verbose= bool(config[params[6]].title()=='True')
else:
    mean_verbose=False
    plot_verbose=False
if no_plot or ((mean_verbose==True or plot_verbose==True) and mx_num_of_sub_clusters>len(c)) :
    print('Mean and Cluster plotting cannot be plotted')
    mean_verbose=False
    plot_verbose=False

if len(config)>=8:
    no_of_times = int(config[params[7]])
else:
    no_of_times = 1000
if len(config)>=9:
    init_times = int(config[params[8]])
else:
    init_times = 1
#finds the distance between two vectors

def distance(a,b,sqr_or_not=False):
    dist=0
    for i in range(len(a)):
        dist+=(a[i]-b[i])**2
    if sqr_or_not==True:
        return dist
    return dist**(1/2)

#initializes mean for

def means_init(k,vectors):
    means = [vectors[random.randint(0,len(vectors)-1)]]
    means_len = 1
    while means_len!=k:
        mx_dist_for_mean = 0
        idx=0
        for i in range(len(vectors)):
            mn_dist = distance(vectors[i],means[0],True)
            for j in range(1,means_len):
                dist = distance(vectors[i],means[j],True)
                if dist < mn_dist:
                    mn_dist =dist
            if mn_dist > mx_dist_for_mean:
                mx_dist_for_mean = mn_dist
                idx = i
        means.append(vectors[idx])
        means_len+=1
    return means

#finds the nearest vector for candid_vector, from given vectors

def find_nearest(vectors,candid_vector):
    if len(vectors)==1:
        return vectors
    dist = []
    for i in range(len(vectors)):
        dist.append(distance(vectors[i],candid_vector))
    nearest_vecs = []
    mn = min(dist)
    for i in range(len(vectors)):
        if mn == dist[i]:
            nearest_vecs.append(vectors[i])
    return nearest_vecs

def plot_means(means):
    for i in range(len(means)):
        plt.scatter(means[i][0],means[i][1],c=c[i])

class cluster:

    def __init__(self,vectors,mean,level=0,cluster_no=0):
        self.level = level
        self.cluster_no = cluster_no
        self.vectors = vectors
        self.no_vectors = len(self.vectors)
        self.vec_length = len(self.vectors[0])
        self.mean = mean
        self.no_of_sub_clusters = 0
        self.sub_clusters = []
        self.end = False
        if self.no_vectors < min_cluster_size:
            self.end=True

    def Recursive_search(self,candid_vector,ans):
        # cluster.plot_cluster(self)
        if self.end==True:
            ans.extend(find_nearest(self.vectors,candid_vector))
            return
        dist = []
        for i in range(self.no_of_sub_clusters):
            dist.append(distance(self.sub_clusters[i].mean,candid_vector))
        mn_dist = min(dist)
        for i in range(len(dist)):
            if mn_dist==dist[i]:
                cluster.Recursive_search(self.sub_clusters[i],candid_vector,ans)

    def search(self,candid_vector):
        ans = []
        self.Recursive_search(candid_vector,ans)
        return find_nearest(ans,candid_vector)

    def cost(self,means):
        J=0
        for i in range(self.no_of_sub_clusters):
            for j in range(self.no_vectors):
                J += distance(means[i],vectors[j])
        return J
    def K_Mean(self):
        global min_cluster_size
        global no_of_times
        global init_times
        global mean_verbose
        global plot_verbose

        vec_length = self.vec_length
        no_vectors = self.no_vectors

        k=mx_num_of_sub_clusters
        final_means = []
        final_centroids = []
        flg = False
        mn=0

        for loop in range(init_times):
            means = means_init(k,self.vectors)
            if mean_verbose==True:
                plot_means(means)
                plt.xlabel(str(self.level) + ' initial')
                plt.show()
            prev_means = means
            centroids = [0 for i in range(no_vectors)]
            for t in range(no_of_times):
                if mean_verbose==True:
                    plot_means(means)
                new_means=[[0 for j in range(vec_length)] for i in range(k)]

                # print(t,'th iteration:---------------------')
                # print(means)
                cnt = [0 for i in range(k)]
                for i in range(len(self.vectors)):
                    mn_dist = distance(means[0],self.vectors[i])
                    mn_idx = 0
                    # print(vectors[i],means[0],mn_dist)
                    for j in range(1,k):
                        dist = distance(means[j],self.vectors[i])
                        if dist<mn_dist:
                            mn_idx=j
                            mn_dist=dist
                        # print(vectors[i],means[j],dist,mn_idx,mn_dist)
                    centroids[i]=mn_idx
                    cnt[mn_idx]+=1
                    for j in range(vec_length):
                        new_means[centroids[i]][j]+=self.vectors[i][j]

                for i in range(k):
                    for j in range(vec_length):
                        if cnt[i]!=0:
                            new_means[i][j]/=cnt[i]
                means=new_means
                if prev_means==means:
                    break
                prev_means=means
            if mean_verbose==True:
                plt.xlabel('level: ' + str(self.level) + ' cluster: ' + str(self.cluster_no))
                plt.show()
            J = self.cost(means)
            if flg==False:
                mn=J
                final_means = means
                final_centroids=centroids
                flg=True
            else:
                if J<mn:
                    mn=J
                    final_means = means
                    final_centroids = centroids

        sub_clus = {}
        for i in range(no_vectors):
            sub_clus.setdefault(final_centroids[i],[]).append(self.vectors[i])

        for k,v in sub_clus.items():
            self.sub_clusters.append(cluster(v,final_means[k],self.level+1,k))
        self.no_of_sub_clusters = len(self.sub_clusters)
        if self.no_of_sub_clusters==1:
            self.sub_clusters[0].end=True
        print('Possible clusters: ',self.no_of_sub_clusters,'level: ',self.level)
        print('Vectors: ',self.no_vectors)
        for k in range(self.no_of_sub_clusters):
            print(self.sub_clusters[k].no_vectors)

    def K_Mean_Clusters(self):
        if self.end==False:
            self.K_Mean()
            for i in range(self.no_of_sub_clusters):
                if len(self.sub_clusters[i].vectors)<min_cluster_size :
                    self.sub_clusters[i].end = True
                cluster.K_Mean_Clusters(self.sub_clusters[i])

    def plot_cluster(self,recursive = True):
        for i in range(self.no_of_sub_clusters):
            for j in range(self.sub_clusters[i].no_vectors):
                plt.scatter(self.sub_clusters[i].vectors[j][0],self.sub_clusters[i].vectors[j][1],c=c[i])
        if self.no_of_sub_clusters!=0:
            plt.xlabel('level: ' + str(self.level) + ' cluster: ' + str(self.cluster_no))
            plt.show()
            if recursive==True:
                for i in range(self.no_of_sub_clusters):
                    if self.sub_clusters[i]!=None:
                        self.sub_clusters[i].plot_cluster()

#paramteres to set

filename = "vectors"

def readFile(filename):
    conv_type=list
    vectors = []
    with open(filename,'r') as f:
        for line in f:
            vectors.append(conv_type([float(i) for i in line.split()]))
    return vectors

vectors = readFile(file_vectors)

print('constructing hierarchical clustering...\n\n')
root = cluster(vectors,None,0)
root.K_Mean_Clusters()

print('hierarchical cluster has been constructed\n\n')

candid_vectors = readFile(file_candid_vectors)

print('starting searching procedure')

template = 'nearest vector: {0} distance: {1} candid vector: {2}\n'

total_time = 0

distance_cluster = []

with open(output_file,'w') as f:
    for i in range(len(candid_vectors)):
        start = time.time()
        found_vec = root.search(candid_vectors[i])
        end = time.time()
        total_time += end-start
        distance_cluster.append(distance(found_vec[0],candid_vectors[i]))
        f.write(template.format(str(found_vec[0]),str(distance_cluster[-1]),str(candid_vectors[i])))

print('searching procedure has been ended.')
print('time taken by searching: '+str(total_time)+'s')
if plot_verbose==True:
    root.plot_cluster()

ans = input('\n\nwant to check accuracy by comparing it with brute - force solution[y\\N]:  ')

if ans.lower() == 'y':
    print('bruter-force searching has been begun')

    total_time = 0

    out_brute_file = '.ans_brute_force'
    distance_brute = []
    with open(out_brute_file, 'w') as f:
        for i in range(len(candid_vectors)):
            start = time.time()
            found_vec = find_nearest(vectors, candid_vectors[i])
            end = time.time()
            total_time+=end-start
            distance_brute.append(distance(found_vec[0],candid_vectors[i]))
            f.write(template.format(str(found_vec[0]), str(distance_brute[-1]),str(candid_vectors[i])))

    print('brute_force searching has been done')
    print('time taken by brute-force searching: '+str(total_time)+'s')

    print('\n\ncomparing both resultes for differences')

    difference=0

    for i in range(len(distance_cluster)):
        if distance_cluster[i]!=distance_brute[i]:
            difference+=1

    print('total number of differences: ' + str(difference))
    print('Searching accuracy: ',str((1-(difference/len(distance_cluster)))*100) + '%')


import torch
import torch.nn as nn
from cpu_offload import CPUOffload
import torchvision
class TorchCPUOffload(nn.Module):
    def __init__(self, module):
        super().__init__()
        self.module = module

    def forward(self, *args, **kwargs):
        with torch.autograd.graph.save_on_cpu(pin_memory=True):
            return self.module(*args, **kwargs)
# 定义 VGG16 模型
class VGG16(nn.Module):
    def __init__(self, num_classes=1000,num_channels = 64):
        super(VGG16, self).__init__()
        self.num_channels = num_channels
        # 第一段卷积层
        self.conv1_1 = nn.Conv2d(3, self.num_channels, kernel_size=3, padding=1)
        self.ReLU1_1 = nn.ReLU(inplace=True)
        self.conv1_2 = nn.Conv2d(self.num_channels, self.num_channels, kernel_size=3, padding=1)
        self.ReLU1_2 = nn.ReLU(inplace=True)
        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)

        # 第二段卷积层
        self.conv2_1 = nn.Conv2d(self.num_channels, self.num_channels * 2, kernel_size=3, padding=1)
        self.ReLU2_1 = nn.ReLU(inplace=True)
        self.conv2_2 = nn.Conv2d(self.num_channels * 2, self.num_channels * 2, kernel_size=3, padding=1)
        self.ReLU2_2 = nn.ReLU(inplace=True)
        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)

        # 第三段卷积层
        self.conv3_1 = nn.Conv2d(self.num_channels * 2,self.num_channels * 4, kernel_size=3, padding=1)
        self.ReLU3_1 = nn.ReLU(inplace=True)
        self.conv3_2 = nn.Conv2d(self.num_channels * 4, self.num_channels * 4, kernel_size=3, padding=1)
        self.ReLU3_2 = nn.ReLU(inplace=True)
        self.conv3_3 = nn.Conv2d(self.num_channels * 4, self.num_channels * 4, kernel_size=3, padding=1)
        self.ReLU3_3 = nn.ReLU(inplace=True)
        self.pool3 = nn.MaxPool2d(kernel_size=2, stride=2)

        # 第四段卷积层
        self.conv4_1 = nn.Conv2d(self.num_channels * 4, self.num_channels * 8, kernel_size=3, padding=1)
        self.ReLU4_1 = nn.ReLU(inplace=True)
        self.conv4_2 = nn.Conv2d(self.num_channels * 8, self.num_channels * 8, kernel_size=3, padding=1)
        self.ReLU4_2 = nn.ReLU(inplace=True)
        self.conv4_3 = nn.Conv2d(self.num_channels * 8, self.num_channels * 8, kernel_size=3, padding=1)
        self.ReLU4_3 = nn.ReLU(inplace=True)
        self.pool4 = nn.MaxPool2d(kernel_size=2, stride=2)

        # 第五段卷积层
        self.conv5_1 = nn.Conv2d(self.num_channels * 8, self.num_channels * 8, kernel_size=3, padding=1)
        self.ReLU5_1 = nn.ReLU(inplace=True)
        self.conv5_2 = nn.Conv2d(self.num_channels * 8, self.num_channels * 8, kernel_size=3, padding=1)
        self.ReLU5_2 = nn.ReLU(inplace=True)
        self.conv5_3 = nn.Conv2d(self.num_channels * 8, self.num_channels * 8, kernel_size=3, padding=1)
        self.ReLU5_3 = nn.ReLU(inplace=True)
        self.pool5 = nn.AdaptiveAvgPool2d((7,7))
        self.classifier = nn.Sequential(
            nn.Linear(self.num_channels * 8 * 7 * 7, 4096),
            nn.ReLU(inplace=True),
            nn.Linear(4096, 4096),
            nn.ReLU(inplace=True),
            nn.Linear(4096, num_classes)
        )

    def forward(self, x):
        # with torch.autograd.graph.save_on_cpu(pin_memory=True):
        x = self.conv1_1(x) # 122

        x =self.ReLU1_1(x) #121
        # with torch.autograd.graph.save_on_cpu(pin_memory=True):
        x = self.conv1_2(x) #121

        x = self.ReLU1_2(x) #120
        x = self.pool1(x) #60

        x = self.conv2_1(x) # 60

        x =self.ReLU2_1(x) #59

        x = self.conv2_2(x) #59

        x = self.ReLU2_2(x) #58
        x = self.pool2(x) #29

        x = self.conv3_1(x) # 29

        x =self.ReLU3_1(x) #28

        x = self.conv3_2(x) #28

        x = self.ReLU3_2(x) #27

        x = self.conv3_3(x) #27

        x = self.ReLU3_3(x) #26
        x = self.pool3(x) #13

        x = self.conv4_1(x) # 13

        x =self.ReLU4_1(x) #12

        x = self.conv4_2(x) #12

        x = self.ReLU4_2(x) #11

        x = self.conv4_3(x) #11

        x = self.ReLU4_3(x) #10
        x = self.pool4(x) #5

        x = self.conv5_1(x) # 5

        x =self.ReLU5_1(x) #4

        x = self.conv5_2(x) #4

        x = self.ReLU5_2(x) #3

        x = self.conv5_3(x) #3

        x = self.ReLU5_3(x) #2
        x = self.pool5(x) #1
        # x = self.features(x)
        x = torch.flatten(x, 1)
        x = self.classifier(x)
        return x

class Net(nn.Module):
    def __init__(self,  offload_ratio= 0,num_channels=64):
        super(Net, self).__init__()
        self.model = VGG16(num_classes=10,num_channels= num_channels)
        self.offload_ratio = offload_ratio
    def forward(self, x):
        with CPUOffload(offload_ratio=self.offload_ratio, tag="vgg16"):
            x = self.model(x)
        return x


'''
 Given a square matrix, calculate the absolute difference between the sums of its diagonals.
 For example, the square matrix ARR is shown below:
 1 2 3
 4 5 6
 9 8 9
 The left-to-right diagonal = 1+5+9 = 15 and The right to left diagonal = 3+5+9=17. Their absolute difference is 15-17=2.

 three different ways to sum the two diagonal in a two dimensional list
'''
import numpy as np

#----------------------------------------------------------------  1
def printDiagonal_1(): # with numpy

    a = [
        [11, 2, 4],
        [4, 5, 6],
        [10, 8, -12]
        ]

    diagonal_1 = np.asarray(a)

    diagonal_2 = np.asarray(a)
    diagonal_2 = np.fliplr(diagonal_2)

    sumDia_1 = np.trace(diagonal_1) # get sum
    sumDia_2 = np.trace(diagonal_2) # get sum

    result = abs(sumDia_1-sumDia_2) # abs of sum

    print("result: ", result)

#----------------------------------------------------------------  2
def printDiagonal_2(): # manual
    a = [
        [11, 2,  4],
        [4,  5,  6],
        [10, 8, -12]
        ]

    diagonal_1 = 0
    diagonal_2 = 0
    n = len(a)-1

    for i in range(len(a)):
        diagonal_1 = diagonal_1 + a[i][i]

        diagonal_2 = diagonal_2 + a[i][n-i]

    dif = abs(diagonal_1 - diagonal_2)

    print(dif)

#----------------------------------------------------------------  3
def printDiagonal_3():

    b = [
        [11, 2,  4],
        [4,  5,  6],
        [10, 8, -12]
        ]

    a = [
        [1, 2, 3, 5],
        [4, 5, 6, 7],
        [9, 8, 9, 8],
        [19,77,9,40]
        ]

    long = len(a)
    sumDiagonal_1 = 0
    sumDiagonal_2 = 0
    contador = long-1

    for i in range(long):
        for j in range(len(a[i])):
            if i == j: # first diagonal
                sumDiagonal_1 = sumDiagonal_1 + a[i][j]
            if j == contador: # second diagonal
                sumDiagonal_2 = sumDiagonal_2 + a[i][j]
                contador += -1

    result = abs(sumDiagonal_1-sumDiagonal_2)

    print(result)

printDiagonal_3()