"""postprocessing.py -- perform postprocessing and observable calculation tasks

Patrick Fasano
University of Notre Dame

- 10/25/17 (pjf): Created.
- 05/30/19 (pjf): Move obscalc-ob.res out to results subdirectory.
- 09/04/19 (pjf): Update for new em-gen, using l and s one-body RMEs.
- 09/11/19 (pjf): Update for new obscalc-ob input format (single vs. multi-file).
- 10/11/19 (pjf): Implement initial (basic) two-body postprocessing.
- 10/12/19 (pjf): Fix call to generation of mfdn_smwf.info.
- 12/11/19 (pjf): Use new results storage helper functions from mcscript.
- 08/14/20 (pjf): Implement generation of weak interaction one-body observables.
- 08/24/20 (pjf):
    + Implement one- and two-body postprocessing.
    + Remove old postprocessing scripting.
- 08/26/20 (pjf): Fix formatting of one-body electric transition operator names.
- 09/09/20 (pjf): Update to include ob_observable_sets.
- 09/16/20 (pjf):
    + Remove generate_electroweak().
    + Remove electromagnetic-specific code in evaluate_ob_observables().
    + Fix input of MFDn-generated OBDMEs.
    + Include observable sets as well as user-specified observables.
    + Stage out transitions-output and OBDMEs.
- 09/20/20 (pjf): Change/normalize res filenames.
- 09/24/20 (pjf): Fix transitions-tb res filename.
- 09/26/20 (pjf): Avoid LIMIT with UPDATE.
- 10/12/20 (pjf):
    + Improve diagnostic output when constructing transitions database.
    + Add progress updates to postprocessor master loops.
    + Prevent merging across nuclei.
- 10/14/20 (pjf):
    + Fix unpacking of J0 from ob_observables.
    + Suppress empty transitions-tb res file.
- 10/16/20 (pjf): Raise exception if no transitions are to be done.
- 10/21/20 (pjf): Factor out db construction from db connection.
- 05/14/21 (pjf):
    + Pass hw to mfdn-transitions.
    + Check parsed RME for NaN.
- 07/08/21 (pjf):
    + Add support for postfixes.
    + Ignore non-half-integer J values.
- 10/18/21 (mac): Provide "wf_source_glob_pattern" task parameter.
- 10/19/21 (pjf): Fix two-body observables if not one-body observables enabled.
- 12/18/21 (mac): Proceed with warning if failure loading am module.
- 03/28/22 (pjf): Fix disk thrashing when building database.
- 05/29/22 (pjf): Remove cleanup tasks from run_postprocessor.
- 06/30/22 (pjf): Deprecate run_postprocessor.
- 10/04/22 (pjf): Ensure that "diagonal" transitions always use same wave function
  for bra and ket.
- 12/16/22 (mac): Provide diagonalization results data to mask functions.
"""
import collections
import deprecated
import glob
import hashlib
import itertools
import math
import os
import re
import sqlite3
import warnings

import mcscript
import mfdnres
try:
    import am
except:
    warnings.warn("Error loading am module (may be due to missing gsl).  Some functionality will be lost.")

from . import (
    environ,
    library,
    mfdn_v15,
    modes,
    operators,
    utils,
    )

def evaluate_ob_observables(task, postfix=""):
    """Evaluate one-body observables with obscalc-ob.

    Arguments:
        task (dict): as described in module docstring
        postfix (string, optional): identifier to add to generated files
    """

    work_dir = "work{:s}".format(postfix)

    # accumulate obscalc-ob input lines
    lines = []

    # initial comment
    lines.append("# task: {}".format(task))
    lines.append("")

    # indexing setup
    lines += [
        "set-indexing {:s}".format(environ.orbitals_filename(postfix)),
        "set-output-file {:s} append".format(environ.obscalc_ob_res_filename(postfix)),
        ]

    # set up operators
    ob_observables = operators.ob.generate_ob_observable_sets(task)[0]
    ob_observables += task.get("ob_observables", [])
    for (operator_id, _, operator) in ob_observables:
        lines.append(
            "define-operator {:s} {:s}".format(
                operator_id,
                environ.obme_filename(postfix, operator)
                )
            )

    # get filenames for static densities and extract quantum numbers
    obdme_files = {}
    filenames = glob.glob(os.path.join(work_dir, "mfdn.statrobdme.*"))
    regex = re.compile(
        # directory prefix
        r"{}".format(os.path.join(work_dir, "")) +
        # prolog
        r"(?P<code>.+)\.statrobdme"
        # sequence number
        r"\.seq(?P<seq>\d{3})"
        # 2J
        r"\.2J(?P<twoJ>\d{2})"
        # parity (v14 only)
        r"(\.p(?P<g>\d))?"
        # n
        r"\.n(?P<n>\d{2})"
        # 2T
        r"\.2T(?P<twoT>\d{2})"
        )
    conversions = {
        "code": str,
        "seq": int,
        "twoJ": int,
        "g": lambda x: int(x) if x is not None else 0,
        "n": int,
        "twoT": int
        }
    for filename in filenames:
        match = regex.match(filename)
        if match is None:
            print(regex)
            raise ValueError("bad statrobdme filename: {}".format(filename))
        info = match.groupdict()

        # convert fields
        for key in info:
            conversion = conversions[key]
            info[key] = conversion(info[key])
        if "g" not in info:
            info["g"] = 0

        # extract quantum numbers
        qn = (info["twoJ"]/2., info["g"], info["n"])
        qn_pair = (qn, qn)

        obdme_files[qn_pair] = (filename, info["code"])

    # define-transition-densities 2Jf gf nf 2Ji gi fi robdme_info_filename robdme_filename
    # get filenames for static densities and extract quantum numbers
    filenames = glob.glob(os.path.join(work_dir, "*.robdme.*"))
    regex = re.compile(
        r"{}".format(os.path.join(work_dir, "")) +
        # prolog
        r"(?P<code>.+)\.robdme"
        # final sequence number (MFDn only)
        r"(\.seq(?P<seqf>\d+))?"
        # final 2J
        r"\.2J(?P<twoJf>\d+)"
        # final parity (v14/postprocessor only)
        r"(\.(p|g)(?P<gf>\d))?"
        # final n
        r"\.n(?P<nf>\d+)"
        # final 2T (MFDn only)
        r"(\.2T(?P<twoTf>\d+))?"
        # initial sequence number (MFDn only)
        r"(\.seq(?P<seqi>\d+))?"
        # initial 2J
        r"\.2J(?P<twoJi>\d+)"
        # initial parity (v14/postprocessor only)
        r"(\.(p|g)(?P<gi>\d))?"
        # initial n
        r"\.n(?P<ni>\d+)"
        # inital 2T (MFDn only)
        r"(\.2T(?P<twoTi>\d+))?"
        )
    conversions = {
        "code": str,
        "seqf": lambda x: int(x) if x is not None else 0,
        "twoJf": int,
        "gf": lambda x: int(x) if x is not None else 0,
        "nf": int,
        "twoTf": lambda x: int(x) if x is not None else 0,
        "seqi": lambda x: int(x) if x is not None else 0,
        "twoJi": int,
        "gi": lambda x: int(x) if x is not None else 0,
        "ni": int,
        "twoTi": lambda x: int(x) if x is not None else 0,
        }
    for filename in filenames:
        match = regex.match(filename)
        if match is None:
            raise ValueError("bad robdme filename: {}".format(filename))
        info = match.groupdict()

        # convert fields
        for key in info:
            conversion = conversions[key]
            info[key] = conversion(info[key])

        if "gf" not in info:
            info["gf"] = 0
        if "gi" not in info:
            info["gi"] = 0

        # extract quantum numbers
        qn_bra = (info["twoJf"]/2., info["gf"], info["nf"])
        qn_ket = (info["twoJi"]/2., info["gi"], info["ni"])
        qn_pair = (qn_bra, qn_ket)

        obdme_files[qn_pair] = (filename, info["code"])

    # sort by sequence number of final state, then sequence number of initial state
    for qn_pair,(filename,code) in sorted(obdme_files.items()):
        ((J_bra, g_bra, n_bra), (J_ket, g_ket, n_ket)) = qn_pair
        if code == "mfdn":
            lines.append(
                "define-densities {J_bra:4.1f} {g_bra:d} {n_bra:d}  {J_ket:4.1f} {g_ket:d} {n_ket:d} {filename:s} {info_filename:s}".format(
                    J_bra=J_bra, g_bra=g_bra, n_bra=n_bra,
                    J_ket=J_ket, g_ket=g_ket, n_ket=n_ket,
                    filename=filename,
                    info_filename=os.path.join(work_dir, "mfdn.rppobdme.info"),
                )
            )
        elif code == "transitions":
            lines.append(
                "define-densities {J_bra:4.1f} {g_bra:d} {n_bra:d}  {J_ket:4.1f} {g_ket:d} {n_ket:d} {filename:s}".format(
                    J_bra=J_bra, g_bra=g_bra, n_bra=n_bra,
                    J_ket=J_ket, g_ket=g_ket, n_ket=n_ket,
                    filename=filename,
                )
            )
        else:
            raise mcscript.exception.ScriptError("unknown density code {}".format(info["code"]))

    # ensure trailing line
    lines.append("")

    # write input file
    mcscript.utils.write_input(
        environ.obscalc_ob_filename(postfix),
        input_lines=lines,
        verbose=False
        )

    # invoke em-gen
    mcscript.call(
        [
            environ.shell_filename("obscalc-ob")
        ],
        input_lines=lines,
        mode=mcscript.CallMode.kSerial
    )

    # copy results out (if in multi-task run)
    print("Saving basic output files...")
    descriptor = task["metadata"]["descriptor"]
    filename_prefix = "{:s}-transitions-ob-{:s}{:s}".format(mcscript.parameters.run.name, descriptor, postfix)
    res_filename = "{:s}.res".format(filename_prefix)
    mcscript.task.save_results_single(
        task, environ.obscalc_ob_res_filename(postfix), res_filename, "res"
    )

def allowed_by_multipolarity(qn_pair, Tz_pair, operator_qn):
    """Apply multipolarity selection to qn_pair.

    Arguments:
        qn_pair (tuple of tuple): (qnf,qni)
        Tz_pair (tuple of int): (Tzf,Tzi)
        operator (tuple of int): (J0, g0, Tz0)

    Returns:
        allowed (bool): whether or not qn pair satisfies multipolarity condition
    """

    # apply multipolarity configuration
    (bra_qn, ket_qn) = qn_pair
    (bra_J, bra_g, bra_n) = bra_qn
    (ket_J, ket_g, ket_n) = ket_qn
    (bra_Tz, ket_Tz) = Tz_pair
    (J0, g0, Tz0) = operator_qn
    allowed = True
    allowed = allowed and mfdnres.am.allowed_triangle(bra_J,J0,ket_J)
    allowed = allowed and (bra_g+ket_g+g0)%2 == 0
    # note: an operator with Tz0 can actually be used for transitions
    # with +Tz0 or -Tz0
    # allowed = allowed and (ket_Tz + Tz0) == bra_Tz
    allowed = allowed and abs(bra_Tz-ket_Tz)==abs(Tz0)

    return allowed

def allowed_by_masks(task, qn_pair):
    """Apply masking functions to qn_pair.

    Each mask function should have a declaration the form

        mask_function(task,mask_params,qn_pair,verbose=False)

    Task dictionary fields:
        "postprocessor_mask"
        "postprocessor_mask_verbose"

    Arguments:
        task (dict): task dictionary
        qn_pair (tuple of tuple): (qnf,qni)

    Returns:
        allowed (bool): whether or not qn pair satisfies masks
    """

    # extract mask configuration
    mask_list = task.get("postprocessor_mask", [])
    verbose = task.get("postprocessor_mask_verbose", False)

    # apply masks
    if verbose:
        print("Mask: vetting {}".format(qn_pair))
    allowed = True
    for mask_function_params in mask_list:
        if not isinstance(mask_function_params, tuple):
            # wrap mask without parameters in tuple
            mask_function_params = (mask_function_params, {})
        (mask_function, mask_params) = mask_function_params
        mask_function_value = mask_function(task, mask_params, qn_pair, verbose=verbose)
        if verbose:
            print("  Mask: mask {} {}".format(mask_function.__name__, mask_function_value))
        allowed &= mask_function_value
    if verbose:
        print("  Mask: vetted {} {}".format(qn_pair, allowed))

    return allowed

def get_run_descriptor_pair(bra_mesh_data, ket_mesh_data, qn_pair, operator_qn):
    """Get (run, descriptor) pair for a given set of state and operator quantum numbers.

    Arguments:
        bra_mesh_data (list of mfdnres.MFDnResultsData): results mesh for bra sources
        ket_mesh_data (list of mfdnres.MFDnResultsData): results mesh for ket sources
        qn_pair (tuple of tuples): (qnf,qni)
        operator_qn (tuple): (J0,g0,Tz0)

    Returns:
        (tuple of tuples): ((bra_run,bra_descriptor),(ket_run,ket_descriptor))
    """
    # convenience variables
    (bra_qn, ket_qn) = qn_pair
    (bra_J, bra_g, bra_n) = bra_qn
    (ket_J, ket_g, ket_n) = ket_qn
    (J0, g0, Tz0) = operator_qn
    bra_J = am.HalfInt(round(2*bra_J), 2)
    ket_J = am.HalfInt(round(2*ket_J), 2)

    bra_run_descriptor_pair = None
    ket_run_descriptor_pair = None
    for bra_mesh_point in bra_mesh_data:
        if bra_qn not in bra_mesh_point.levels:
            continue
        # extract convenience variables
        bra_M = am.HalfInt(round(2*bra_mesh_point.params["M"]),2)

        for ket_mesh_point in ket_mesh_data:
            # special case: ensure that "diagonal" transitions are always
            # truly diagonal, to ensure that moment phases are well defined
            if (bra_qn == ket_qn) and (Tz0 == 0) and (g0 == 0):
                if ket_mesh_point != bra_mesh_point:
                    continue
            if ket_qn not in ket_mesh_point.levels:
                continue
            ket_M = am.HalfInt(round(2*ket_mesh_point.params["M"]),2)

            # check for Clebsch zero
            cg_coefficient = am.ClebschGordan(
                bra_J, bra_M, J0, ket_M-bra_M, ket_J, ket_M
            )
            if abs(cg_coefficient) < 1e-6:
                continue

            # update runs and descriptors
            bra_run_descriptor_pair = (
                bra_mesh_point.params["run"],
                bra_mesh_point.params["descriptor"]
                )
            ket_run_descriptor_pair = (
                ket_mesh_point.params["run"],
                ket_mesh_point.params["descriptor"]
                )

    return (bra_run_descriptor_pair, ket_run_descriptor_pair)

def init_postprocessor_db(task, postfix=""):
    """Initialize sqlite3 database for postprocessor runs.

    Sets up database structure and populates it with operators, and (bra,ket)
    pairs for two-body observables.

    Arguments:
        task (dict): as described in module docstring
        postfix (string, optional): identifier to add to generated files
    """

    # remove and rebuild any previously constructed database
    mcscript.call(["rm", "-vf", "transitions{}.sqlite".format(postfix)])
    db = sqlite3.connect("transitions{}.sqlite".format(postfix))
    db.row_factory = sqlite3.Row

    # enable foreign key support
    db.execute("PRAGMA foreign_keys = ON;")

    ################################################################
    # create operator table
    ################################################################
    db.execute(
        """CREATE TABLE tb_operators (
            operator_id  TEXT  NOT NULL  PRIMARY KEY  UNIQUE,
            J0  INTEGER NOT NULL,
            g0  INTEGER NOT NULL,
            Tz0 INTEGER NOT NULL
        );"""
    )

    # populate operator information
    tb_observables_by_qn = operators.tb.get_tbme_targets(task)
    for operator_qn in tb_observables_by_qn:
        db.executemany(
            "INSERT INTO tb_operators VALUES (?,?,?,?)",
            [(operator_name, *operator_qn) for operator_name in tb_observables_by_qn[operator_qn]]
        )
    db.commit()

    ################################################################
    # slurp res files and create bra and ket data objects
    ################################################################
    SORT_KEY_DESCRIPTOR = (("lanczos", int), ("M", float))
    UNMERGEABLE_KEYS = {"nuclide","natorb_base_state","natural_orbital_iteration"}
    # slurp source wave function info
    wf_source_res_dir_list = []
    for run in task["wf_source_run_list"]:
        wf_source_res_dir_list += [library.get_res_directory(run)]
    wf_source_glob_pattern = task.get("wf_source_glob_pattern","*.res")
    wf_source_mesh_data = mfdnres.input.slurp_res_files(
        wf_source_res_dir_list,
        filename_format="ALL",
        glob_pattern = wf_source_glob_pattern,
        verbose=True
    )

    # construct bra and ket info
    bra_selector = task["wf_source_bra_selector"]
    ket_selector = task["wf_source_ket_selector"]
    bra_mesh_data = mfdnres.analysis.selected_mesh_data(
        wf_source_mesh_data, bra_selector
        )
    if len(bra_mesh_data) == 0:
        raise mcscript.exception.ScriptError(
            "No MFDn res files found matching selector: "+str(bra_selector)
        )
    bra_mesh_data = mfdnres.analysis.sorted_mesh_data(
        bra_mesh_data, SORT_KEY_DESCRIPTOR
        )
    bra_merged_data = mfdnres.analysis.merged_mesh(
        bra_mesh_data, set(bra_selector.keys()) & UNMERGEABLE_KEYS
        )
    assert len(bra_merged_data) == 1
    bra_merged_data = bra_merged_data[0]
    task["metadata"]["bra_results_data"] = bra_merged_data  # provide access to results data for use by masking functions
    if bra_selector == ket_selector:
        # special case where bra and ket selection is equal:
        # allow canonicalization of transitions, and don't duplicate work
        canonicalize = True
        ket_mesh_data = bra_mesh_data[:]
        ket_merged_data = bra_merged_data
    else:
        canonicalize = False
        ket_mesh_data = mfdnres.analysis.selected_mesh_data(
            wf_source_mesh_data, ket_selector
            )
        if len(ket_mesh_data) == 0:
            raise mcscript.exception.ScriptError(
                "No MFDn res files found matching selector: "+str(ket_selector)
            )
        ket_mesh_data = mfdnres.analysis.sorted_mesh_data(
            ket_mesh_data, SORT_KEY_DESCRIPTOR
            )
        ket_merged_data = mfdnres.analysis.merged_mesh(
            ket_mesh_data, set(ket_selector.keys()) & UNMERGEABLE_KEYS
            )
        assert len(ket_merged_data) == 1
        ket_merged_data = ket_merged_data[0]
        task["metadata"]["ket_results_data"] = ket_merged_data  # provide access to results data for use by masking functions

    # extract Tz for bra and ket for convenience
    (bra_Z, bra_N) = bra_merged_data.params["nuclide"]
    bra_Tz = (bra_Z - bra_N)/2
    (ket_Z, ket_N) = ket_merged_data.params["nuclide"]
    ket_Tz = (ket_Z - ket_N)/2

    # diagnostic output
    print("Bra mesh points:")
    for mesh_point in bra_mesh_data:
        print(" ", mesh_point.params.get("run"), mesh_point.params.get("descriptor"))
    print("Ket mesh points:")
    for mesh_point in ket_mesh_data:
        print(" ", mesh_point.params.get("run"), mesh_point.params.get("descriptor"))

    ################################################################
    # create bra and ket level tables, and populate from meshes
    ################################################################
    db.execute(
        """CREATE TABLE bra_levels (
        bra_level_id INTEGER NOT NULL  PRIMARY KEY  UNIQUE,
        bra_J  REAL    NOT NULL,
        bra_g  INTEGER NOT NULL,
        bra_n  INTEGER NOT NULL
        );"""
    )
    db.execute(
        """CREATE TABLE ket_levels (
        ket_level_id INTEGER NOT NULL  PRIMARY KEY  UNIQUE,
        ket_J  REAL    NOT NULL,
        ket_g  INTEGER NOT NULL,
        ket_n  INTEGER NOT NULL
        );"""
    )

    ################################################################
    # populate level tables
    ################################################################
    db.executemany(
        "INSERT INTO bra_levels (bra_J,bra_g,bra_n) VALUES (?,?,?)",
        [(J,g,n) for (J,g,n) in bra_merged_data.levels if abs(round(2*J)-2*J)<=0.1]
    )
    bra_id_list = db.execute(
        "SELECT bra_J,bra_g,bra_n,bra_level_id FROM bra_levels"
        )
    bra_id_dict = {(J,g,n): level_id for (J,g,n,level_id) in bra_id_list}

    db.executemany(
        "INSERT INTO ket_levels (ket_J,ket_g,ket_n) VALUES (?,?,?)",
        [(J,g,n) for (J,g,n) in ket_merged_data.levels if abs(round(2*J)-2*J)<=0.1]
    )
    ket_id_list = db.execute(
        "SELECT ket_J,ket_g,ket_n,ket_level_id FROM ket_levels"
        )
    ket_id_dict = {(J,g,n): level_id for (J,g,n,level_id) in ket_id_list}

    ################################################################
    # create two-body transitions table
    ################################################################
    db.execute(
        """CREATE TABLE tb_transitions (
            bra_run        TEXT NOT NULL,
            bra_descriptor TEXT NOT NULL,
            bra_level_id   INTEGER  NOT NULL
                REFERENCES bra_levels (bra_level_id)
                ON DELETE RESTRICT
                ON UPDATE CASCADE,
            ket_run        TEXT NOT NULL,
            ket_descriptor TEXT NOT NULL,
            ket_level_id   INTEGER  NOT NULL
                REFERENCES ket_levels (ket_level_id)
                ON DELETE RESTRICT
                ON UPDATE CASCADE,
            operator_id TEXT NOT NULL
                REFERENCES tb_operators (operator_id)
                ON DELETE RESTRICT
                ON UPDATE CASCADE,
            rme NUMERIC,
            CONSTRAINT uniq UNIQUE (bra_level_id, ket_level_id, operator_id)
        );"""
    )

    ################################################################
    # populate two-body transitions table
    ################################################################
    # construct list of (bra,ket,tbo) tuples
    bra_ket_tbo_product = itertools.product(
        bra_id_dict.keys(), ket_id_dict.keys(), tb_observables_by_qn.keys()
        )
    for (bra_qn, ket_qn, operator_qn) in bra_ket_tbo_product:
        # check canonical order
        if canonicalize and not (bra_qn <= ket_qn):
            continue

        # apply masks
        if not allowed_by_multipolarity((bra_qn,ket_qn), (bra_Tz,ket_Tz), operator_qn):
            continue
        if not allowed_by_masks(task, (bra_qn,ket_qn)):
            continue

        (bra_run_descriptor_pair, ket_run_descriptor_pair) = get_run_descriptor_pair(
            bra_mesh_data, ket_mesh_data, (bra_qn, ket_qn), operator_qn
            )
        if (bra_run_descriptor_pair is None) or (ket_run_descriptor_pair is None):
            continue
        db.executemany(
            "INSERT INTO tb_transitions VALUES (?,?,?, ?,?,?, ?, NULL)",
            [
                (*bra_run_descriptor_pair, bra_id_dict[bra_qn],
                *ket_run_descriptor_pair, ket_id_dict[ket_qn],
                operator_name)
                for operator_name in tb_observables_by_qn[operator_qn]]
            )
    db.commit()

    ################################################################
    # create one-body transitions table
    ################################################################
    db.execute(
        """CREATE TABLE ob_transitions (
            bra_run        TEXT NOT NULL,
            bra_descriptor TEXT NOT NULL,
            bra_level_id   INTEGER  NOT NULL
                REFERENCES bra_levels (bra_level_id)
                ON DELETE RESTRICT
                ON UPDATE CASCADE,
            ket_run        TEXT NOT NULL,
            ket_descriptor TEXT NOT NULL,
            ket_level_id   INTEGER  NOT NULL
                REFERENCES ket_levels (ket_level_id)
                ON DELETE RESTRICT
                ON UPDATE CASCADE,
            finished BOOL,
            CONSTRAINT uniq UNIQUE (
                bra_run, bra_descriptor, bra_level_id,
                ket_run, ket_descriptor, ket_level_id
                )
        );"""
    )

    ################################################################
    # populate one-body transitions table
    ################################################################
    # construct list of (bra,ket,ob_qn) tuples
    ob_observables = operators.ob.generate_ob_observable_sets(task)[0]
    ob_observables += task.get("ob_observables", [])
    bra_ket_ob_qn_product = itertools.product(
        bra_id_dict.keys(), ket_id_dict.keys(),
        {operator_qn for (_,operator_qn,_) in ob_observables}
    )
    for (bra_qn, ket_qn, operator_qn) in bra_ket_ob_qn_product:
        # check canonical order
        if canonicalize and not (bra_qn <= ket_qn):
            continue

        # apply masks
        if not allowed_by_multipolarity((bra_qn,ket_qn), (bra_Tz,ket_Tz), operator_qn):
            continue
        if not allowed_by_masks(task, (bra_qn,ket_qn)):
            continue

        (bra_run_descriptor_pair, ket_run_descriptor_pair) = get_run_descriptor_pair(
            bra_mesh_data, ket_mesh_data, (bra_qn, ket_qn), operator_qn
            )
        if (bra_run_descriptor_pair is None) or (ket_run_descriptor_pair is None):
            continue
        db.execute(
            "INSERT OR IGNORE INTO ob_transitions VALUES (?,?,?, ?,?,?, NULL)",
            (*bra_run_descriptor_pair, bra_id_dict[bra_qn],
            *ket_run_descriptor_pair, ket_id_dict[ket_qn])
            )
    db.commit()

    # sanity check -- raise exception if nothing to do in task
    #   get total count of tb transitions
    (tb_count,) = db.execute(
        "SELECT COUNT(*) FROM `tb_transitions`;"
    ).fetchone()
    # get total count of tb transitions
    (ob_count,) = db.execute(
        "SELECT COUNT(*) FROM `ob_transitions`;"
    ).fetchone()
    print("Transition counts: tb transitions {:d}, ob transitions {:d}".format(tb_count, ob_count))
    if (tb_count + ob_count) == 0:
        raise mcscript.exception.ScriptError(
            "No transitions to be calculated."
        )

    # close (and finalize) database
    db.close()

def get_postprocessor_db_connection(postfix=""):
    """Connect to sqlite3 database for postprocessor runs.

    Returns:
        db (sqlite3.Connection): database connection
        postfix (string, optional): identifier to add to generated files
    """

    # connect to database
    db = sqlite3.connect("transitions{}.sqlite".format(postfix))
    db.row_factory = sqlite3.Row

    # check if two-body transition table exists
    res = db.execute(
        """SELECT name FROM sqlite_master
        WHERE type='table' AND name='tb_transitions';"""
        )
    if len(res.fetchall()) == 0:
        raise mcscript.exception.ScriptError("postprocessor database not correctly initialized")

    return db

def parse_two_body_observable(res, tokenized_lines):
    """
    """
    (J0,g0,Tz0,operator_id) = tokenized_lines[0]
    tb_observable_dict = res.setdefault("two_body_observables", {})
    transition_dict = tb_observable_dict.setdefault(operator_id,{})
    for tokenized_line in tokenized_lines[1:]:
        qnf = (float(tokenized_line[0]), int(tokenized_line[1]), int(tokenized_line[2]))
        qni = (float(tokenized_line[3]), int(tokenized_line[4]), int(tokenized_line[5]))
        rme = float(tokenized_line[6])
        if math.isnan(rme) or math.isinf(rme):
            raise ValueError("invalid rme: {}".format(rme))
        transition_dict[(qnf,qni)] = rme

def parse_transitions_results(in_file, verbose=False):
    """Parse transitions results file.

    Arguments:
        in_file (stream): input file stream (already opened by caller)
        verbose (bool,optional): enable verbose output
    """
    section_handlers = {
        "Two-body observable": parse_two_body_observable,
    }

    # perform high-level parsing into sections
    res_file_lines = [row for row in in_file]
    tokenized_lines = mfdnres.tools.split_and_prune_lines(res_file_lines)
    sections = mfdnres.tools.extracted_sections(tokenized_lines)

    res = {}
    for (section_name,tokenized_lines) in sections:
        if section_name in section_handlers:
            section_handlers[section_name](res, tokenized_lines)

    return res

def run_postprocessor_two_body(task, postfix="", one_body=False):
    """Evaluate matrix elements of two-body operators using mfdn-transitions.

    This handler calls initialize_postprocessor_db().

    Arguments:
        task (dict): as described in module docstring
        postfix (string, optional): identifier to add to generated files
        one_body (bool, optional): calculate incidental one-body densities
    """
    # convenience variables
    descriptor = task["metadata"]["descriptor"]
    work_dir = "work{:s}".format(postfix)
    transitions_executable = environ.mfdn_postprocessor_filename(
        task.get("mfdn-transitions_executable", "xtransitions")
    )
    ob_observables = operators.ob.generate_ob_observable_sets(task)[0]
    ob_observables += task.get("ob_observables", [])

    # create work directory if it doesn't exist yet
    mcscript.utils.mkdir(work_dir, exist_ok=True, parents=True)

    # make debugging output directories
    mcscript.utils.mkdir("transitions-output", exist_ok=True)

    # open database
    db = get_postprocessor_db_connection(postfix)

    # get set of operator quantum numbers
    cursor = db.execute(
        """SELECT DISTINCT `J0`,`g0`,`Tz0`
        FROM `tb_operators` LEFT JOIN `tb_transitions` USING(`operator_id`)
        WHERE rme IS NULL;
        """
        )
    operator_qn_list = list(cursor)

    # get total count of tb transitions
    (total_count,) = db.execute(
        "SELECT COUNT(*) FROM `tb_transitions`;"
    ).fetchone()

    ################################################################
    # begin master loop for two-body operators
    ################################################################
    timer = mcscript.utils.TaskTimer(remaining_time=mcscript.parameters.run.get_remaining_time())
    while db.execute("SELECT * FROM `tb_transitions` WHERE rme is NULL;").fetchone():
        # print status message
        (incomplete_count,) = db.execute(
            "SELECT COUNT(*) FROM `tb_transitions` WHERE rme is NULL;"
        ).fetchone()
        print("-"*64)
        print("Remaining two-body transitions: {:d}/{:d}".format(incomplete_count, total_count))
        print("-"*64)

        # start run timer (may raise exception if insufficient time remaining)
        timer.start_timer()

        # go to work directory
        os.chdir(work_dir)

        # get operator quantum numbers
        operator_qn = db.execute(
            """SELECT `J0`,`g0`,`Tz0`
            FROM `tb_operators` INNER JOIN `tb_transitions` USING(`operator_id`)
            WHERE rme IS NULL
            ORDER BY J0 ASC, g0 ASC, Tz0 ASC
            LIMIT 1;
            """
        ).fetchone()

        # get bra wavefunction specifier
        (bra_run, bra_descriptor, bra_level_id, bra_J, bra_g, bra_n) = db.execute(
            """SELECT bra_run, bra_descriptor, bra_level_id, bra_J, bra_g, bra_n
            FROM tb_transitions
                INNER JOIN tb_operators USING(operator_id)
                INNER JOIN bra_levels USING(bra_level_id)
            WHERE rme IS NULL AND (J0,g0,Tz0) = (?,?,?)
            ORDER BY bra_run ASC, bra_descriptor ASC, bra_J ASC, bra_g ASC, bra_n ASC
            LIMIT 1;
            """,
            operator_qn
        ).fetchone()

        # get operators
        operator_id_list = [row['operator_id'] for row in db.execute(
            """SELECT DISTINCT operator_id
            FROM tb_operators
                INNER JOIN tb_transitions USING(operator_id)
            WHERE rme IS NULL AND (J0,g0,Tz0) = (?,?,?)
                AND (bra_run,bra_descriptor,bra_level_id) = (?,?,?)
            ORDER BY operator_id ASC
            LIMIT 8;
            """,
            (*operator_qn, bra_run, bra_descriptor, bra_level_id)
        ).fetchall()]

        # get ket source
        (ket_run, ket_descriptor) = db.execute(
            """SELECT ket_run, ket_descriptor
            FROM tb_transitions
            WHERE rme IS NULL
                AND (bra_run,bra_descriptor,bra_level_id) = (?,?,?)
                AND operator_id IN ({:s})
            ORDER BY ket_run ASC, ket_descriptor ASC
            LIMIT 1;
            """.format(','.join('?'*len(operator_id_list))),
            (bra_run, bra_descriptor, bra_level_id, *operator_id_list)
        ).fetchone()

        # get ket quantum numbers
        ket_qn_id_list = db.execute(
            """SELECT DISTINCT ket_J, ket_g, ket_n, ket_level_id
            FROM tb_transitions
                INNER JOIN ket_levels USING(ket_level_id)
            WHERE rme IS NULL
                AND (bra_run,bra_descriptor,bra_level_id) = (?,?,?)
                AND operator_id IN ({:s})
                AND (ket_run, ket_descriptor) = (?,?)
            ORDER BY ket_J ASC, ket_g ASC, ket_n ASC
            LIMIT 8;
            """.format(','.join('?'*len(operator_id_list))),
            (bra_run, bra_descriptor, bra_level_id, *operator_id_list, ket_run, ket_descriptor)
        ).fetchall()
        ket_qn_list = [(J,g,n) for (J,g,n,ket_id) in ket_qn_id_list]
        ket_id_list = [ket_id for (J,g,n,ket_id) in ket_qn_id_list]

        # locate wave functions
        bra_wf_prefix = library.get_wf_prefix(bra_run,bra_descriptor)
        ket_wf_prefix = library.get_wf_prefix(ket_run,ket_descriptor)

        # check if we can pick up some OBDMEs for free
        if one_body:
            (num_free_obdmes,) = db.execute(
                """SELECT COUNT(*) FROM ob_transitions
                WHERE finished IS NULL
                    AND (bra_run,bra_descriptor,bra_level_id) = (?,?,?)
                    AND (ket_run,ket_descriptor) = (?,?)
                    AND ket_level_id IN ({:s});
                """.format(','.join('?'*len(ket_id_list))),
                (bra_run, bra_descriptor, bra_level_id,
                ket_run, ket_descriptor, *ket_id_list)
            ).fetchone()
        else:
            num_free_obdmes = 0

        # do calculation
        max_ket_J = max([ket_J for (ket_J,_,_) in ket_qn_list])
        min_ket_J = min([ket_J for (ket_J,_,_) in ket_qn_list])
        max_deltaJ = max(abs(max_ket_J-bra_J), max_ket_J+bra_J, abs(min_ket_J-bra_J), min_ket_J+bra_J)
        if num_free_obdmes > 0:
            max_J0 = max([J0 for _,(J0,_,_),_ in ob_observables])
            max2K = 2*int(min(max_deltaJ, max_J0))
        else:
            max2K = 0
        transitions_inputlist = {
            "basisfilename_bra": "{:s}/mfdn_MBgroups".format(bra_wf_prefix),
            "smwffilename_bra": "{:s}/mfdn_smwf".format(bra_wf_prefix),
            "infofilename_bra": "{:s}/mfdn_smwf.info".format(bra_wf_prefix),
            "TwoJ_bra": round(2*bra_J),
            "n_bra": int(bra_n),

            "basisfilename_ket": "{:s}/mfdn_MBgroups".format(ket_wf_prefix),
            "smwffilename_ket": "{:s}/mfdn_smwf".format(ket_wf_prefix),
            "infofilename_ket": "{:s}/mfdn_smwf.info".format(ket_wf_prefix),
            "TwoJ_ket": [round(2*ket_J) for (ket_J,ket_g,ket_n) in ket_qn_list],
            "n_ket": [int(ket_n) for (ket_J,ket_g,ket_n) in ket_qn_list],

            "obdme": True if num_free_obdmes > 0 else False,
            "max2K": max2K,
            "hbomeg": task.get("hw", 0.0),
            "numTBtrans": len(operator_id_list),
            "TBMEoperators": ["tbme-{}".format(basename) for basename in operator_id_list],
        }
        mcscript.utils.write_namelist(
            "transitions.input",
            input_dict={"transition_data": transitions_inputlist}
            )
        mcscript.call(["rm", "--force", "transitions.out", "transitions.res"])  # remove old output so file watchdog can work
        mcscript.call(
            [transitions_executable],
            mode=mcscript.CallMode.kHybrid,
            file_watchdog=mcscript.control.FileWatchdog("transitions.out"),
            file_watchdog_restarts=3
        )

        fp = open('transitions.res')
        res = parse_transitions_results(fp)
        fp.close()

        for (operator_id, transition_dict) in res["two_body_observables"].items():
            operator_id = operator_id.replace('tbme-','')
            for ((bra_qn,ket_qn), rme) in transition_dict.items():
                db.execute(
                    """UPDATE tb_transitions
                    SET rme = ?
                    WHERE (bra_level_id,ket_level_id,operator_id) = (
                        SELECT bra_level_id,ket_level_id,operator_id
                        FROM tb_transitions
                            INNER JOIN bra_levels USING(bra_level_id)
                            INNER JOIN ket_levels USING(ket_level_id)
                        WHERE (bra_J, bra_g, bra_n, ket_J, ket_g, ket_n, operator_id) =
                        (?,?,?,?,?,?,?)
                        LIMIT 1
                    );
                    """,
                    (rme, *bra_qn, *ket_qn, operator_id)
                )
        db.commit()

        # mark free OBDMEs as finished
        if one_body:
            db.executemany("""
                UPDATE ob_transitions SET finished = 1
                WHERE (bra_run,bra_descriptor,bra_level_id) = (?,?,?)
                    AND (ket_run,ket_descriptor,ket_level_id) = (?,?,?)
                """,
                [(bra_run,bra_descriptor,bra_level_id,ket_run,ket_descriptor,ket_level_id) for ket_level_id in ket_id_list]
                )
            db.commit()

        # save output (for debugging)
        filename_template = "{:s}-transitions-{:s}{:s}-{:s}.{:s}"
        transitions_output_dir = os.path.join("..", "transitions-output")
        group_hash = hashlib.sha1(
            repr(
                ((bra_J,bra_g,bra_n), ket_qn_list, operator_id_list)
                ).encode()
            ).hexdigest()[:8]
        out_filename = os.path.join(
            transitions_output_dir,
            filename_template.format(
                mcscript.parameters.run.name, descriptor, postfix, group_hash, "out"
            )
        )
        mcscript.call(["cp", "--verbose", "transitions.out", out_filename])
        res_filename = os.path.join(
            transitions_output_dir,
            filename_template.format(
                mcscript.parameters.run.name, descriptor, postfix, group_hash, "res"
            )
        )
        mcscript.call(["cp", "--verbose", "transitions.res", res_filename])
        timer.stop_timer()

        # return to task directory
        os.chdir("..")

    ################################
    # output and finalization
    ################################
    filename_prefix = "{:s}-transitions-tb-{:s}{:s}".format(mcscript.parameters.run.name, descriptor, postfix)
    res_filename = "{:s}.res".format(filename_prefix)

    # construct res file
    #   get operators and loop
    cursor = db.execute(
        """SELECT J0, g0, Tz0, operator_id
        FROM tb_operators
        ORDER BY operator_id ASC;
        """
        )
    lines = []
    for (J0, g0, Tz0, operator_id) in cursor:
        lines += ["[Two-body observable]"]
        lines += ["# {:>3s} {:>3s} {:>3s}  {:s}".format("J0", "g0", "Tz0", "name")]
        lines += ["  {:>3d} {:>3d} {:>3d}  {:s}".format(J0, g0, Tz0, operator_id)]
        data = db.execute("""
            SELECT bra_J, bra_g, bra_n, ket_J, ket_g, ket_n, rme
            FROM tb_transitions
                INNER JOIN bra_levels USING(bra_level_id)
                INNER JOIN ket_levels USING(ket_level_id)
            WHERE operator_id = ?
            ORDER BY bra_J ASC, bra_g ASC, bra_n ASC,
                ket_J ASC, ket_g ASC, ket_n ASC;
            """,
            (operator_id,)
        )
        lines += ["# {:>4s} {:>3s} {:>3s}  {:>4s} {:>3s} {:>3s}  {:>15s}".format(
            "Jf", "gf", "nf", "Ji", "gi", "ni", "rme"
            )
        ]
        for row in data:
            lines += ["  {bra_J:>4.1f} {bra_g:>3d} {bra_n:>3d}  {ket_J:>4.1f} {ket_g:>3d} {ket_n:>3d}  {rme:15.8e}".format(**row)]
        lines += [""]
    mcscript.utils.write_input(res_filename, lines, verbose=False)

    # only copy out res file if it contains more than a single newline
    if len(lines) > 1:
        # copy results out
        mcscript.task.save_results_single(
            task, res_filename, res_filename, "res"
        )

    # copy transitions-output
    out_file_list = glob.glob(os.path.join("transitions-output", "*"))
    mcscript.task.save_results_multi(
        task, out_file_list, descriptor, "transitions-output", command="cp"
    )

def run_postprocessor_one_body(task, postfix=""):
    """Evaluate one-body density matrix elements using the postprocessor.

    Arguments:
        task (dict): as described in module docstring
        postfix (string, optional): identifier to add to generated files
    """
    # convenience variables
    descriptor = task["metadata"]["descriptor"]
    work_dir = "work{:s}".format(postfix)
    transitions_executable = environ.mfdn_postprocessor_filename(
        task.get("mfdn-transitions_executable", "xtransitions")
    )
    ob_observables = operators.ob.generate_ob_observable_sets(task)[0]
    ob_observables += task.get("ob_observables", [])

    # create work directory if it doesn't exist yet
    mcscript.utils.mkdir(work_dir, exist_ok=True, parents=True)

    # make debugging output directories
    mcscript.utils.mkdir("transitions-output", exist_ok=True)

    # open database
    db = get_postprocessor_db_connection(postfix)

    # get total count of ob transition densities
    (total_count,) = db.execute(
        "SELECT COUNT(*) FROM `ob_transitions`;"
    ).fetchone()

    ################################################################
    # begin master loop for one-body operators
    ################################################################
    timer = mcscript.utils.TaskTimer(remaining_time=mcscript.parameters.run.get_remaining_time())
    while db.execute("SELECT * FROM `ob_transitions` WHERE finished is NULL;").fetchone():
        # print status message
        (incomplete_count,) = db.execute(
            "SELECT COUNT(*) FROM `ob_transitions` WHERE finished is NULL;"
        ).fetchone()
        print("-"*64)
        print("Remaining one-body transitions: {:d}/{:d}".format(incomplete_count, total_count))
        print("-"*64)

        # start run timer (may raise exception if insufficient time remaining)
        timer.start_timer()

        # go to work directory
        os.chdir(work_dir)

        # get bra wavefunction specifier
        (bra_run, bra_descriptor, bra_level_id, bra_J, bra_g, bra_n) = db.execute(
            """SELECT bra_run, bra_descriptor, bra_level_id, bra_J, bra_g, bra_n
            FROM ob_transitions
                INNER JOIN bra_levels USING(bra_level_id)
            WHERE finished IS NULL
            ORDER BY bra_run ASC, bra_descriptor ASC, bra_J ASC, bra_g ASC, bra_n ASC
            LIMIT 1;
            """).fetchone()

        # get ket source
        (ket_run, ket_descriptor) = db.execute(
            """SELECT ket_run, ket_descriptor
            FROM ob_transitions
            WHERE finished IS NULL
                AND (bra_run,bra_descriptor,bra_level_id) = (?,?,?)
            ORDER BY ket_run ASC, ket_descriptor ASC
            LIMIT 1;
            """,
            (bra_run, bra_descriptor, bra_level_id)
        ).fetchone()

        # get ket quantum numbers
        ket_qn_id_list = db.execute(
            """SELECT DISTINCT ket_J, ket_g, ket_n, ket_level_id
            FROM ob_transitions
                INNER JOIN ket_levels USING(ket_level_id)
            WHERE finished IS NULL
                AND (bra_run,bra_descriptor,bra_level_id) = (?,?,?)
                AND (ket_run, ket_descriptor) = (?,?)
            ORDER BY ket_J ASC, ket_g ASC, ket_n ASC
            LIMIT 8;
            """,
            (bra_run, bra_descriptor, bra_level_id, ket_run, ket_descriptor)
        ).fetchall()
        ket_qn_list = [(J,g,n) for (J,g,n,ket_id) in ket_qn_id_list]
        ket_id_list = [ket_id for (J,g,n,ket_id) in ket_qn_id_list]

        # locate wave functions
        bra_wf_prefix = library.get_wf_prefix(bra_run,bra_descriptor)
        ket_wf_prefix = library.get_wf_prefix(ket_run,ket_descriptor)

        # do calculation
        max_ket_J = max([ket_J for (ket_J,_,_) in ket_qn_list])
        min_ket_J = min([ket_J for (ket_J,_,_) in ket_qn_list])
        max_deltaJ = max(abs(max_ket_J-bra_J), max_ket_J+bra_J, abs(min_ket_J-bra_J), min_ket_J+bra_J)
        max_J0 = max([J0 for _,(J0,_,_),_ in ob_observables])
        max2K = 2*int(min(max_deltaJ, max_J0))
        transitions_inputlist = {
            "basisfilename_bra": "{:s}/mfdn_MBgroups".format(bra_wf_prefix),
            "smwffilename_bra": "{:s}/mfdn_smwf".format(bra_wf_prefix),
            "infofilename_bra": "{:s}/mfdn_smwf.info".format(bra_wf_prefix),
            "TwoJ_bra": round(2*bra_J),
            "n_bra": int(bra_n),

            "basisfilename_ket": "{:s}/mfdn_MBgroups".format(ket_wf_prefix),
            "smwffilename_ket": "{:s}/mfdn_smwf".format(ket_wf_prefix),
            "infofilename_ket": "{:s}/mfdn_smwf.info".format(ket_wf_prefix),
            "TwoJ_ket": [round(2*ket_J) for (ket_J,ket_g,ket_n) in ket_qn_list],
            "n_ket": [int(ket_n) for (ket_J,ket_g,ket_n) in ket_qn_list],

            "obdme": True,
            "max2K": max2K,
            "hbomeg": task.get("hw", 0.0),
        }
        mcscript.utils.write_namelist(
            "transitions.input",
            input_dict={"transition_data": transitions_inputlist}
            )
        mcscript.call(["rm", "--force", "transitions.out", "transitions.res"])  # remove old output so file watchdog can work
        mcscript.call(
            [transitions_executable],
            mode=mcscript.CallMode.kHybrid,
            file_watchdog=mcscript.control.FileWatchdog("transitions.out"),
            file_watchdog_restarts=3
        )

        # mark OBDMEs as finished
        db.executemany("""
            UPDATE ob_transitions SET finished = 1
            WHERE (bra_run,bra_descriptor,bra_level_id) = (?,?,?)
                AND (ket_run,ket_descriptor,ket_level_id) = (?,?,?)
            """,
            [(bra_run,bra_descriptor,bra_level_id,ket_run,ket_descriptor,ket_level_id) for ket_level_id in ket_id_list]
            )
        db.commit()

        # save output (for debugging)
        filename_template = "{:s}-transitions-{:s}{:s}-{:s}.{:s}"
        transitions_output_dir = os.path.join("..", "transitions-output")
        group_hash = hashlib.sha1(
            repr(
                ((bra_J,bra_g,bra_n), ket_qn_list)
                ).encode()
            ).hexdigest()[8:16]
        out_filename = os.path.join(
            transitions_output_dir,
            filename_template.format(
                mcscript.parameters.run.name, descriptor, postfix, group_hash, "out"
            )
        )
        mcscript.call(["cp", "--verbose", "transitions.out", out_filename])
        res_filename = os.path.join(
            transitions_output_dir,
            filename_template.format(
                mcscript.parameters.run.name, descriptor, postfix, group_hash, "res"
            )
        )
        mcscript.call(["cp", "--verbose", "transitions.res", res_filename])
        timer.stop_timer()

        # return to task directory
        os.chdir("..")

    ################################
    # output and finalization
    ################################
    # copy transitions-output
    out_file_list = glob.glob(os.path.join("transitions-output", "*"))
    mcscript.task.save_results_multi(
        task, out_file_list, task["metadata"]["descriptor"], "transitions-output", command="cp"
    )

def save_postprocessor_obdme(task, postfix=""):
    """Save postprocessor OBDMEs."""
    # convenience variables
    descriptor = task["metadata"]["descriptor"]
    work_dir = "work{:s}".format(postfix)
    # copy obdme out (if in multi-task run)
    if task.get("save_obdme"):
        archive_file_list = glob.glob(os.path.join(work_dir, "*.robdme*"))
        mcscript.task.save_results_multi(
            task, archive_file_list, descriptor, "obdme"
        )

def cleanup_workdir(task, postfix=""):
    """Remove temporary postprocessor work files.

    Arguments:
        task (dict): as described in module docstring
        postfix (string, optional): identifier to add to generated files
    """
    scratch_file_list = glob.glob("work{:s}/*".format(postfix))
    mcscript.call(["rm", "-vf"] + scratch_file_list)

@deprecated.deprecated(reason="use handlers.task_handler_mfdn_postprocessor_run instead")
def run_postprocessor(task, postfix=""):
    """Execute both phases of postprocessor.

    Arguments:
        task (dict): as described in module docstring
        postfix (string, optional): identifier to add to generated files
    """
    run_postprocessor_two_body(task, postfix=postfix, one_body=True)
    run_postprocessor_one_body(task, postfix=postfix)


import boto3

s3 = boto3.client('s3')

def get_buckets(bucket_name=""):
    response = s3.list_buckets()['Buckets']
    if bucket_name:
        match = [s for s in response if s['Name'].__contains__(
            bucket_name.lower())]
        if match:
            print("found matching buckets:")
            for bucket in match:
                print(f'{bucket["Name"]}')
        else:
            print("no bucket with such a name!")
    else:
        for bucket in response:
            print(f'{bucket["Name"]}')

if __name__ == '__main__':
    get_buckets("prod")


import time

def b_to_d():

    #Binary to Denary conversion
    binary = input("Input a number in binary:")
    denary = 0
    for digit in binary:

      denary = denary*2 + int(digit)
    print("Your denary number is: " + str(denary))
    return

def d_to_b():
    #Denary to Binary conversion
    denary = int(input("Input a denary number:"))
    binary=""
    while denary>0:

      binary = str(denary%2) + binary
      denary = denary//2
    print("Your binary number is: " + binary)
    return

print("Choose from denary to binary or from binary to denary. For the former type '0' and for the latter type '1'.")
decider = input()
decider = int(decider)
if decider == 0:
  d_to_b();
elif decider == 1:
  b_to_d();

time.sleep(10);
quit();


import cv2
import numpy as np

from matplotlib import pyplot as plt

# function(nothing to do)
def nothing(x):
    pass
cv2.namedWindow('image')
#import image
img = cv2.imread('C:\\Users\\na-31\\Desktop\\2dan.jpg',0)
# #create Trackbars
cv2.createTrackbar('med_ksize','image',2,10,nothing)
cv2.createTrackbar('th_min','image',5,255,nothing)
cv2.createTrackbar('th_max','image',15,255,nothing)

kernel = np.ones((4,2),np.uint8)

beforemed= 5
before1 = 5
before2 = 15
firestflag = 1

while(1):
    k = cv2.waitKey(1) & 0xFF
    if k == 27:
        break

    thre1 = cv2.getTrackbarPos('th_min','image')
    thre2 = cv2.getTrackbarPos('th_max','image')
    medksize = 2*(cv2.getTrackbarPos('med_ksize','image')) + 1

    print(thre1)
    print(thre2)
    print(medksize)

    if before1 != thre1 or before2 != thre2 or beforemed != medksize or firestflag == 1:
        before1 = thre1
        before2 = thre2
        beforemed = medksize
        firestflag = 0

        # median
        img_med = cv2.medianBlur(img, medksize)

        #detect edge
        edges = cv2.Canny(img_med,thre1,thre2)
        #remove noise
        gradient = cv2.morphologyEx(edges, cv2.MORPH_GRADIENT, kernel)

        #edge
        plt.subplot(121),plt.imshow(img,cmap = 'gray')
        plt.title('Original Image'), plt.xticks([]), plt.yticks([])
        plt.subplot(122),plt.imshow(gradient,cmap = 'gray')
        plt.title('Edge Image'), plt.xticks([]), plt.yticks([])

        plt.show()

cv2.destroyAllWindows()


# -*- coding: utf-8 -*-
"""
Created on Thu Aug  4 05:55:45 2022

@author: Rakesh
"""

###Please note this assignment has been finished based on few coding reference from Github ##

###problem Statement 1###########################

#Importing packages #
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.preprocessing import scale #normalization#
import seaborn as sns
from scipy.cluster.hierarchy import linkage #linkage for clustering
import scipy.cluster.hierarchy as sch #hiearchichal clustering#
from sklearn.cluster import KMeans ##Kmeans clustering ##

##importing data##
wine_data = pd.read_csv('D:/DATA SCIENCE ASSIGNMENT/Datasets_PCA/wine.csv')

wine_data.head()
wine_data.describe()
##lets drop Type columns in data set as it doenst add value ##

new_wine_date2 = wine_data.drop(['Type'], axis=1)
##checking na and null value##
new_wine_date2.isnull().sum()
new_wine_date2.isna().sum()
#checking duplicate value##
dup1 = new_wine_date2.duplicated()
sum(dup1)

##scaling data#
new_wine_date_norm = scale(new_wine_date2)

from sklearn.decomposition import PCA
##PCA value calculation#
pca= PCA(n_components=6)

new_wine_date_pca= pca.fit_transform(new_wine_date_norm)

var = pca.explained_variance_ratio_ ##getting variance in data#

pca.components_ ###view pca value

cumsum_var = np.cumsum(np.round(var,decimals=4)*100) ##getting cumulative variance in percentage

plt.plot(cumsum_var,color = 'blue') ##plotting pca variance#

new_wine_date_pca = pd.DataFrame(new_wine_date_pca)

new_wine_date_pca.columns = 'comp0', 'comp1' , 'comp2' , 'comp3' , 'comp4' , 'comp5' ##namimg pca columns

new_wine_date_pca_final =  pd.concat([wine_data.Type,new_wine_date_pca.iloc[:,0:3]], axis = 1)

## Boxplottimg to check outlier##

sns.boxplot(new_wine_date_pca_final.comp0);plt.title('Boxplot');plt.show()
sns.boxplot(new_wine_date_pca_final.comp1);plt.title('Boxplot');plt.show()
sns.boxplot(new_wine_date_pca_final.comp2);plt.title('Boxplot');plt.show()

##there are outlier in comp2
IQR =  new_wine_date_pca_final['comp2'].quantile(0.75)- new_wine_date_pca_final['comp2'].quantile(0.25)
lower_limit_comp2 = new_wine_date_pca_final['comp2'].quantile(0.25)-(IQR*1.5)
higher_limit_comp2 = new_wine_date_pca_final['comp2'].quantile(0.75)+(IQR*1.5)
new_wine_date_pca_final['comp2']=pd.DataFrame(np.where(new_wine_date_pca_final['comp2']>higher_limit_comp2,higher_limit_comp2,
                                                       np.where(new_wine_date_pca_final['comp2']<lower_limit_comp2,lower_limit_comp2,new_wine_date_pca_final['comp2'])))
sns.boxplot(new_wine_date_pca_final.comp2);plt.title('Boxplot');plt.show()

##hiearchical clustering##

linkage_complete = linkage(new_wine_date_pca_final, method = 'complete', metric='euclidean')
plt.figure(figsize=(15,8));plt.title('Hiearchical clustering for complete linkage');plt.xlabel('Index');plt.ylabel('Distance')
sch.dendrogram(linkage_complete,leaf_rotation=0, leaf_font_size=10)
plt.show()

##kmeans ##
##gettting cluster value for cluster range froom 2-9
TWSS = []
k = list(range(2,9))
for i in k:
    kmeans = KMeans(n_clusters=i)
    kmeans.fit(new_wine_date_pca_final)
    TWSS.append(kmeans.inertia_)

TWSS
##scree plotting##
plt.plot(k,TWSS, 'ro-');plt.xlabel('No_of_cluster');plt.ylabel('total_within_SS')

##model building ##
model_new_wine_date_pca_final= KMeans(n_clusters=3)
model_new_wine_date_pca_final.fit(new_wine_date_pca_final)

##storing clustering in original datafram##
model_new_wine_date_pca_final.labels_ ##assigning label to each rows#
cluster_new_wine_date_pca_final= pd.DataFrame(model_new_wine_date_pca_final.labels_) ##converting numpy in panda series object#
new_wine_date_pca_final['cluster']= cluster_new_wine_date_pca_final

##replacing index of column##
new_wine_date_pca_final =new_wine_date_pca_final.iloc[: , [4,0,1,2,3]]

#renaming original cluster data to match with calculated cluster data
#here the given clustered data is almost similar to the calcluated cluster data found out using pca values
new_wine_date_pca_final['Type'].replace({1:0, 2:2, 3:1},inplace = True)

new_wine_date_pca_final.to_csv('new_wine_data', encoding ='utf-8')

import os
os.getcwd()
##########################################Problem 5###############################################

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA
from sklearn.preprocessing import scale #normalization#
import seaborn as sns
from scipy.cluster.hierarchy import linkage #linkage for clustering
import scipy.cluster.hierarchy as sch #hiearchichal clustering#
from sklearn.cluster import KMeans ##Kmeans clustering ##

##loading dataset##
heart= pd.read_csv('D:/DATA SCIENCE ASSIGNMENT/Datasets_PCA/heart disease.csv')

##lets drop target column as it doesnt add value ##
heart_new = heart.drop(['target'], axis= 1)

heart_new.describe()
##cheking na & null value##
heart_new.isna().sum()
heart_new.isnull().sum()

#checking duplicated value ##
dup1 = heart_new.duplicated()
sum(dup1)

heart_new = heart_new.drop_duplicates()

##scaling data##

heart_new_norm = scale(heart_new)

##calculating pca values##
pca = PCA(n_components=6)
heart_new_pca =pca.fit_transform(heart_new_norm)

##calculating variance for PCA value ##
var =pca.explained_variance_ratio_

##viewing pca values
pca.components_

##calculating cumulative variance of the PCA values in percentages##

cumsum_var= np.cumsum(np.round(var,decimals=4)*100)

##plotting the variance data##
plt.plot(cumsum_var, color ='blue')

heart_new_pca = pd.DataFrame(heart_new_pca)
heart_new_pca.columns = 'comp0' , 'comp1' , 'comp2' , 'comp3' ,'comp4' , 'comp5'

##lets take first 3 component ##
heart_new_pca_final = heart_new_pca.iloc[: , 0:3]

##boxlotting for outlier analysis#

sns.boxplot(heart_new_pca_final.comp0);plt.title('Boxplot');plt.show()
sns.boxplot(heart_new_pca_final.comp1);plt.title('Boxplot');plt.show()
sns.boxplot(heart_new_pca_final.comp2);plt.title('Boxplot');plt.show()

##removing outlier using IQR#

IQR =  heart_new_pca_final['comp1'].quantile(0.75)- heart_new_pca_final['comp1'].quantile(0.25)
lower_limit_comp1 = heart_new_pca_final['comp1'].quantile(0.25)-(IQR*1.5)
higher_limit_comp1 = heart_new_pca_final['comp1'].quantile(0.75)+(IQR*1.5)
heart_new_pca_final['comp1']=pd.DataFrame(np.where(heart_new_pca_final['comp1']>higher_limit_comp1,higher_limit_comp1,
                                                       np.where(heart_new_pca_final['comp1']<lower_limit_comp1,lower_limit_comp1,heart_new_pca_final['comp1'])))
sns.boxplot(heart_new_pca_final.comp1);plt.title('Boxplot');plt.show()

IQR =  heart_new_pca_final['comp2'].quantile(0.75)- heart_new_pca_final['comp2'].quantile(0.25)
lower_limit_comp2 = heart_new_pca_final['comp2'].quantile(0.25)-(IQR*1.5)
higher_limit_comp2 = heart_new_pca_final['comp2'].quantile(0.75)+(IQR*1.5)
heart_new_pca_final['comp2']=pd.DataFrame(np.where(heart_new_pca_final['comp2']>higher_limit_comp2,higher_limit_comp2,
                                                       np.where(heart_new_pca_final['comp2']<lower_limit_comp2,lower_limit_comp2,heart_new_pca_final['comp2'])))
sns.boxplot(heart_new_pca_final.comp2);plt.title('Boxplot');plt.show()

##linkage for hiearchical cluster and dendrgram##
linkage_complete =linkage(heart_new_pca_final ,method='complete', metric='euclidean')
plt.figure(figsize=(15,8)); plt.title('Hiearchical clustering dendrogram for complete linkage'); plt.xlabel('Index');plt.ylabel('Distance')
sch.dendrogram(linkage_complete , leaf_rotation=0 , leaf_font_size=10)
plt.show()

##kmeand model building##
TWSS = []
k = list(range(2,9))
for i in k:
    kmeans = KMeans(n_clusters=i)
    kmeans.fit(heart_new_pca_final)
    TWSS.append(kmeans.inertia_)
TWSS

##scree plotting ##

plt.plot(k, TWSS , 'ro-');plt.xlabel('No_of_clusters');plt.ylabel('total_within_SS')

##model building ##
model_heart_new_pca_final= KMeans(n_clusters=3)
model_heart_new_pca_final.fit(heart_new_pca_final)
model_heart_new_pca_final.labels_ #getting labels asigned to each row
cluster_heart_new_pca_final =pd.Series(model_heart_new_pca_final.labels_) ##converting numpy into panda series#
heart_new_pca_final['cluster'] = cluster_heart_new_pca_final
heart_new_pca_final =heart_new_pca_final.iloc[: , [3,0,1,2]]

##since the original cluster is given in only 2 cluster in statement , so let calculate Kmeans for 2 cluster#

model_heart_new_pca_final= KMeans(n_clusters=2)
model_heart_new_pca_final.fit(heart_new_pca_final)
model_heart_new_pca_final.labels_ #getting labels asigned to each row
cluster_heart_new_pca_final =pd.Series(model_heart_new_pca_final.labels_) ##converting numpy into panda series#
heart_new_pca_final['cluster'] = cluster_heart_new_pca_final
heart_new_pca_final =heart_new_pca_final.iloc[: , [3,0,1,2]]

heart_new_pca_final1= pd.concat([heart.target , heart_new_pca_final.iloc[: , 0:4]], axis=1)
heart_new_pca_final1['target'].replace({1:0 , 0:1}, inplace= True)
#comparison is done and found out that the given cluster[target] in question is different to cluster values for pca components


# 1833. 雪糕的最大数量
# 夏日炎炎，小男孩 Tony 想买一些雪糕消消暑。

# 商店中新到 n 支雪糕，用长度为 n 的数组 costs 表示雪糕的定价，其中 costs[i] 表示第 i 支雪糕的现金价格。Tony 一共有 coins 现金可以用于消费，他想要买尽可能多的雪糕。

# 给你价格数组 costs 和现金量 coins ，请你计算并返回 Tony 用 coins 现金能够买到的雪糕的 最大数量 。

# 注意：Tony 可以按任意顺序购买雪糕。

# 来源：力扣（LeetCode）
# 链接：https://leetcode-cn.com/problems/maximum-ice-cream-bars
# 著作权归领扣网络所有。商业转载请联系官方授权，非商业转载请注明出处。
from typing import List
import bisect

class Solution:
    def maxIceCream(self, costs: List[int], coins: int) -> int:
        costs.sort()
        pre = [0]
        for cost in costs:
            pre.append(pre[-1]+cost)
        ind = bisect.bisect_right(pre,coins)
        return ind-1

class Solution2:
    def maxIceCream(self, costs: List[int], coins: int) -> int:
        costs.sort()
        res = 0
        for cost in costs:
            if cost>coins:
                break
            coins -=cost
            res+=1
        return res

costs = [10,6,8,7,7,8]
coins = 5
sol = Solution()
print(sol.maxIceCream(costs,coins))


from functools import reduce
from LoadData import *

#ejercicio: map y filter
#1. definir una funcion map para convertir el campo nombre de cada estado a miniscula
#2. aplicar con map
#3. definir una funcion filter para filtrar humedales con estados "en recuperacion"
#4. aplicar filter

#solucion
#1. definir funcion map para convertir el campo nombre de cada estado a minuscula

#2. aplicar con map
# def minusculas(r):
#     return { r["nombre"],
#                 r["hectarias"],
#                 r["aves"],
#                 r["flora"],
#                 r["estado"].lower()}

# #3. definir una funcion filter para filtrar humedales con estados "en recuperacion"

# #4. aplicar filtro

# def filtroEstado(humedal):
#     return humedal["estado"] == "en conservacion"

# nombreArchivo = input("que archivo vamos a leer? \n")
# miDataSe = LoadData(nombreArchivo)
# print(miDataSe)

# dataSetFiltrado = list(filter(filtroEstado, miDataSe))
# print("humedal en estado conservacion: \n")
# print(dataSetFiltrado)

# registro = {"nombre":"PAMPALINDA",
#             "direccion":"Calle 5a 62-00",
#             "hectarias":20.0,
#             "aves":54,
#             "flora":37,
#             "estado":"En conservación"}
# campo_mayus = registro["nombre"].lower()
#print("campo nombre a minuscula:")
#print(campo_mayus)

# miDataSe2 = list(map(minusculas, miDataSe))
# print("estado a minuscula: \n")
# print(miDataSe2)

#funcion reduce:
# def  Misuma(a,b):
#     return a+b

# listaNumerosPares = list(range(1,101,))
# print("numeros pares")
# print(listaNumerosPares)
# suma = 0
# #seri sumatoria, usando un iterador
# for i in listaNumerosPares:
#     suma = suma + i
# print(suma)

# listaCadena = ["Muchos", "años", "despues", ",", "frente", "al", "peloton"]

#serie sumatoria, usando la funcion reduce
#sumaTotalReduce = reduce(Misuma, listaNumerosPares)
# sumaTotalReduce = reduce(lambda x,y: x+y, listaNumerosPares)
# productoTotalReduce = reduce(lambda x,y: x*y , listaNumerosPares)
# inicioCienAnioSoledad = reduce(lambda x,y: x+" "+y, listaCadena)
# print(f"suma con reduce:{sumaTotalReduce}")
# print(f"producto con reduce:{productoTotalReduce}")
# print(f"cadena concatenada:{inicioCienAnioSoledad}")

def convertirAnios(x):
    return {
        x["nombre"],
        x["Edad"]/12,
        x["Edad"]
    }

def filtrarVegetarianos(x):
    x["Vegetariano"] == True

listaEstudiantes = [
    {"nombre":"Duvan", "Edad":552, "Vegetariano":False},
    {"nombre":"ANDRES", "Edad":24, "Vegetariano":False},
    {"nombre":"alejandra", "Edad":27, "Vegetariano":True}]

listtaEdadAnios = list(map(convertirAnios, listaEstudiantes))
print(f"map:{listtaEdadAnios}")

listaVegetarianos = list(filter(lambda x:x["Vegetariano"]==True,listaEstudiantes))
listtaEdadAnios = list(map(convertirAnios, listaVegetarianos))
print(f"filtro vegetariono:{listtaEdadAnios}")

# listaDatos = reduce(lambda x,y: x["nombre"]+"," y["nombre"], listaEstudiantes)
# print(f"cadena estudiante:{listaDatos}")
# listaNumeros = list(range(1,100,1))
# print("numeros")
# print(listaNumeros)


from collections import Counter, defaultdict
import csv
from itertools import chain, islice, takewhile
import math
from operator import itemgetter
from pathlib import Path
import pickle
import pprint
import sys
from typing import Dict, Iterable, List, Set, Tuple

from tqdm import tqdm

class PrettyPct(float):
    def __init__(self, x):
        self.x = 100*x

    def __repr__(self):
        if self.x == 0:
            return '0.000'
        elif self.x > 0.001:
            return f'{self.x:0.3f}'
        else:
            return f'{self.x:0.1e}'

def boltzmann(xs: Iterable[float]) -> float:
    Z = sum(math.exp(-x) for x in xs)
    return sum(x * math.exp(-x) / Z for x in xs)

def mean(xs: Iterable[float]) -> float:
    return sum(x for x in xs) / len(xs)

def var(xs: Iterable[float], x_mean: float) -> float:
    return sum((x-x_mean)**2 for x in xs) / len(xs)

def mean_and_sd(xs: Iterable[float]) -> Tuple[float, float]:
    x_mean = mean(xs)
    return x_mean, math.sqrt(var(xs, x_mean))

def recursive_conversion(nested_dict):
    if not isinstance(nested_dict, defaultdict):
        return nested_dict

    for k in nested_dict:
        # print(k)
        sub_dict = nested_dict[k]
        nested_dict[k] = recursive_conversion(sub_dict)
    return dict(nested_dict)

def read_data(p_data, k) -> List[Tuple]:
    with open(p_data) as fid:
        reader = csv.reader(fid); next(reader)
        # the data files are always sorted
        data = [(row[0], -float(row[1]))
                for row in takewhile(lambda row: bool(row), islice(reader, k))] #if row[1]]
        # data=[]
        # for row in islice(reader, k):
        #     data.append((row[0], -float(row[1])))

    return sorted(data, key=itemgetter(1))[:k]

def get_smis_from_data(p_data) -> Set:
    with open(p_data) as fid:
        reader = csv.reader(fid); next(reader)
        smis = {row[0] for row in reader}

    return smis

def compare_results(found: List[Tuple], true: List[Tuple],
                    avg: bool = True, smis: bool = True, scores: bool = True
                    ) -> Tuple[float, float, float]:
    k = len(found)
    found_smis, found_scores = zip(*found)
    true_smis, true_scores = zip(*true)

    if avg:
        found_avg = mean(found_scores)
        true_avg = mean(true_scores)
        f_avg = found_avg / true_avg
    else:
        f_avg = None

    # if boltzmann:
    #     found_boltzmann = boltzmann(found_scores)
    #     true_boltzmann = boltzmann(true_scores)
    #     f_correct_boltzmann = found_boltzmann / true_boltzmann
    # else:
    #     f_correct_boltzmann = None

    if smis:
        found_smis = set(found_smis)
        true_smis = set(true_smis)
        correct_smis = len(found_smis & true_smis)
        f_smis = correct_smis / len(true_smis)
    else:
        f_smis = None

    if scores:
        missed_scores = Counter(true_scores)
        missed_scores.subtract(found_scores)
        n_missed_scores = sum(
            count if count > 0 else 0
            for count in missed_scores.values()
        )
        f_scores = (k - n_missed_scores) / k
    else:
        f_scores = None

    return f_avg, f_smis, f_scores

def calculate_metric_results(it_rep_results):
    avg_means_sds = [[]] * len(it_rep_results)
    smis_means_sds = [[]] * len(it_rep_results)
    scores_means_sds = [[]] * len(it_rep_results)
    for i in it_rep_results:
        rep_results = zip(*it_rep_results[i].values())
        fs_avgs, fs_smis, fs_scores = rep_results

        avg_means_sds[i] = (mean_and_sd(fs_avgs))
        smis_means_sds[i] = (mean_and_sd(fs_smis))
        scores_means_sds[i] = (mean_and_sd(fs_scores))

    avg_means, avg_sds = zip(*avg_means_sds)
    avg_means = list(map(PrettyPct, avg_means))
    avg_sds = list(map(PrettyPct, avg_sds))

    smis_means, smis_sds = zip(*smis_means_sds)
    smis_means = list(map(PrettyPct, smis_means))
    smis_sds = list(map(PrettyPct, smis_sds))

    scores_means, scores_sds = zip(*scores_means_sds)
    scores_means = list(map(PrettyPct, scores_means))
    scores_sds = list(map(PrettyPct, scores_sds))

    return ((avg_means, avg_sds),
            (smis_means, smis_sds),
            (scores_means, scores_sds))

def top_k_metrics():
    true = pickle.load(open(sys.argv[1], 'rb'))
    parent_dir = Path(sys.argv[2])
    k = int(sys.argv[3])

    try:
        true = sorted(true.items(), key=itemgetter(1))[:k]
    except AttributeError:
        true = sorted(true, key=itemgetter(1))[:k]

    nested_dict = lambda: defaultdict(nested_dict)
    results = nested_dict()
    random = nested_dict()
    common_smis = nested_dict()

    for child in tqdm(parent_dir.iterdir(), desc='Analyzing runs', unit='run'):
        if not child.is_dir():
            continue

        fields = str(child.stem).split('_')
        _, model, metric, _, rep, *_ = fields
        rep = int(rep)
        # _, model, metric, _, _, repeat, *_ = fields

        p_data = child / 'data'
        for p_iter in tqdm(p_data.iterdir(), desc='Analyzing iterations',
                           leave=False):
            try:
                it = int(p_iter.stem.split('_')[-1])
            except ValueError:
                continue

            found = read_data(p_iter, k)
            if metric == 'random':
                random[it][rep] = compare_results(found, true)
            else:
                results[model][metric][it][rep] = compare_results(found, true)
                common_smis[model][metric][it][rep] = {smi for smi, _ in found}

    d_model_metric_it_rep_results = recursive_conversion(results)
    random = recursive_conversion(random)
    common_smis = recursive_conversion(common_smis)

    # pprint.pprint(d_model_metric_it_rep_results)

    d_model_results = {}
    for model in d_model_metric_it_rep_results:
        d_metric_it_rep_results = d_model_metric_it_rep_results[model]

        d_metric_avg = {}
        d_metric_smis = {}
        d_metric_scores = {}

        for metric in d_metric_it_rep_results:
            d_it_rep_results = d_metric_it_rep_results[metric]

            avg, smis, scores = calculate_metric_results(d_it_rep_results)

            d_metric_avg[metric] = avg[0], avg[1]
            d_metric_smis[metric] = smis[0], smis[1]
            d_metric_scores[metric] = scores[0], scores[1]

        d_model_results[model] = {
            'avg': d_metric_avg,
            'smis': d_metric_smis,
            'scores': d_metric_scores
        }
    pprint.pprint(d_model_results, compact=True)

    if random:
        avg, smis, scores = calculate_metric_results(random)
        random = {
            'avg': (avg[0], avg[1]),
            'smis': (smis[0], smis[1]),
            'scores': (scores[0], scores[1])
        }
        pprint.pprint(random, compact=True)

def top_k_intersections():
    parent_dir = Path(sys.argv[2])
    k = int(sys.argv[3])

    nested_dict = lambda: defaultdict(nested_dict)
    common_smis = nested_dict()

    for child in tqdm(parent_dir.iterdir(), desc='Analyzing runs', unit='run'):
        if not child.is_dir():
            continue

        fields = str(child.stem).split('_')
        _, model, metric, _, rep, *_ = fields
        rep = int(rep)

        p_data = child / 'data'
        for p_iter in tqdm(p_data.iterdir(), desc='Analyzing iterations',
                           leave=False):
            try:
                it = int(p_iter.stem.split('_')[-1])
            except ValueError:
                continue

            found = read_data(p_iter, k)
            if metric == 'random':
                pass
                # random[it][rep] = compare_results(found, true)
            else:
                common_smis[model][metric][it][rep] = {smi for smi, _ in found}

    common_smis = recursive_conversion(common_smis)
    for model in common_smis:
        for metric in common_smis[model]:
            n_smis_by_iter = []
            for it in common_smis[model][metric]:
                smis_sets = list(common_smis[model][metric][it].values())
                n_smis_by_iter.append(len(set.intersection(*smis_sets)))
            common_smis[model][metric] = n_smis_by_iter
    pprint.pprint(common_smis, compact=True)

def unions():
    parent_dir = Path(sys.argv[2])

    nested_dict = lambda: defaultdict(nested_dict)
    total_smis = nested_dict()
    random = nested_dict()

    for child in tqdm(parent_dir.iterdir(), desc='Analyzing runs', unit='run'):
        if not child.is_dir():
            continue

        fields = str(child.stem).split('_')
        _, model, metric, _, rep, *_ = fields
        rep = int(rep)

        p_data = child / 'data'
        for p_iter in tqdm(p_data.iterdir(), desc='Analyzing iterations',
                           leave=False):
            try:
                it = int(p_iter.stem.split('_')[-1])
            except ValueError:
                continue

            if metric == 'random':
                random[it][rep] = get_smis_from_data(p_iter)
            else:
                total_smis[model][metric][it][rep] = get_smis_from_data(p_iter)

    total_smis = recursive_conversion(total_smis)
    for model in total_smis:
        for metric in total_smis[model]:
            n_smis_by_iter = []
            for it in range(6):#total_smis[model][metric]:
                smis_sets = list(total_smis[model][metric][it].values())
                n_smis_by_iter.append(len(set.union(*smis_sets)))
            total_smis[model][metric] = n_smis_by_iter

    random = recursive_conversion(random)
    n_random_smis_by_iter = []
    # for it in range(6):
    #     smis_sets = list(random[it].values())
    #     n_random_smis_by_iter.append(len(set.union(*smis_sets)))

    pprint.pprint(total_smis, compact=True)
    pprint.pprint(n_random_smis_by_iter, compact=True)

if __name__ == "__main__":
    if len(sys.argv) < 5:
        top_k_metrics()
        exit()
    if sys.argv[4] == 'intersection':
        top_k_intersections()
        exit()
    if sys.argv[4] == 'union':
        unions()
        exit()


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import tensorflow.keras as keras
import json
import random
import re
import joblib
from profanity_check import predict, predict_prob
import os
from config.definitions import ROOT_DIR

from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences

import tensorflow as tf

import requests

from keras.models import load_model

tokenizer = Tokenizer()

def load_keras_model():
    global model
    model = tf.keras.models.load_model(os.path.join(ROOT_DIR, 'static', 'mymodel76-100.h5'))
    # Graph is needed to execute tensor computations:
    # https://www.tensorflow.org/guide/intro_to_graphs
    global graph
    # Need to use compat.v1 versioning to fix AttributeError: module 'tensorflow' has no attribute 'get_default_graph'
    graph = tf.compat.v1.get_default_graph()
    return model

def prepare_tokenization():
    VOCAB_SIZE = 18261
    dataset = open(os.path.join(ROOT_DIR, 'static', 'allCorpusLinesLAST.txt'), encoding='utf-8').read()
    tokenizer = Tokenizer(num_words=VOCAB_SIZE)
    train_poetry = dataset.splitlines()
    tokenizer.fit_on_texts(train_poetry)
    return tokenizer

def create_poem(model, tokenizer, seed):
    random_count = 0
    synonym_count = 0
    next_words = 150
    # Remove most special signs from the seed
    poem = re.sub(r"[^a-zA-Z0-9' ]","", seed.lower())
    previous_word = ""

    for word in range(next_words):
        encoded_seed = tokenizer.texts_to_sequences([poem])
        encoded_seed = pad_sequences(encoded_seed, maxlen=50, padding='pre')

        # Use argmax to find the most probable class
        y_pred = np.argmax(model.predict(encoded_seed), axis=-1)

        predicted_word = ""
        # find word in dictionary
        for word, index in tokenizer.word_index.items():
            if index == y_pred:
                if word != previous_word:
                    previous_word = word
                    if word[-3:] != "eov":
                        predicted_word = word
                    else:
                        predicted_word = word.replace("eov","\n")
                else:
                    if synonym_count < random.randrange(0,3):
                        try:
                        # Get and clean synonym
                            previous_word = word
                            synonym = re.sub(r"[^a-zA-Z0-9' ]","", get_thesaurus_synonym(word.replace("eov","")).lower())
                            if synonym not in seed:
                                predicted_word = word = synonym
                                synonym_count += 1
                            else:
                                previous_word, word, predicted_word, random_count = process_random_word(previous_word, word, predicted_word, random_count, seed)
                        except Exception:
                            if random_count < random.randrange(0,3):
                                previous_word, word, predicted_word, random_count = process_random_word(previous_word, word, predicted_word, random_count, seed)
                    else:
                        if random_count < random.randrange(0,2):
                                    previous_word, word, predicted_word, random_count = process_random_word(previous_word, word, predicted_word, random_count, seed)
                        else:
                            break

        # add predicted word to seed
        poem = poem + ' ' + predicted_word

    print("poem = " + poem)
    return check_poem_offensiveness(poem)

def get_thesaurus_synonym(word):
    response = requests.get("https://words.bighugelabs.com/api/2/2da4fa3679f36cded7f904c1b08b614c/" + word + "/json")
    data = response.json()
    if "adjective" in data:
        if "syn" in data["adjective"]:
            # Take random adjective from json data
            synonym = data["adjective"]["syn"][random.randrange(0, len(data["adjective"]["syn"]))]
    elif "noun" in data:
        if "syn" in data["noun"]:
            # Take random noun from json data
            synonym = data["noun"]["syn"][random.randrange(0, len(data["noun"]["syn"]))]
    elif "verb" in data:
        if "syn" in data["verb"]:
            # Take random verb from json data
            synonym = data["verb"]["syn"][random.randrange(0, len(data["verb"]["syn"]))]
    elif "verb" and "noun" and "syn" not in data:
        # Take random word from json data
        synonym = data[random.randrange(0, len(data))]
    synonym = str(synonym)
    return check_offensiveness(synonym)

def get_random_word():
    response = requests.get("https://random-word-api.herokuapp.com/word")
    data = response.json()
    random_word = str(data[0])
    return check_offensiveness(random_word)

def check_offensiveness(word):
    if predict([word]) == 0:
        return word
    else:
        return get_random_word()

def check_poem_offensiveness(poem):
    if predict_prob([poem]) < 0.5:
        return poem
    else:
        return "Roses are red \n violets are blue \n this poem wasn't nice \n so try something new"

def process_random_word(previous_word, word, predicted_word, random_count, seed):
    try:
        previous_word = word
        # Get and clean random word
        word = predicted_word = re.sub(r"[^a-zA-Z0-9' ]","", get_random_word().lower())
        random_count += 1
    except Exception:
        try:
            randomSeedWordIndex = int(random.randrange(0, len(seed.split())))
            # Get and clean synonym of random word of the seed
            predicted_word = word = re.sub(r"[^a-zA-Z0-9' ]","", get_thesaurus_synonym(seed.split()[randomSeedWordIndex]).lower())
            previous_word = word
        except Exception:
            predicted_word = ""
    return previous_word, word, predicted_word, random_count


import json, requests
from django.core.cache import cache
from django.core.management.base import BaseCommand
from django.utils import timezone

def vaccine_appointments():
    # Response link was acquired from the LA County signup page (http://publichealth.lacounty.gov/acd/ncorona2019/vaccine/hcwsignup/pods/)
    # To view data source, load above link, and view NETWORK>RESPONSE tab for pod-data.js in your browser developer console
    response = requests.get('http://publichealth.lacounty.gov/acd/ncorona2019/js/pod-data.js')
    # Clean response text and convert to JSON
    cleaned_response = response.text.split('var unfiltered = ')[1]
    json_response = json.loads(cleaned_response)
    available_clinics = []

    # Filter only available clinics
    for j in json_response:
        # Only include clinics with clickable appointment links
        # Exclude clinics with 'no appointments are currently available' comment
        if j['link'] and 'no appointments are currently available' not in j['comments']:
            available_clinics.append(j)

    for a in available_clinics:
        # If clinic doesn't have a name, find it's parent record and join child to parent
        if not a['name']:
            parent = [j for j in json_response if j['id'] == a['xParent']]
            if parent:
                a['name'] = parent[0]['name']
                a['addr1'] = parent[0]['addr1']
                a['addr2'] = parent[0]['addr2']
            else:
                available_clinics.remove(a)
        # Derive zip code from addr2 header for searchability
        a['zip'] = a['addr2'][-5:]
        # If appointment date/time is listed, combine into comments
        if a['date'] or a['time']:
            a['comments'] = '%s, %s<br>%s' % (a['date'], a['time'], a['comments'])
        if not a['comments']:
            a['comments'] = 'Registration link is open'

    cache.set('vaccines_lacounty', available_clinics, None)

    # Set Last Updated timestamp to now
    last_updated = timezone.now()
    cache.set('vaccines_lacounty_timestamp', last_updated, None)

    return available_clinics, last_updated

class Command(BaseCommand):

    help = 'Scrape vaccine appointment sites'

    def handle(self, *args, **options):
        vaccine_appointments()


from typing import List, TypeVar

from htm_rl.representations.sar import BaseSar, Superposition
from htm_rl.utils import isnone

T = TypeVar('T')

SuperpositionList = List[Superposition]

Sar = BaseSar[List[int], int, int]
SarSuperposition = BaseSar[SuperpositionList, Superposition, Superposition]

def sar_superposition_has_reward(sar: SarSuperposition) -> bool:
    return sar.reward is not None and 1 in sar.reward

class SarSuperpositionFormatter:
    state_chars = ['X', '-', '#', '^']
    action_chars = ['<', '>', '^']
    reward_chars = ['.', '+', '-']

    n_rows: int
    n_cols: int

    def __init__(self, n_rows: int, n_cols: int):
        self.n_rows = n_rows
        self.n_cols = n_cols

    def format(self, sar: SarSuperposition) -> str:
        return ' '.join([
            self._str_from_state_superposition(sar.state),
            self._str_from_superposition(sar.action, self.action_chars),
            self._str_from_superposition(sar.reward, self.reward_chars),
        ])

    @staticmethod
    def _str_from_superposition(x: List[int], mapping: List[str], fixed_len: int = None) -> str:
        n_x = len(isnone(x, []))
        n = max(n_x, isnone(fixed_len, len(mapping)))
        d = n - n_x
        return ''.join(
            mapping[x[i - d]] if d <= i else ' '
            for i in range(n_x + d)
        )

    def _str_from_state_superposition(self, state: List[Superposition]) -> str:
        def to_str(x: Superposition) -> str:
            return self._str_from_superposition(x, self.state_chars)

        n_rows, n_cols = self.n_rows, self.n_cols
        return '\n'.join(
            '|'.join(to_str(state[row * n_cols + col]) for col in range(n_cols))
            for row in range(n_rows)
        )


from django.shortcuts import render, get_object_or_404
from django.http import Http404, HttpResponseRedirect, HttpResponse
from pokeapp.models import Pokemon
from .forms import PostForm
import json, os
from pprint import pprint
from django.conf import settings
from django.contrib import messages

# Create your views here.
def index(request):
    all_pokemons = Pokemon.objects.all()
    context = {
        'all_pokemons' : all_pokemons
    }
    return render(request, 'pokemon/index.html', context)

def create(request):
    if request.method == 'POST':
        # create a form instance and populate it with data from the request:
        form = PostForm(request.POST)
        # check whether it's valid:
        if form.is_valid():
            new_pokemon = form.save(commit=False)
            # process the data in form.cleaned_data as required
            json_data = open(os.path.join(settings.BASE_DIR, 'pokemons.json'), encoding='latin1' )
            data = json.load(json_data)
            checkpokemon = Pokemon.objects.filter(name = request.POST['name']).count()
            if checkpokemon:
                 messages.warning(request, 'Este pokemon ya existe en la base de datos.')
                 return HttpResponseRedirect('http://127.0.0.1:8000/pokemons/create')
            for pokemon in data:
                if pokemon['name'] == request.POST['name']:
                    new_pokemon.picture = pokemon['image_url']
                    new_pokemon.pokemon_type = pokemon['types'][0]
                    new_pokemon.birthday = request.POST['birthday']
                    new_pokemon.name = request.POST['name']
                    new_pokemon.zodiac_sign = request.POST['zodiac_sign']
                    new_pokemon.blood_type = request.POST['blood_type']
                    new_pokemon.latitude = request.POST['latitude']
                    new_pokemon.longitude = request.POST['longitude']
                    new_pokemon.save()
                    messages.success(request, 'Pokemon ' + new_pokemon.name + ' agregado correctamente')
                    break
            else:
                messages.warning(request, 'Este no es un pokemon valido.')
    # if a GET (or any other method) we'll create a blank form
    else:
        form = PostForm()

    return render(request, 'pokemon/create.html', {'form': form})

def reports(request):
    if request.method == 'POST':
        month = request.POST['month']
        all_pokemons = Pokemon.objects.filter(birthday__month=month)
        context = {
            'all_pokemons' : all_pokemons
        }
        return render(request, 'pokemon/bymonth.html', context )
    else:
        return render(request, 'pokemon/reports.html', context=None)

def bytype(request):
    if request.method == 'POST':
        poke_type = request.POST['pokemon_type']
        all_pokemons = Pokemon.objects.filter(pokemon_type=poke_type)
        context = {
            'all_pokemons' : all_pokemons
        }
        return render(request, 'pokemon/bymonth.html', context )

def detail(request, pokemon_id):
    pokemon = get_object_or_404(Pokemon, pk=pokemon_id)
    return render(request, 'pokemon/detail.html', {'pokemon' : pokemon})


#! /usr/bin/env python3

def iseven(number):
    """
    If the number is even, returns True, otherwise returns false.
    """
    return (number %2) == 0

for i in range(1, 101):
    if iseven(i):
        print(i)


from logging import warn
from typing import Any, Dict, Optional

from architectures.discriminator.blocks import DiscriminatorBlock
from architectures.discriminator.losses import (LSDiscriminatorialLoss,
                                                LSGeneratorLoss)
from architectures.segmentor.compose import ResnestUNet
from architectures.segmentor.losses import WeightedPartialCE, DiceLoss
from torch import nn
from torch._C import Size
from torch.functional import Tensor

class OctaScribbleNet(nn.Module):

    def __init__(
        self,
        raw_input_shape: Size,
        mask_input_shape: Size,
        is_training: bool,
        pretrian: bool,
        weight_path: str = 'resnest50-528c19ca.pth',
        num_classes: int = 2, num_filters: int = 64,
        instance_noise: bool = True, label_noise: bool = True,
        segmentor_gating_level: int = 4,
        discriminator_depth: int = 4,
        encoder_gating: bool = False,
        weakly_supervise: bool = True,
        ):
        """Vanila ScribbleNet architecture designed by Valvano et al.
        params:
        raw_input_shape: torch.Size         Shape of sample raw input.
        mask_input_shape: torch.Size        Shape of sample mask target with channels responding to the number of classes.
        is_training: bool                   Model training state.
        num_classes: int                    Number of classes.
        instance_noise: bool                Enable instance noise in training.
        label_noise: bool                   Enable label noise in training.
        segmentor_gating_level              Attention gating level. Default is 4 (All layer). Negative value will disable all gating.
        discriminator_depth: int            Discrimator depth. Default is 4 (Equal to ResnestUnet depth).
        """
        super().__init__()
        if mask_input_shape[1] != num_classes:
            warn('Number channels in mask input is not same as number of classes. Can cause an error when model discriminator is in use.')
        self.segmentor = ResnestUNet(num_classes=num_classes, pretrain=pretrian, weight_path=weight_path, gating_level=segmentor_gating_level, encoder_gating=encoder_gating)
        # Attention depth from resnest u-net is 4.
        if discriminator_depth > 0:
            self.discriminator = DiscriminatorBlock(
                input_shape=mask_input_shape, is_training=is_training, depth=discriminator_depth, num_filters=num_filters, instance_noise=instance_noise, label_noise=label_noise
            )
        # Loss Definition
        if weakly_supervise:
            self.supervised_loss = WeightedPartialCE(num_classes=num_classes, manual=True)
        else:
            self.supervised_loss = DiceLoss()
        self.discriminatorial_loss = LSDiscriminatorialLoss()
        self.generator_loss = LSGeneratorLoss()
        self.is_train = is_training

    def forward(self, x: Tensor, y: Optional[Tensor] = None) -> Dict[str, Any]:
        raise NotImplementedError


# -*- coding: utf-8 -*-
"""
Created on Thu Dec  8 00:33:34 2022

@author: MaxKo
"""
def TreeScan(t, r, c, rt, ct):
    visibleFromWest = True
    visibleFromEast = True
    visibleFromNorth = True
    visibleFromSouth = True

    for treeIndex in range(r - 1, -1, -1):
        if int(trees[treeIndex][c]) >= t:
            visibleFromNorth = False
            break
    for treeIndex in range(r + 1, rt):
        if int(trees[treeIndex][c]) >= t:
            visibleFromSouth = False
            break
    for treeIndex in range(c - 1, -1, -1):
        if int(trees[r][treeIndex]) >= t:
            visibleFromWest = False
            break
    for treeIndex in range(c + 1, ct):
        if int(trees[r][treeIndex]) >= t:
            visibleFromEast = False
            break
    if visibleFromNorth or visibleFromSouth or visibleFromWest or visibleFromEast:
        return 1
    else:
        return 0

trees = []

with open('Input/Day8Input.txt') as f:
    while True:
        treeline = f.readline().strip()
        if not treeline:
            break
        trees.append(treeline)

rows = len(trees)
columns = len(trees[0])
visibleTrees = 0
for row in range(0, rows):
    for column in range(0, columns):
        if row == 0 or row == rows - 1 or column == 0 or column == columns - 1:
            visibleTrees += 1
        else:
            visibleTrees += TreeScan(int(trees[row][column]), row, column, rows, columns)

print(visibleTrees)


# Reciprocal cycles
#
# A unit fraction contains 1 in the numerator. The decimal representation of the
# unit fractions with denominators 2 to 10 are given:
#
# 1/2	= 	0.5
# 1/3	= 	0.(3)
# 1/4	= 	0.25
# 1/5	= 	0.2
# 1/6	= 	0.1(6)
# 1/7	= 	0.(142857)
# 1/8	= 	0.125
# 1/9	= 	0.(1)
# 1/10	= 	0.1
#
# Where 0.1(6) means 0.166666..., and has a 1-digit recurring cycle. It can be
# seen that 1/7 has a 6-digit recurring cycle.
#
# Find the value of d < 1000 for which 1/d contains the longest recurring cycle
# in its decimal fraction part.

import time

def cycle_length(num):
  while num % 2 == 0:
    num /= 2
  while num % 5 == 0:
    num /= 5
  if num == 1:
    return 0
  i = 1
  while True:
    if (10 ** i - 1) % num == 0:
      return i
    i += 1

t0 = time.time()

best_i = 0
best_cycle = 0
for i in xrange(1, 1000):
  cycles = cycle_length(i)
  if best_cycle < cycles:
    best_cycle = cycles
    best_i = i

t1 = time.time()

print(best_i)
print('time %f' % (t1 - t0))
# 983
# time 0.168501


from PyQt5.QtWidgets import QDialog
import inputDialog

class Dialog(QDialog):
    def __init__(self, parent=None, hintmsg=''):
        super().__init__(parent)
        self.dialog_ui = inputDialog.Ui_Dialog()
        self.dialog_ui.setupUi(self)
        self.setWindowTitle('输入')
        self.dialog_ui.label.setText(hintmsg)
        self.dialog_ui.okbutton.clicked.connect(self.accept)
        # self.setWindowModality(Qt.ApplicationModal)

    def get_name(self):
        # return '666'
        return self.dialog_ui.Input.text()


class Solution(object):
    def maxProfit(self, prices):
        """
        :type prices: List[int]
        :rtype: int
        """
        res = 0
        cur = 0

        for i in range(1, len(prices)):
            cur = cur + prices[i] - prices[i - 1]
            cur = max(0, cur)
            res = max(res, cur)

        return res