from collections import defaultdict, namedtuple
from typing import List

import SCons
from SCons.Tool import install

ALIAS_MAP = "AIB_ALIAS_MAP"
BASE_COMPONENT = "AIB_BASE_COMPONENT"
BASE_ROLE = "AIB_BASE_ROLE"
COMPONENT = "AIB_COMPONENT"
REVERSE_COMPONENT_DEPENDENCIES = "AIB_COMPONENTS_EXTRA"
DEFAULT_COMPONENT = "AIB_DEFAULT_COMPONENT"
INSTALLED_FILES = "AIB_INSTALLED_FILES"
META_COMPONENT = "AIB_META_COMPONENT"
META_ROLE = "AIB_META_ROLE"
ROLE = "AIB_ROLE"
ROLE_DECLARATIONS = "AIB_ROLE_DECLARATIONS"
SUFFIX_MAP = "AIB_SUFFIX_MAP"
TASKS = "AIB_TASKS"

SuffixMap = namedtuple(
    "SuffixMap",
    ["directory", "default_role"],
)

class RoleInfo:
    """A component/role union Node."""

    def __init__(self, component, role, files=None, dependencies=None):
        self.id = "{}-{}".format(component, role)
        self.component = component
        self.role = role
        if files is None:
            self.files = set()
        else:
            self.files = set(files)

        if dependencies is None:
            self.dependencies = set()
        else:
            self.dependencies = set(dependencies)

    def __str__(self):
        return "RoleInfo({})".format(self.id)

    def __repr__(self):
        return self.__str__()

class DeclaredRole:
    def __init__(self, name, dependencies=None, transitive=False, silent=False):
        self.name = name

        if dependencies is None:
            self.dependencies = set()
        else:
            self.dependencies = {dep for dep in dependencies if dep is not None}

        self.silent = silent

def declare_role(env, **kwargs):
    """Construct a new role declaration"""
    return DeclaredRole(**kwargs)

def declare_roles(env, roles, base_role=None, meta_role=None):
    """Given a list of role declarations, validate them and store them in the environment"""
    role_names = [role.name for role in roles]
    if len(role_names) != len(set(role_names)):
        raise Exception("Cannot declare duplicate roles")

    # Ensure that all roles named in dependency lists actually were
    # passed in as a role.
    for role in roles:
        for d in role.dependencies:
            if d not in role_names:
                raise Exception("Role dependency '{}' does not name a declared role".format(d))

    if isinstance(base_role, str):
        if base_role not in role_names:
            raise Exception(
                "A base_role argument was provided but it does not name a declared role")
    elif isinstance(base_role, DeclaredRole):
        if base_role not in roles:
            raise Exception("A base_role argument was provided but it is not a declared role")
    elif base_role is not None:
        raise Exception("The base_role argument must be a string name of a role or a role object")
    else:
        # Set it to something falsey
        base_role = str()

    if isinstance(meta_role, str):
        if meta_role not in role_names:
            raise Exception(
                "A meta_role argument was provided but it does not name a declared role")
    elif isinstance(meta_role, DeclaredRole):
        if meta_role not in roles:
            raise Exception("A meta_role argument was provided but it is not a declared role")
    elif meta_role is not None:
        raise Exception("The meta_role argument must be a string name of a role or a role object")
    else:
        # Set it to something falsy
        meta_role = str()

    silents = [role for role in roles if role.silent]
    if len(silents) > 1:
        raise Exception("No more than one role can be declared as silent")

    # If a base role was given, then add it as a dependency of every
    # role that isn't the base role (which would be circular).
    if base_role:
        for role in roles:
            if role.name != base_role:
                role.dependencies.add(base_role)

    # Become a dictionary, so we can look up roles easily.
    roles = {role.name: role for role in roles}

    # If a meta role was given, then add every role which isn't the
    # meta role as one of its dependencies.
    if meta_role:
        roles[meta_role].dependencies.update(r for r in roles.keys() if r != meta_role)

    # TODO: Check for DAG

    # TODO: What if base_role or meta_role is really None?
    env[BASE_ROLE] = base_role
    env[META_ROLE] = meta_role
    env[ROLE_DECLARATIONS] = roles

def generate_alias_name(env, component, role, task):
    """Generate a scons alias for the component and role combination"""
    return "{task}-{component}{role}".format(
        task=task,
        component=component,
        role="" if env[ROLE_DECLARATIONS][role].silent else "-" + role,
    )

def get_alias_map_entry(env, component, role):
    c_entry = env[ALIAS_MAP][component]

    try:
        return c_entry[role]
    except KeyError:
        r_entry = RoleInfo(component=component, role=role)
        c_entry[role] = r_entry

        declaration = env[ROLE_DECLARATIONS].get(role)
        for dep in declaration.dependencies:
            dep_entry = get_alias_map_entry(env, component, dep)
            r_entry.dependencies.add(dep_entry)

        meta_component = env.get(META_COMPONENT)
        if meta_component and component != meta_component:
            meta_c_entry = get_alias_map_entry(env, meta_component, role)
            meta_c_entry.dependencies.add(r_entry)

        base_component = env.get(BASE_COMPONENT)
        if base_component and component != base_component:
            base_c_entry = get_alias_map_entry(env, base_component, role)
            r_entry.dependencies.add(base_c_entry)

        meta_role = env.get(META_ROLE)
        if (meta_role and role != meta_role and meta_component and component != meta_component):
            meta_r_entry = get_alias_map_entry(env, component, meta_role)
            meta_c_r_entry = get_alias_map_entry(env, meta_component, meta_role)
            meta_c_r_entry.dependencies.add(meta_r_entry)

        return r_entry

def get_component(node):
    return getattr(node.attributes, COMPONENT, None)

def get_role(node):
    return getattr(node.attributes, ROLE, None)

def scan_for_transitive_install(node, env, _path):
    """Walk the children of node finding all installed dependencies of it."""
    component = get_component(node.sources[0])
    role = get_role(node.sources[0])
    if component is None:
        return []

    scanned = getattr(node.attributes, "AIB_SCANNED", None)
    if scanned is not None:
        return scanned

    # Access directly by keys because we don't want to accidentally
    # create a new entry via get_alias_map_entry and instead should
    # throw a KeyError if we got here without valid components and
    # roles
    alias_map = env[ALIAS_MAP]
    entry = alias_map[component][role]
    role_deps = env[ROLE_DECLARATIONS].get(role).dependencies
    results = set()

    # We have to explicitly look at the various BASE files here since it's not
    # guaranteed they'll be pulled in anywhere in our grandchildren but we need
    # to always depend upon them. For example if env.AutoInstall some file 'foo'
    # tagged as common base but it's never used as a source for the
    # AutoInstalled file we're looking at or the children of our children (and
    # so on) then 'foo' would never get scanned in here without this explicit
    # dependency adding.
    base_component = env.get(BASE_COMPONENT)
    if base_component and component != base_component:
        base_role_entry = alias_map[base_component][role]
        if base_role_entry.files:
            results.update(base_role_entry.files)

    base_role = env.get(BASE_ROLE)
    if base_role and role != base_role:
        component_base_entry = alias_map[component][base_role]
        if component_base_entry.files:
            results.update(component_base_entry.files)

    if (base_role and base_component and component != base_component and role != base_role):
        base_base_entry = alias_map[base_component][base_role]
        if base_base_entry.files:
            results.update(base_base_entry.files)

    installed_children = set(grandchild for child in node.children()
                             for direct_children in child.children()
                             for grandchild in direct_children.get_executor().get_all_targets()
                             if direct_children.get_executor() and grandchild.has_builder())

    for child in installed_children:
        auto_installed_files = get_auto_installed_files(env, child)
        if not auto_installed_files:
            continue

        child_role = get_role(child)
        if child_role == role or child_role in role_deps:
            child_component = get_component(child)
            child_entry = get_alias_map_entry(env, child_component, child_role)

            # This is where component inheritance happens. We need a default
            # component for everything so we can store it but if during
            # transitive scanning we see a child with the default component here
            # we will move that file to our component. This prevents
            # over-stepping the DAG bounds since the default component is likely
            # to be large and an explicitly tagged file is unlikely to depend on
            # everything in it.
            if child_component == env.get(DEFAULT_COMPONENT):
                setattr(node.attributes, COMPONENT, component)
                for f in auto_installed_files:
                    child_entry.files.discard(f)
                entry.files.update(auto_installed_files)
            elif component != child_component:
                entry.dependencies.add(child_entry)

            results.update(auto_installed_files)

    # Produce deterministic output for caching purposes
    results = sorted(results, key=str)
    setattr(node.attributes, "AIB_SCANNED", results)

    return results

def scan_for_transitive_install_pseudobuilder(env, node):
    return scan_for_transitive_install(node, env, None)

def tag_components(env, target, **kwargs):
    """Create component and role dependency objects"""
    target = env.Flatten([target])
    component = kwargs.get(COMPONENT)
    role = kwargs.get(ROLE)
    if component is not None and (not isinstance(component, str) or " " in component):
        raise Exception("AIB_COMPONENT must be a string and contain no whitespace.")

    if component is None:
        raise Exception("AIB_COMPONENT must be provided; untagged targets: {}".format(
            [t.path for t in target]))

    if role is None:
        raise Exception("AIB_ROLE was not provided.")

    for t in target:
        t.attributes.keep_targetinfo = 1
        setattr(t.attributes, COMPONENT, component)
        setattr(t.attributes, ROLE, role)

    entry = get_alias_map_entry(env, component, role)

    # We cannot wire back dependencies to any combination of meta role, meta
    # component or base component. These cause dependency cycles because
    # get_alias_map_entry will do that wiring for us then we will try to
    # map them back on themselves in our loop.
    if (component != env.get(BASE_COMPONENT) and role != env.get(META_ROLE)
            and component != env.get(META_COMPONENT)):
        for component in kwargs.get(REVERSE_COMPONENT_DEPENDENCIES, []):
            component_dep = get_alias_map_entry(env, component, role)
            component_dep.dependencies.add(entry)

    return entry

def auto_install_task(env, component, role):
    """Auto install task."""
    entry = get_alias_map_entry(env, component, role)
    return list(entry.files)

def auto_install_pseudobuilder(env, target, source, **kwargs):
    """Auto install pseudo-builder."""
    source = env.Flatten([source])
    source = [env.File(s) for s in source]
    entry = env.TagComponents(source, **kwargs)

    installed_files = []
    for s in source:

        target_for_source = target

        if not target_for_source:

            # AIB currently uses file suffixes to do mapping. However, sometimes we need
            # to do the mapping based on a different suffix. This is used for things like
            # dSYM files, where we really just want to describe where .dSYM bundles should
            # be placed, but need to actually handle the substructure. Currently, this is
            # only used by separate_debug.py.
            #
            # TODO: Find a way to do this without the tools needing to coordinate.
            suffix = getattr(s.attributes, "aib_effective_suffix", s.get_suffix())
            auto_install_mapping = env[SUFFIX_MAP].get(suffix)

            if not auto_install_mapping:
                raise Exception("No target provided and no auto install mapping found for:", str(s))

            target_for_source = auto_install_mapping.directory

        # We've already auto installed this file and it may have belonged to a
        # different role since it wouldn't get retagged above. So we just skip
        # this files since SCons will already wire the dependency since s is a
        # source and so the file will get installed. A common error here is
        # adding debug files to the runtime component file if we do not skip
        # this.
        existing_installed_files = get_auto_installed_files(env, s)
        if existing_installed_files:
            continue

        # We must do an early subst here so that the _aib_debugdir
        # generator has a chance to run while seeing 'source'. We need
        # to do two substs here.  The first is to expand an variables
        # in `target_for_source` while we can see `source`. This is
        # needed for things like _aib_debugdir. Then, we need to do a
        # second subst to expand DESTDIR, interpolating
        # `target_for_source` in as $TARGET. Yes, this is confusing.
        target_for_source = env.subst(target_for_source, source=s)
        target_for_source = env.Dir(env.subst('$DESTDIR/$TARGET', target=target_for_source))

        aib_additional_directory = getattr(s.attributes, "aib_additional_directory", None)
        if aib_additional_directory is not None:
            target_for_source = env.Dir(aib_additional_directory, directory=target_for_source)

        new_installed_files = env.Install(target=target_for_source, source=s)
        setattr(s.attributes, INSTALLED_FILES, new_installed_files)
        setattr(new_installed_files[0].attributes, 'AIB_INSTALL_FROM', s)
        installed_files.extend(new_installed_files)

    entry.files.update(installed_files)
    return installed_files

def finalize_install_dependencies(env):
    """Generates task aliases and wires install dependencies."""

    # Wire up component dependencies and generate task aliases
    for task, func in env[TASKS].items():
        generate_dependent_aliases = True

        # The task map is a map of string task names (i.e. "install" by default)
        # to either a tuple or function. If it's a function we assume that we
        # generate dependent aliases for that task, otherwise if it's a tuple we
        # deconstruct it here to get the function (the first element) and a
        # boolean indicating whether or not to generate dependent aliases for
        # that task. For example the "archive" task added by the auto_archive
        # tool disables them because tarballs do not track dependencies so you
        # do not want archive-foo to build archive-bar as well if foo depends on
        # bar.
        if isinstance(func, tuple):
            func, generate_dependent_aliases = func

        for component, rolemap in env[ALIAS_MAP].items():
            for role, info in rolemap.items():
                alias_name = generate_alias_name(env, component, role, task)
                alias = env.Alias(alias_name, func(env, component, role))
                if generate_dependent_aliases:
                    dependent_aliases = env.Flatten([
                        env.Alias(generate_alias_name(env, d.component, d.role, task))
                        for d in info.dependencies
                    ])
                    env.Alias(alias, dependent_aliases)

def auto_install_emitter(target, source, env):
    """When attached to a builder adds an appropriate AutoInstall to that Builder."""

    for t in target:
        if isinstance(t, str):
            t = env.File(t)

        if env.get("AIB_IGNORE", False):
            continue

        # There is no API for determining if an Entry is operating in
        # a SConf context. We obviously do not want to auto tag, and
        # install conftest Programs. So we filter them out the only
        # way available to us.
        #
        # We're working with upstream to expose this information.
        if "conftest" in str(t):
            continue

        # Get the suffix, unless overridden
        suffix = getattr(t.attributes, "aib_effective_suffix", t.get_suffix())
        auto_install_mapping = env[SUFFIX_MAP].get(suffix)

        if auto_install_mapping is not None:
            env.AutoInstall(
                auto_install_mapping.directory,
                t,
                AIB_COMPONENT=env.get(COMPONENT, env.get(DEFAULT_COMPONENT, None)),
                AIB_ROLE=env.get(ROLE, auto_install_mapping.default_role),
                AIB_COMPONENTS_EXTRA=env.get(REVERSE_COMPONENT_DEPENDENCIES, []),
            )

    return (target, source)

def add_suffix_mapping(env, suffix, role=None):
    """Map suffix to role"""
    if isinstance(suffix, str):
        if role not in env[ROLE_DECLARATIONS]:
            raise Exception("target {} is not a known role available roles are {}".format(
                role, env[ROLE_DECLARATIONS].keys()))
        env[SUFFIX_MAP][env.subst(suffix)] = role

    if not isinstance(suffix, dict):
        raise Exception("source must be a dictionary or a string")

    for _, mapping in suffix.items():
        role = mapping.default_role
        if role not in env[ROLE_DECLARATIONS]:
            raise Exception("target {} is not a known role. Available roles are {}".format(
                target, env[ROLE_DECLARATIONS].keys()))

    env[SUFFIX_MAP].update({env.subst(key): value for key, value in suffix.items()})

def suffix_mapping(env, directory="", default_role=False):
    """Generate a SuffixMap object from source and target."""
    return SuffixMap(directory=directory, default_role=default_role)

def get_auto_installed_files(env, node):
    return getattr(node.attributes, INSTALLED_FILES, [])

def list_components(env, **kwargs):
    """List registered components for env."""
    print("Known AIB components:")
    for key in env[ALIAS_MAP]:
        print("\t", key)

def list_hierarchical_aib_recursive(mapping, counter=0):
    if counter == 0:
        print("  " * counter, mapping.id)
    counter += 1
    for dep in mapping.dependencies:
        print("  " * counter, dep.id)
        list_hierarchical_aib_targets(dep, counter=counter)

def list_hierarchical_aib_targets(dag_mode=False):
    def target_lister(env, **kwargs):
        if dag_mode:
            installed_files = set(env.FindInstalledFiles())
            for f in installed_files:
                scan_for_transitive_install(f, env, None)

        mapping = env[ALIAS_MAP][env[META_COMPONENT]][env[META_ROLE]]
        list_hierarchical_aib_recursive(mapping)

    return target_lister

def list_recursive(mapping) -> List[str]:
    items = set()
    items.add(mapping.id)
    for dep in mapping.dependencies:
        items |= list_recursive(dep)
    return items

def list_targets():
    def target_lister(env, **kwargs):
        mapping = env[ALIAS_MAP][env[META_COMPONENT]][env[META_ROLE]]
        tasks = sorted(list(env[TASKS].keys()))
        roles = sorted(list(env[ROLE_DECLARATIONS].keys()))
        targets_with_role = list(list_recursive(mapping)) + [mapping.id]
        targets: List[str] = []
        for target_role in targets_with_role:
            # Does this target_role end with one of our speicifed roles
            matching_roles = list(filter(target_role.endswith, [f"-{role}" for role in roles]))
            assert len(matching_roles) == 1

            targets.append(target_role[:-len(matching_roles[0])])

        # dedup and sort targets
        targets = sorted(list(set(targets)))
        print(
            "The following are AIB targets. Note that runtime role is implied if not specified. For example, install-mongod"
        )
        tasks_str = ','.join(tasks)
        print(f"TASK={{{tasks_str}}}")
        roles_str = ','.join(roles)
        print(f"ROLE={{{roles_str}}}")
        for target in targets:
            print(f"  TASK-{target}-ROLE")

    return target_lister

def get_role_declaration(env, role):
    return env[ROLE_DECLARATIONS][role]

def exists(_env):
    """Always activate this tool."""
    return True

def generate(env):
    """Generate the auto install builders."""
    env["AUTO_INSTALL_ENABLED"] = True

    # Matches the autoconf documentation:
    # https://www.gnu.org/prep/standards/html_node/Directory-Variables.html
    env["DESTDIR"] = env.Dir(env.get("DESTDIR", "#install"))
    env["PREFIX"] = env.get("PREFIX", ".")
    env["PREFIX_BINDIR"] = env.get("PREFIX_BINDIR", "$PREFIX/bin")
    env["PREFIX_LIBDIR"] = env.get("PREFIX_LIBDIR", "$PREFIX/lib")
    env["PREFIX_SHAREDIR"] = env.get("PREFIX_SHAREDIR", "$PREFIX/share")
    env["PREFIX_DOCDIR"] = env.get("PREFIX_DOCDIR", "$PREFIX_SHAREDIR/doc")
    env["PREFIX_INCLUDEDIR"] = env.get("PREFIX_INCLUDEDIR", "$PREFIX/include")
    env[SUFFIX_MAP] = {}
    env[ALIAS_MAP] = defaultdict(dict)

    env.AppendUnique(AIB_TASKS={
        "install": auto_install_task,
    })

    env.AddMethod(
        scan_for_transitive_install_pseudobuilder,
        "GetTransitivelyInstalledFiles",
    )
    env.AddMethod(get_role_declaration, "GetRoleDeclaration")
    env.AddMethod(get_auto_installed_files, "GetAutoInstalledFiles")
    env.AddMethod(tag_components, "TagComponents")
    env.AddMethod(auto_install_pseudobuilder, "AutoInstall")
    env.AddMethod(add_suffix_mapping, "AddSuffixMapping")
    env.AddMethod(declare_role, "Role")
    env.AddMethod(declare_roles, "DeclareRoles")
    env.AddMethod(finalize_install_dependencies, "FinalizeInstallDependencies")
    env.AddMethod(suffix_mapping, "SuffixMap")
    env.Tool("install")

    # TODO: we should probably expose these as PseudoBuilders and let
    # users define their own aliases for them.
    env.Alias("list-aib-components", [], [list_components])
    env.AlwaysBuild("list-aib-components")

    env.Alias("list-hierarchical-aib-targets", [], [list_hierarchical_aib_targets(dag_mode=False)])
    env.AlwaysBuild("list-hierarchical-aib-targets")

    env.Alias("list-hierarchical-aib-dag", [], [list_hierarchical_aib_targets(dag_mode=True)])
    env.AlwaysBuild("list-hierarchical-aib-dag")

    env.Alias("list-targets", [], [list_targets()])
    env.AlwaysBuild("list-targets")

    for builder in ["Program", "SharedLibrary", "LoadableModule", "StaticLibrary"]:
        builder = env["BUILDERS"][builder]
        base_emitter = builder.emitter
        # TODO: investigate if using a ListEmitter here can cause
        # problems if AIB is not loaded last
        new_emitter = SCons.Builder.ListEmitter([base_emitter, auto_install_emitter])
        builder.emitter = new_emitter

    base_install_builder = install.BaseInstallBuilder
    assert base_install_builder.target_scanner is None

    base_install_builder.target_scanner = SCons.Scanner.Scanner(
        function=scan_for_transitive_install,
        path_function=None,
    )


def row2dict(row, fields: set=set()):
    d = {}
    for column in row.__table__.columns:
        if column.name in fields:
            d[column.name] = str(getattr(row, column.name))
    return d

def rows2dict(rows, fields: set=set()):
    arr = []
    for row in rows:
        arr.append(row2dict(row, fields))
    return arr

class ResponseWrapper:
    @staticmethod
    def ok(message: str='success', data: object=None):
        result = dict()
        result['message'] = message
        result['data'] = data
        return result


from io import BytesIO
from typing import Optional, Union
from pathlib import Path

from PIL import Image
import aiohttp

from hoshino import MessageSegment

from hoshino import res_dir
from hoshino.util.sutil import get_random_file, anti_harmony as anti_harmony_img

class ResImg:
    def __init__(self, abs_path: Union[str, Path]):
        if isinstance(abs_path, str):
            abs_path = Path(abs_path)
        #if not abs_path.is_relative_to(res_dir):
            #raise ValueError('Cannot access outside RESOUCE_DIR')
        self._path = abs_path

    @property
    def path(self):
        """资源文件的路径，供bot内部使用"""
        return self._path

    @property
    def cqcode(self) -> MessageSegment:
        with open(self.path, 'rb') as f:
            b = f.read()
        return MessageSegment.image(file=b)
        #return MessageSegment.image(f'file:///{os.path.abspath(self.path)}')

    @property
    def url(self) -> str:
        return self.path.as_uri()

    def open(self) -> Image.Image:
        try:
            return Image.open(self.path)
        except FileNotFoundError:
            print(f'缺少图片资源：{self.path}')
            raise

class ResRec:
    def __init__(self, abs_path: Union[str, Path]):
        if isinstance(abs_path, str):
            abs_path = Path(abs_path)
        #if not abs_path.is_relative_to(res_dir):
            #raise ValueError('Cannot access outside RESOUCE_DIR')
        self._path = abs_path

    @property
    def path(self):
        """资源文件的路径，供bot内部使用"""
        return self._path

    @property
    def cqcode(self) -> MessageSegment:
        with open(self.path, 'rb') as f:
            b = f.read()
        return MessageSegment.record(file=b)

    @property
    def url(self) -> str:
        return self.path.as_uri()

class Res:
    """
    Res资源封装类
    img 和 image 代表的分别为ResImg和 MessageSegment 对象
    img用于图像操作， image用于发送
    """
    base_dir = Path(res_dir)
    image_dir = base_dir.joinpath('image')
    record_dir = base_dir.joinpath('record')

    if not image_dir.exists():
        image_dir.mkdir()

    if not record_dir.exists():
        record_dir.mkdir()

    @classmethod
    def img(cls, p: Union[str, Path]) -> ResImg:
        if isinstance(p, str):
            p = Path(p)
        if p.exists():
            return ResImg(p)
        elif cls.image_dir.joinpath(p).exists():
            return ResImg(cls.image_dir.joinpath(p))
        else:
            raise ValueError('file not found')

    @classmethod
    def image(cls, p: Union[str, Path], anti_harmony: bool=False) -> MessageSegment:
        """
        将资源转换为可发送的图片
        params:
        p: 资源路径
        anti_harmony: 是否启用图片反和谐
        """
        img = cls.img(p).open()
        if anti_harmony:
            img = anti_harmony_img(img)
        return cls.image_from_memory(img)

    @classmethod
    def record(cls, p: Union[str, Path]) -> MessageSegment:
        if isinstance(p, str):
            p = Path(p)
        if p.exists():
            return ResRec(p).cqcode
        elif cls.record_dir.joinpath(p).exists():
            return ResRec(cls.record_dir.joinpath(p)).cqcode
        else:
            raise ValueError('file not found')

    @classmethod
    def get_random_img(cls, folder: Union[str, Path]) -> ResImg:
        """
        随机获取一个给定路径下的img， 以res_dir为基准目录
        """
        image_path = cls.base_dir.joinpath(folder)
        image_name = get_random_file(image_path)
        return cls.img(image_path.joinpath(image_name))

    @classmethod
    def get_random_record(cls, folder=None) -> MessageSegment:
        """
        随机获取一个给定路径下的record， 以res_dir为基准目录
        """
        record_path = cls.base_dir.joinpath(folder)
        rec_name = get_random_file(record_path)
        return cls.record(record_path.joinpath(rec_name))

    @classmethod
    def image_from_memory(cls, data: Union[bytes, Image.Image]) -> MessageSegment:
        if isinstance(data, Image.Image):
            out = BytesIO()
            data.save(out, format='png')
            data = out.getvalue()
        if not isinstance(data, bytes):
            raise ValueError(f'不支持的参数类型 {type(data)}')
        return MessageSegment.image(file=data, cache=False)

    @classmethod
    async def image_from_url(cls, url: str, anti_harmony: bool=False, timeout=30) -> MessageSegment:
        """
        download the image from url and return a MessageSegment in base64
        should be used in async function
        params:
        url: image url
        anti_harmony: bool = False 是否启用图片反和谐
        """
        timeout = aiohttp.ClientTimeout(total=timeout)
        async with aiohttp.ClientSession(timeout=timeout) as session:
            async with session.get(url) as resp:
                if resp.status != 200:
                    raise ValueError(f'请求失败 {resp.status}')
                data = await resp.read()
        if anti_harmony:
            data = anti_harmony_img(Image.open(BytesIO(data)))
        return cls.image_from_memory(data)


# This script shows the netlist database for the Inverter Database Using the new version of spectreIOlib
# Example for EE536B tutorial
# Qiaochu Zhang, from Mike Chen's Mixed-Signal Group, Ming Hsieh Dept. of ECE, USC
# 03/12/2020

#==================================================================
#*****************  Loading the libraries  ************************
#==================================================================

import sys
#sys.path.insert(0,'/shares/MLLibs/GlobalLibrary')
import os
home_address  = os.getcwd()
#sys.path.insert(0, home_address+'/MLLibs/GlobalLibrary')
sys.path.insert(0,'/home/mohsen/PYTHON_PHD/GlobalLibrary')
#download GlobalLibrary from GitHub, and change the path accordingly
#from Netlist_Database import Folded_Cascode_spice, ClassAB_spice

from tensorflow_circuit import TF_DEFAULT, make_var, np_elu
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import time
import math
import tensorflow as tf

from scipy.io import savemat
from pickle import dump

#==================================================================
#*******************  Initialization  *****************************
#==================================================================

tedad = 5
# number of parameter candidates you will see in the end. This number will control the number of
# gradient descent which will be performed.

#==================================================================
#****************  Loading the Regressors  ************************
#==================================================================

# Define the class of an Inverter
class INV(TF_DEFAULT): #change the name of the class to the name of your module

    def __init__(self,tech=65):

        self.tech=tech
        self.default_loading()

    def default_loading(self):
        if self.tech==65: # change the path of the regression files
            drive       = home_address+'/reg_files/tb_inv'
            sx_f        = drive + '/scX_tb_inv.pkl'
            sy_f        = drive + '/scY_tb_inv.pkl'
            w_f         = drive + '/w8_tb_inv.p'
            self.w_json = drive + '/model_tb_inv.json'
            self.w_h5   = drive + '/reg_tb_inv.h5'

            self.minx  = np.array([100e-9,100e-9 ])# parameter lower bound
            self.maxx  = np.array([1000e-9,1000e-9 ])# parameter upper bound
            self.stppar  = np.array([100e-9,100e-9])# sampling step

        self.loading(sx_f,sy_f,w_f)

#==================================================================
#*****************  Building the graph  ***************************
#==================================================================
def graph_tf(sxin,INV):
# sxin: scaled input parameters, INV: name of the class; and you should include all
# the names of the modules inside the parenthesis

    sx_INV = sxin[:,0:2] # define input parameters for INV
    print(sx_INV)
    x_INV  = INV.tf_rescalex(sx_INV) #rescale input parameters back to real value
    sy_INV = INV.tf_reg_relu(sx_INV) # calculate y = f(x)
    y_INV  = INV.tf_rescaley(sy_INV) #rescale output metrics back to real value
    print(y_INV)

    #==================================================================
    #***************  Define constraints and Cost(P)  *****************
    #==================================================================

    delay = y_INV #define metrics of your neural network

# use the metrics of all the modules to calculate top level specs, and store them in an array
    specs = []
    specs.append((delay-3e-12))
    specs.append(tf.reshape(tf.reduce_sum(x_INV),[1,1]))

# use the specs to construct constraints
    constraints = []
    constraints.append((tf.nn.relu(specs[0])/INV.scYscale[0])*100)
    constraints.append(tf.abs(specs[1]))

# You can define different loss functions based on different combinations of constraints
    hardcost=constraints[0]
    usercost=tf.reduce_sum(constraints)

    return hardcost,usercost,specs,[x_INV],[y_INV],[delay],constraints

#==================================================================
#*********************  Main code  ********************************
#==================================================================

if __name__ == '__main__':

    #==================================================================
    #*****************  Building the graph  ***************************
    #==================================================================
    tf.compat.v1.disable_eager_execution()
    #----------Initialize----------
    tf.compat.v1.reset_default_graph()

    # Define to optimizers. You can change the values in the parenthesis, which is the learning rate
    optimizer1 = tf.compat.v1.train.AdamOptimizer(0.001)
    optimizer2 = tf.compat.v1.train.AdamOptimizer(0.001)

    #----------load regressors----------

    # Define an object in class INV
    inv = INV()
    # Initialize all design parameters. We use random initialization in this case. We have in total 2 design parameters, 1 module here
    sxin = make_var("INV","INV", (1,2), tf.random_uniform_initializer(-np.ones((1,2)),np.ones((1,2))))
    #==================================================================
    #********************  Tensorflow Initiation  *********************
    #==================================================================
    hardcost,usercost,tf_specs,tf_param,tf_metric,tf_mid, tf_const = graph_tf(sxin,inv)

    # Optimizer1 will minimize hardcost, and optimizer 2 will minimize usercost. Both are defined previously.
    opt1=optimizer1.minimize(hardcost)
    opt2=optimizer2.minimize(usercost)
    init=tf.compat.v1.global_variables_initializer()

    calc=1

    lastvalue=-1000000
    lst_params=[]
    lst_metrics=[]
    lst_specs=[]
    lst_value=[]
    lst_midvalues=[]

    for j in range(tedad):
        const=[]
        with tf.compat.v1.Session() as sess:
            sess.run(init)
            #==================================================================
            #*****************  Tensorflow Gradient Descent  ******************
            #==================================================================
            tstart=time.time()
            for i in range(100): # the value in the parenthesis is the total steps of gradient descent that optimizer 1 will perform. You can change to another value.
                try:
                    _,value,smallspecs,last_const = sess.run([opt1,hardcost,tf_specs,tf_const])
                except:
                    print('Terminated due to error!')
                    break

                print('Optimizer1 = %1.0f:, %1.0f : %1.3f \n'%(j, i, value))
                const.append(smallspecs)
                if math.isnan(value):
                    break
                if np.abs(lastvalue-value)<epsilon:
                    break
                else:
                    lastvalue=value

            for i in range(100):# the value in the parenthesis is the total steps of gradient descent that optimizer 2 will perform. You can change to another value.
                try:
                    _,value,smallspecs,last_const = sess.run([opt2,hardcost,tf_specs,tf_const])
                except:
                    print('Terminated due to error!')
                    break
                print('Optimizer2 = %1.0f:, %1.0f : %1.3f \n'%(j, i, value))
                const.append(smallspecs)

                if math.isnan(value):
                    break
                if np.abs(lastvalue-value)<epsilon:
                    break
                else:
                    lastvalue=value

            #==================================================================
            #**********************  Saving the values  ***********************
            #==================================================================
            print('%1.0f : %1.3f \n'%(i, value))
            tend=time.time()
            np_sxin = sess.run(sxin)
            parameters = sess.run(tf_param)
            metrics    = sess.run(tf_metric)
            midvalues  = sess.run(tf_mid)

#            print('user1: %1.2f, user2: %1.2f, user3: %1.2f, user4: %1.2f, user5: %1.2f\n' %(sess.run(user1),sess.run(user2),sess.run(user3),sess.run(user4),sess.run(user5)))
            print('the elapsed time %1.2f S\n' %(tend-tstart))

        const_np=np.array(const)
        lst_params.append(parameters)
        lst_metrics.append(metrics)
        lst_specs.append(const[-1])
        lst_value.append(value)
        lst_midvalues.append(midvalues)

        mydict= {'lst_params':lst_params,'lst_metrics':lst_metrics,'lst_specs':lst_specs,'lst_value':lst_value}
        dump( mydict, open( 'regsearch_results1_'+str(cload)+'.p', "wb" ) )
#        savemat('regsearch_constraints.mat',{'const_np':const_np})

#        sp_value,_,sp_specs,sp_params,sp_metrics,sp_mids,sp_const= graph_spice(np_sxin,folded_cascode,classab,folded_cascode_spice,classab_spice)


# Simple linear Reg
# Importing the libraries
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

# Importing the dataset
dataset = pd.read_csv('Salary_Data.csv')
X = dataset.iloc[:, :-1].values
y = dataset.iloc[:, 1].values

# Splitting the dataset into the Training set and Test set
from sklearn.cross_validation import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 1/3, random_state = 0)

# Feature Scaling
#No feat scal in simpl reg, lib auto takes care
"""from sklearn.preprocessing import StandardScaler
sc_X = StandardScaler()
X_train = sc_X.fit_transform(X_train)
X_test = sc_X.transform(X_test)
sc_y = StandardScaler()
y_train = sc_y.fit_transform(y_train)"""

#Fitting simple linear reg in training set
from sklearn.linear_model import LinearRegression
regressor = LinearRegression()
regressor.fit(X_train, y_train)

#Prediciting test se results
y_pred = regressor.predict(X_test)

#Visualizing training set results
plt.scatter(X_train, y_train, color = 'red')
plt.plot(X_train, regressor.predict(X_train), color='blue') #forming a predicted line using the training examples
plt.title('sal vs exp(train set)')
plt.xlabel('years of exp')
plt.ylabel('sal')
plt.show()

#visualizing test set result
plt.scatter(X_test, y_test, color = 'red')
plt.plot(X_train, regressor.predict(X_train), color='blue') #same as of line 35, coz we are forming the hypothesis using training data
plt.title('sal vs exp(tesr set)')
plt.xlabel('years of exp')
plt.ylabel('sal')
plt.show()

Regression techniques/SImple Linear Regression/My_simple_linear_reg.py


from app import ma

from app.models import *

from marshmallow import fields

class UserSchema(ma.TableSchema):
    class Meta:
        table = User.__table__

    #requests = fields.Nested('RequestSchema',  many=True, only=["id", "transactions", "other", "credit", "status"])
    #audits = fields.Nested('AuditLogSchema',  many=True, only=["id", "target_type", "target_id", "state_before", "state_after"])

class OtherSchema(ma.TableSchema):
    class Meta:
        table = Other.__table__

    requests = fields.Nested('RequestSchema',  many=True, only=["id", "user", "transactions", "credit", "status"])

class CatalogSchema(ma.TableSchema):
    class Meta:
        table = Catalog.__table__

class RequestResponseSchema(ma.TableSchema):
    class Meta:
        table = RequestResponse.__table__
    user = fields.Nested('UserSchema', only=["id", "email", "fullname", "role"])
    request = fields.Nested('RequestSchema', only=["id", "user", "other", "credit", "status"])

class RequestSchema(ma.TableSchema):
    class Meta:
        table = Request.__table__
    user_id = fields.Int()
    other_id = fields.Int()

    responses = fields.Nested("RequestResponseSchema", many=True, only=["id", "action", "comment", "user"])
    transactions = fields.Nested("TransactionSchema", many=True, only=["id", "stock", "amount"])
    user = fields.Nested('UserSchema', only=["id", "email", "fullname", "role"])
    other = fields.Nested('OtherSchema', only=["id", "fullname", "phone", "staff"])
    store = fields.Nested('StoreSchema')

class TransactionSchema(ma.TableSchema):
    class Meta:
        table = Transaction.__table__
    request_id = fields.Int()

    stock = fields.Nested('StockSchema', only=["id", "catalog", "amount", "store"])
    request = fields.Nested('RequestSchema', only=["id", "user", "other", "credit", "status"])
    store = fields.Nested('StoreSchema')

class StockReportSchema(ma.TableSchema):
    class Meta:
        table = StockReport.__table__
    stock_id = fields.Int()

class AuditLogSchema(ma.TableSchema):
    class Meta:
        table = AuditLog.__table__
    user_id = fields.Int()
    action = fields.String()

    user = fields.Nested('UserSchema', only=["id", "email", "fullname", "role"])

class StockSchema(ma.TableSchema):
    class Meta:
        table = Stock.__table__
    catalog_id = fields.Int()
    store_id = fields.Int()

    catalog = fields.Nested('StockSchema', only=["id", "name", "description", "unit"])
    store = fields.Nested('StoreSchema')

class StoreSchema(ma.TableSchema):
    class Meta:
        table = Store.__table__

class StoreTransferSchema(ma.TableSchema):
    class Meta:
        table = StoreTransfer.__table__

    sent_by = fields.Nested('UserSchema', only=["id", "email", "fullname", "role"])
    received_by = fields.Nested('UserSchema', only=["id", "email", "fullname", "role"])
    approved_by = fields.Nested('UserSchema', only=["id", "email", "fullname", "role"])

    from_store = fields.Nested('StoreSchema')
    to_store = fields.Nested('StoreSchema')

class TransferItemSchema(ma.TableSchema):
    class Meta:
        table = TransferItem.__table__

    store_transfer = fields.Nested('StoreTransferSchema')
    stock = fields.Nested('StockSchema', only=["id", "catalog", "amount", "store"])
    other_stock = fields.Nested('StockSchema', only=["id", "catalog", "amount", "store"])

class HoldItemSchema(ma.TableSchema):
    class Meta:
        table = HoldItem.__table__

    store = fields.Nested('StoreSchema')
    stock = fields.Nested('StockSchema', only=["id", "catalog", "amount"])


import xml_to_od
from _ectt_to_od import ectt_data
from Models.instance import instance
from _objective_function import objective_function
from _initial_solution import initial_solution
from _roulette_wheel_selection import roulette_wheel_selection
from _simulated_annealing import simulated_annealing
from _operators_lookup import operators_lookup
import random
import math
import copy
import time
from Experiments.parameters import parameters
from Experiments.configs import configs
from Experiments.statistics import statistics
import uuid

class alns:

    def __init__(self):

        # Get data from instance file
        if configs.instance_type == 1:
            raw_data = xml_to_od.xml_data()
        else:
            raw_data = ectt_data()

        self.instance_data = instance(raw_data)

        #  It turns out that the usage of different percentages depending on the instance size is beneficial,
        #  i.e., ds is used for small instances with less than 280 lectures and dl for larger instances.
        if self.instance_data.total_lectures <= 280:
            self.destroy_permille = parameters.ds
        else:
            self.destroy_permille = parameters.dl

    # main function to find solution of CB-CTT instance using ALNS
    def find_optimal_solution(self, solution, Uc):
        # reset statistics value
        statistics_instance = statistics()
        statistics_instance.reset()

        obj_func_instance = objective_function("UD2", self.instance_data)

        current_best = solution

        # Calculate cost/objective function of current solution
        current_cost, courses_penalties, curricula_penalties = obj_func_instance.cost(solution)

        avg_cost = current_cost / self.instance_data.total_lectures
        unassigned_lectures_cost = max(float(1), float(avg_cost))
        current_cost += unassigned_lectures_cost * len(Uc)

        global_best = current_best
        global_best_cost = current_cost
        global_best_has_uc = len(Uc) > 0

        if global_best_cost == 0:
            return global_best, global_best_cost

        #  The reference destroy limit nmax 0 is set to d percent of the total number of lectures.
        # calculate upper bound for events to be destoryed
        destroy_upper_bound = self.instance_data.total_lectures * self.destroy_permille
        iteration = 0
        no_best_found_count = 0
        # destroy limit based on iterations since limit -> limit increased after reheat
        iterations_since_reheat = 0
        remaining_iterations = parameters.iteration_limit
        ref_iter_limit = parameters.iteration_limit

        time_start = time.time()

        SA = simulated_annealing(current_cost)

        while (time.time() - time_start) < parameters.time_limit:
            deviation_ref_iter = (remaining_iterations + iterations_since_reheat) - ref_iter_limit
            destroy_decrease_power = math.log((parameters.destroy_decrease - 1) * destroy_upper_bound / parameters.destroy_decrease) \
                                     / math.log(ref_iter_limit + deviation_ref_iter)
            iteration += 1
            iterations_since_reheat += 1
            statistics.iterations += 1
            print("Iteration: ", iteration)
            remaining_iterations = parameters.iteration_limit - ((time.time()-time_start)/parameters.time_limit) * parameters.iteration_limit
            print("Remaining Iterations: ", remaining_iterations)
            # -------------------------------------------------------------------------------------------------------------------- #
            removal_operators_probabilities = roulette_wheel_selection.get_probability_list(operators_lookup.removal_operators_weights)
            # print("Removal Operators Probabilities: ", removal_operators_probabilities)
            removal_operator_index = roulette_wheel_selection.spin_roulettewheel(removal_operators_probabilities)

            repair_operators_probabilities = roulette_wheel_selection.get_probability_list(operators_lookup.repair_operators_weights)
            # print("Repair Operators Probabilities: ", repair_operators_probabilities)

            # GA algorithm in our case it is designed in that way
            # that it will always expect that there is always at least on feasible spot for every unscheduled lecture
            # under the assumption that the unscheduled lectures were removed from a feasibile schedule
            # hence, if we have unfeasible solution we need to handle it with 2 stage repair operator instead
            print("Uc: ", Uc)
            if Uc:
                repair_operator_index = 0
            else:
                repair_operator_index = roulette_wheel_selection.spin_roulettewheel(repair_operators_probabilities)

            lecture_period_operators_probabilities = roulette_wheel_selection.get_probability_list(operators_lookup.lecture_period_operators_weights)
            # print("Lecture-Period Operators Probabilities: ", lecture_period_operators_probabilities)
            lecture_period_operator_index = roulette_wheel_selection.spin_roulettewheel(lecture_period_operators_probabilities)
            if lecture_period_operator_index == 0:
                statistics.two_stage_best_count += 1
            elif lecture_period_operator_index == 1:
                statistics.two_stage_mean_count += 1

            lecture_room_operators_probabilities = roulette_wheel_selection.get_probability_list(operators_lookup.lecture_room_operators_weights)
            # print("Lecture-Room Operators Probabilities: ", lecture_period_operators_probabilities)
            lecture_room_operator_index = roulette_wheel_selection.spin_roulettewheel(lecture_room_operators_probabilities)
            if lecture_period_operator_index == 0:
                statistics.two_stage_greatest_count += 1
            elif lecture_period_operator_index == 1:
                statistics.two_stage_match_count += 1

            priority_rules_probabilities = roulette_wheel_selection.get_probability_list(operators_lookup.priority_rules_weights)
            # print("Priority Rules Probabilities: ", lecture_period_operators_probabilities)
            priority_rule_index = roulette_wheel_selection.spin_roulettewheel(priority_rules_probabilities)
            if lecture_period_operator_index == 0:
                statistics.saturation_degree_count += 1
            elif lecture_period_operator_index == 1:
                statistics.largest_degree_count += 1
            else:
                statistics.random_order_count += 1

            # -------------------------------------------------------------------------------------------------------------------- #
            # number of events to be destroyed (lectures to remove) in this iteration
            destroy_limit = min(self.instance_data.total_lectures - len(Uc), parameters.min_destroy_lectures +
                                (random.randrange(max(parameters.min_destroy_lectures, round(destroy_upper_bound) - round(math.pow(iterations_since_reheat, destroy_decrease_power))))))

            # Find neighbor solution by applying destruction/repair operator to the current solution
            solution_clone = copy.deepcopy(solution)
            new_sol, Uc = self.neighbor(solution_clone, destroy_limit,
                                    removal_operator_index, repair_operator_index, lecture_period_operator_index, lecture_room_operator_index, priority_rule_index,
                                    courses_penalties, curricula_penalties, Uc)

            print("Uc: ", Uc)
            if new_sol is not None:
                # Calculate the new solution's cost
                new_cost, courses_penalties, curricula_penalties = obj_func_instance.cost(new_sol)
                # print("New solution cost: ", new_cost)

                if statistics.time_feasible == 0 and len(Uc) == 0:
                    statistics.iteration_feasible = iteration
                    statistics.time_feasible = time.time() - time_start

                temp_cost = new_cost + len(Uc) * unassigned_lectures_cost
                # The acceptance probability function takes in the old cost, new cost, and current temperature
                # and spits out a number between 0 and 1, which is a sort of recommendation on whether or not to jump to the new solution.
                accept = SA.accept_new_solution(temp_cost, current_cost, remaining_iterations)

                score_w1 = 0
                score_w2 = 0
                score_w3 = 0

                if accept:

                    if len(Uc) > 0:
                        unassigned_lectures_cost = min(unassigned_lectures_cost * parameters.adjust_unscheduled_cost,
                                                       float(self.instance_data.max_cost))
                    else:
                        unassigned_lectures_cost = max(float(1), unassigned_lectures_cost / parameters.adjust_unscheduled_cost)

                    statistics.accepted_count += 1
                    # print("New solution accepted.")
                    if temp_cost > current_cost: # worse than current solution
                        # print("New solution is worse than the current solution.")
                        statistics.worse_count += 1
                        score_w3 = parameters.w3
                    else: # better than current solution
                        # print("New solution is better than the current solution.")
                        statistics.better_count += 1
                        score_w2 = parameters.w2

                    solution = new_sol
                    current_cost = temp_cost
                # else:
                #     # print("New solution NOT accepted.")

                if (global_best_has_uc or new_cost < global_best_cost) and len(Uc) == 0:
                    global_best_has_uc = False
                    # print("New solution is the global best.")
                    statistics.global_best_counts += 1
                    statistics.iteration_best = iteration
                    statistics.time_best = time.time() - time_start
                    global_best = new_sol
                    global_best_cost = new_cost
                    score_w1 = parameters.w1

                if no_best_found_count >= parameters.reheat_limit and iteration < parameters.iteration_limit:
                    SA.reheat(new_cost)
                    no_best_found_count = 0
                    statistics.reheats_count += 1
                    iterations_since_reheat = 0
                    ref_iter_limit = remaining_iterations
                else:
                    no_best_found_count += 1

                if global_best_cost == 0:
                    return global_best, global_best_cost
                #
                # print("Score W1: ", score_w1)
                # print("Score W2: ", score_w3)
                # print("Score W3: ", score_w3)

                psi = max(score_w1, score_w2, score_w3)
                # print("PSI: ", psi)
                lambda_param = 0.8

                # recalculate weights of  operators based on the accepted solution
                operators_lookup.removal_operators_weights[str(removal_operator_index)] = lambda_param * operators_lookup.removal_operators_weights[str(removal_operator_index)] + (1-lambda_param) * psi
                operators_lookup.repair_operators_weights[str(repair_operator_index)] = lambda_param * operators_lookup.repair_operators_weights[str(repair_operator_index)] + (1-lambda_param) * psi

                # we need to recalculate weights only if two-stage operation was performed
                # if GA was performed it means that the below operators weren't used, hence didn't have impact in the result
                # consequently we won't update the weights in this iteration
                if repair_operator_index == 0:
                    operators_lookup.lecture_period_operators_weights[str(lecture_period_operator_index)] = lambda_param * operators_lookup.lecture_period_operators_weights[str(lecture_period_operator_index)] + (1-lambda_param) * psi
                    operators_lookup.lecture_room_operators_weights[str(lecture_room_operator_index)] = lambda_param * operators_lookup.lecture_room_operators_weights[str(lecture_room_operator_index)] + (1-lambda_param) * psi
                    operators_lookup.priority_rules_weights[str(priority_rule_index)] = lambda_param * operators_lookup.priority_rules_weights[str(priority_rule_index)] + (1-lambda_param) * psi

        # print('===============================\n')
        # print("Best solution cost: ", global_best_cost)
        # print('===============================\n')

        statistics.time = time.time() - time_start

        statistics_instance.print_statistics()
        return global_best, global_best_cost

    def neighbor(self, solution, destroy_limit,
                 removal_operator_index, repair_operator_index, lecture_period_operator_index, lecture_room_operator_index, priority_rule_index,
                 courses_penalties, curricula_penalties, Uc):

        # print("Destroy Limit: ", destroy_limit)
        lectures_to_remove = random.randint(1, destroy_limit)
        # print("Lectures to remove number: ", lectures_to_remove)

        # different destroy operators expect different input parameters
        # we are handling it accordingly below
        if removal_operator_index == 0:
            schedule, lectures_removed = operators_lookup.removal_operators[removal_operator_index](solution, lectures_to_remove, self.instance_data, courses_penalties)
        elif removal_operator_index == 1:
            schedule, lectures_removed = operators_lookup.removal_operators[removal_operator_index](solution, lectures_to_remove, self.instance_data, curricula_penalties)
        else:
            schedule, lectures_removed = operators_lookup.removal_operators[removal_operator_index](solution, lectures_to_remove, self.instance_data)

        # append lectured left unscheduled from prev row
        for l in Uc:
            lectures_removed.append(l)

        # repair_operator_index = 0
        # lecture_period_operator_index = 0
        # lecture_room_operator_index = 0
        # priority_rule_index = 0
        if repair_operator_index == 0:
            schedule, Uc = operators_lookup.repair_operators[repair_operator_index](schedule, self.instance_data, lectures_removed,
                                                                                operators_lookup.lecture_period_operators[lecture_period_operator_index],
                                                                                operators_lookup.lecture_room_operators[lecture_room_operator_index],
                                                                                operators_lookup.priority_rules[priority_rule_index])
        elif repair_operator_index == 1:
            schedule = operators_lookup.repair_operators[repair_operator_index](schedule, self.instance_data, lectures_removed)

        return schedule, Uc

    def execute(self):
        Uc = []

        print("START " + configs.instance_name)
        time_start = time.time()
        # Generate initial solution
        initial_solution_instance = initial_solution(self.instance_data)
        init_sol, Uc = initial_solution_instance.generate_solution()

        if statistics.time_feasible == 0 and len(Uc) == 0:
            statistics.iteration_feasible = -1
            statistics.time_feasible = time.time() - time_start

        schedule, cost = self.find_optimal_solution(init_sol, Uc)

        rows = []

        for i in range(self.instance_data.days):
            for j in range(self.instance_data.periods_per_day):
                for k in range(self.instance_data.rooms_count):
                    if schedule[i][j][k] != -1:
                        row = self.instance_data.courses_ids[schedule[i][j][k]] + " " + self.instance_data.rooms[
                            k].id + " " + str(i) + " " + str(
                            j) + '\n'
                        rows.append(row)

        # write formatted input to a file to be used by the next process (max-SAT)
        partial_temp_filename = 'C:/Users/vlere/qq/sol' + configs.instance_name + str(uuid.uuid4())

        with open(partial_temp_filename, 'w+') as f:
            for row in rows:
                f.write(row)

        print("DONE " + configs.instance_name)
        return schedule, cost


def compute_checksum(input: list):

    two_times: int = 0
    three_times: int = 0

    for item in input:
        counter = {}
        for letter in item:
            if letter not in counter:
                counter[letter] = 1
            else:
                counter[letter] += 1
        if 2 in counter.values():
            two_times += 1
        if 3 in counter.values():
            three_times += 1

    checksum: int = two_times * three_times
    return checksum

checklist = [
    'abcdef',
    'bababc',
    'abbcde',
    'abcccd',
    'aabcdd',
    'abcdee',
    'ababab',
]
assert compute_checksum(checklist) == 12

boxes = []
with open('input2.txt') as f:
    for line in f:
        boxes.append(line)

print(compute_checksum(boxes))


import pygame as pyg
import numpy as np
import os

from AI import simulation
from Game import tetris as tet

def activate_scene():
    pyg.font.init()

    current_simulation = simulation.Simulation(
        gen_size=22,
        n_allowed_to_reproduce=5,
        single_reproductions=2,
        max_mutation=1,
        mutation_chance=20,
    )

    GRID_THICKNESS = 2
    SQUARE_SIZE = 40

    X_SQUARES = 10
    Y_SQUARES = 20

    NON_GRID_WIDTH_LEFT = 400
    NON_GRID_WIDTH_RIGHT = 500

    WIDTH, HEIGHT = X_SQUARES * SQUARE_SIZE + GRID_THICKNESS * (
        X_SQUARES - 1
    ) + NON_GRID_WIDTH_LEFT + NON_GRID_WIDTH_RIGHT, Y_SQUARES * SQUARE_SIZE + GRID_THICKNESS * (
        Y_SQUARES - 1
    )

    RIGHT_AREA_OFFSET = WIDTH - NON_GRID_WIDTH_RIGHT

    WIN = pyg.display.set_mode((WIDTH, HEIGHT))
    pyg.display.set_caption("Tetrisonator-10000")

    icon = pyg.image.load(os.path.join("./assets", "Tetris_Logo.png"))
    pyg.display.set_icon(icon)

    # neuron_image = pyg.image.load(os.path.join('assets', "neuron.png"))
    # neuron_image = neuron_image

    grid_surf_vert = pyg.Surface((GRID_THICKNESS, HEIGHT))
    grid_surf_horiz = pyg.Surface(
        (WIDTH - NON_GRID_WIDTH_LEFT - NON_GRID_WIDTH_RIGHT + 2, GRID_THICKNESS)
    )

    grid_surf_vert.fill((50, 50, 50))
    grid_surf_horiz.fill((50, 50, 50))

    tetromino_surf = pyg.Surface((SQUARE_SIZE, SQUARE_SIZE))
    ghost_sprite = pyg.image.load(os.path.join("./assets", "ghost_square.png"))

    colour_by_tetromino_code = {
        1: (0, 255, 255),
        2: (0, 25, 200),
        3: (255, 170, 0),
        4: (255, 255, 0),
        5: (0, 255, 0),
        6: (125, 0, 125),
        7: (255, 0, 0),
    }

    FONT_SIZE = 50
    text_font = pyg.font.SysFont("monospace", FONT_SIZE)

    HOLD_LOCATION = 80

    NEXT_LOCATION = HOLD_LOCATION + 200
    LEVEL_LOCATION = NEXT_LOCATION + 250
    LINES_LOCATION = LEVEL_LOCATION + 150

    GENERATION_LOCATION = 0
    NETWORK_LOCATION = GENERATION_LOCATION + 275

    FPS = 720

    piece_distances = {
        1: (NON_GRID_WIDTH_LEFT - (4 * (SQUARE_SIZE + GRID_THICKNESS))) / 2,
        2: (NON_GRID_WIDTH_LEFT - (3 * (SQUARE_SIZE + GRID_THICKNESS))) / 2,
        3: (NON_GRID_WIDTH_LEFT - (3 * (SQUARE_SIZE + GRID_THICKNESS))) / 2,
        4: (NON_GRID_WIDTH_LEFT - (2 * (SQUARE_SIZE + GRID_THICKNESS))) / 2,
        5: (NON_GRID_WIDTH_LEFT - (3 * (SQUARE_SIZE + GRID_THICKNESS))) / 2,
        6: (NON_GRID_WIDTH_LEFT - (3 * (SQUARE_SIZE + GRID_THICKNESS))) / 2,
        7: (NON_GRID_WIDTH_LEFT - (3 * (SQUARE_SIZE + GRID_THICKNESS))) / 2,
    }

    RED_TUPLE = (255, 60, 60)

    def draw_window(
        grid_lines_vert: list[pyg.Rect],
        grid_lines_horiz: list[pyg.Rect],
        board: tet.TetrisBoard,
        _simulation: simulation.Simulation,
    ):
        # Blitting the ghost
        WIN.fill((0, 0, 0))
        for coord in board.active_piece.ghost.occupying_squares:
            WIN.blit(
                ghost_sprite,
                coord * SQUARE_SIZE
                + coord * GRID_THICKNESS
                + np.array([NON_GRID_WIDTH_LEFT + GRID_THICKNESS, 0]),
            )

        pieces_to_blit = board.all_pieces
        # Blitting all pieces
        for piece in pieces_to_blit:
            for coord in piece.occupying_squares:
                tetromino_surf.fill(colour_by_tetromino_code.get(piece.tetromino_code))
                WIN.blit(
                    tetromino_surf,
                    coord * SQUARE_SIZE
                    + coord * GRID_THICKNESS
                    + np.array([NON_GRID_WIDTH_LEFT + GRID_THICKNESS, 0]),
                )

        # Blitting the grid
        for grid_line in grid_lines_vert:
            WIN.blit(grid_surf_vert, (grid_line.x, grid_line.y))
        for grid_line in grid_lines_horiz:
            WIN.blit(grid_surf_horiz, (grid_line.x, grid_line.y))

        # BLitting the "hold" text
        hold_text_image = text_font.render("Hold", True, (255, 255, 255))
        WIN.blit(
            hold_text_image,
            ((NON_GRID_WIDTH_LEFT - hold_text_image.get_width()) / 2, HOLD_LOCATION),
        )

        # Blitting the held piece
        hold_piece = board.hold
        if hold_piece is not None:
            relative_coords = np.array(np.nonzero(a=hold_piece.base_rotation_grid)).T

            hold_piece_min_distance_from_left = piece_distances.get(
                hold_piece.tetromino_code
            )

            tetromino_surf.fill(colour_by_tetromino_code.get(hold_piece.tetromino_code))
            for coord in relative_coords:
                WIN.blit(
                    tetromino_surf,
                    coord * (SQUARE_SIZE + GRID_THICKNESS)
                    + np.array([hold_piece_min_distance_from_left, HOLD_LOCATION + 80]),
                )

        # Blitting the "Next" text
        hold_text_image = text_font.render("Next", True, (255, 255, 255))
        WIN.blit(
            hold_text_image,
            ((NON_GRID_WIDTH_LEFT - hold_text_image.get_width()) / 2, NEXT_LOCATION),
        )

        # Blitting the next piece
        next_piece = board.next
        relative_coords = np.array(np.nonzero(a=next_piece.base_rotation_grid)).T

        next_piece_min_distance_from_left = piece_distances.get(
            next_piece.tetromino_code
        )

        tetromino_surf.fill(colour_by_tetromino_code.get(next_piece.tetromino_code))
        for coord in relative_coords:
            WIN.blit(
                tetromino_surf,
                coord * (SQUARE_SIZE + GRID_THICKNESS)
                + np.array([next_piece_min_distance_from_left, NEXT_LOCATION + 80]),
            )

        # Blitting the score
        score_image = text_font.render(f"{board.score}", True, (255, 255, 255))
        WIN.blit(score_image, ((NON_GRID_WIDTH_LEFT - score_image.get_width()) / 2, 0))

        # BLitting the level
        level_text_image = text_font.render("level", True, (255, 255, 255))
        WIN.blit(
            level_text_image,
            ((NON_GRID_WIDTH_LEFT - level_text_image.get_width()) / 2, LEVEL_LOCATION),
        )

        level_image = text_font.render(f"{board.level}", True, (255, 255, 255))
        WIN.blit(
            level_image,
            ((NON_GRID_WIDTH_LEFT - level_image.get_width()) / 2, LEVEL_LOCATION + 50),
        )

        # Blitting the line count
        lines_text_image = text_font.render("lines", True, (255, 255, 255))
        WIN.blit(
            lines_text_image,
            ((NON_GRID_WIDTH_LEFT - lines_text_image.get_width()) / 2, LINES_LOCATION),
        )

        lines_count_image = text_font.render(
            f"{board.lines_on_level + (board.level - 1) * 10}", True, (255, 255, 255)
        )
        WIN.blit(
            lines_count_image,
            (
                (NON_GRID_WIDTH_LEFT - lines_count_image.get_width()) / 2,
                LINES_LOCATION + 50,
            ),
        )

        # Blitting the generation and unit
        generation_text_image = text_font.render("generation", True, (255, 255, 255))
        WIN.blit(
            generation_text_image,
            (
                RIGHT_AREA_OFFSET
                + (NON_GRID_WIDTH_RIGHT - generation_text_image.get_width()) / 2,
                GENERATION_LOCATION,
            ),
        )

        generation_image = text_font.render(
            f"{_simulation.generation + 1}", True, (255, 255, 255)
        )
        WIN.blit(
            generation_image,
            (
                RIGHT_AREA_OFFSET
                + (NON_GRID_WIDTH_RIGHT - generation_image.get_width()) / 2,
                GENERATION_LOCATION + 50,
            ),
        )

        unit_text_image = text_font.render("Player", True, (255, 255, 255))
        WIN.blit(
            unit_text_image,
            (
                RIGHT_AREA_OFFSET
                + (NON_GRID_WIDTH_RIGHT - unit_text_image.get_width()) / 2,
                GENERATION_LOCATION + 100,
            ),
        )

        unit_image = text_font.render(
            f"{_simulation.tetting_index + 1}", True, (255, 255, 255)
        )
        WIN.blit(
            unit_image,
            (
                RIGHT_AREA_OFFSET + (NON_GRID_WIDTH_RIGHT - unit_image.get_width()) / 2,
                GENERATION_LOCATION + 150,
            ),
        )

        # Blitting the neural network
        # blit_network(brain=simulation.active_tetting.brain)

        pyg.display.update()

    clock = pyg.time.Clock()
    run = True

    grid_rects_vert = [
        pyg.Rect(x, 0, GRID_THICKNESS, HEIGHT)
        for x in range(
            NON_GRID_WIDTH_LEFT,
            WIDTH - NON_GRID_WIDTH_RIGHT + 3,
            SQUARE_SIZE + GRID_THICKNESS,
        )
    ]
    grid_rects_horiz = [
        pyg.Rect(NON_GRID_WIDTH_LEFT, y, WIDTH, GRID_THICKNESS)
        for y in range(SQUARE_SIZE, HEIGHT, SQUARE_SIZE + GRID_THICKNESS)
    ]

    tetris_board = current_simulation.active_tetting.board
    current_simulation.next_piece()
    t = 1

    while run:
        clock.tick(FPS)

        for event in pyg.event.get():
            if event.type == pyg.QUIT:
                run = False

        current_simulation.update()

        if current_simulation.dead_tetting_flag:
            tetris_board = current_simulation.active_tetting.board

        draw_window(
            grid_lines_vert=grid_rects_vert,
            grid_lines_horiz=grid_rects_horiz,
            board=tetris_board,
            _simulation=current_simulation,
        )

        t += 1

    pyg.quit()

if __name__ == "__main__":
    activate_scene()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
@author: Antje M. Lucas-Moffat
Lucas-Moffat et al., 2022, "Multiple gap-filling for eddy covariance datasets", AgrForMet.
Please cite the paper if using this code.

For a package description:
    See mgf/__init__.py

Functions:
    # gen_index():   Generate index to select data subset from timestamp index
    # def_LUTs():    Define dataframe with look-up tables (LUTs)
    # def_LUT_MDS(): Same as def_LUT() but for MDS only
    # fill_LUT():    Function to fill gaps with LUT
    # calc_IPs():    Calculate interpolation of gaps, two techniques implemented

Note:
    Diurnal interpolation techniques (WDM, FDA, MDA, MDC) are programmed the same way as look-up tables (LUTs).

"""
#%% Initialization
### Imports from python
import numpy as np
import pandas as pd

### Own imports
import mgf

### Options
pd.options.display.max_rows = 20

#%% Function for index generator

def gen_index(t_index, win_days, win_hhs, scenario='none'):
    # gen_index():      Generate index to select data subset from timestamp index
    # t_index:          Time index with time stamp at end of half-hour
    # win_hhs:          Window of ± half-hour blocks at same time of day
    #                   Option win_hhs=-1 for full days
    #                   Option win_hhs=-6 for 3h-periods
    # win_days:         Window of ± adjacent days
    # scenario:         'none' - do nothing (i.e. all half-hours in wanted range)
    #                   'hhs' - wanted index but drop current half-hour (i.e. set to gap)
    #                   'days' - wanted indext but drop half-hours from current day (i.e. set full day to gap)
    # Generator takes into account that flux data is right closed at the end of half-hour
    #
    # FOR TESTING: t_index=data.index[i_g]; win_hhs=5; win_days=0; scenario='hhs'

    index = []
    # Correct time stamp to middle of half-hour to avoid switch of periods/days
    t_index_cor = t_index - pd.DateOffset(hours=0.25)

    if win_hhs == -1: #Full days
        #Add preceding days
        start = t_index_cor - pd.DateOffset(days=win_days)
        #Add succeeding days
        end = t_index_cor + pd.DateOffset(days=win_days)
        index = pd.date_range(start=start.strftime('%Y-%m-%d 00:15:00'), end=end.strftime('%Y-%m-%d 23:45:00'), freq='30min')

    elif win_hhs == -6: #3h-periods
        jjj = pd.DataFrame(index=[t_index_cor],columns=[0]).resample('180min').asfreq()
        index_3h_start = jjj.index[0] + pd.DateOffset(hours=0.25) #Needs also correction
        index = pd.period_range(index_3h_start, periods=6, freq='30min').to_timestamp()
        index_3h = index
        for i_d in range(1,win_days+1):
            #Add preceding days
            index = index.append(index_3h - pd.DateOffset(days=i_d))
            index = index.append(index_3h + pd.DateOffset(days=i_d))

    else: # Half-hour blocks
        #Same day
        index.append(t_index_cor)
        for i_hh in range(1,win_hhs+1):
            index.append(t_index_cor-pd.DateOffset(hours=i_hh*0.5))
            index.append(t_index_cor+pd.DateOffset(hours=i_hh*0.5))
        #Multiple days
        for i_d in range(1,win_days+1):
            #Add preceding days
            ts_index=t_index_cor-pd.DateOffset(days=i_d)
            index.append(ts_index)
            for i_hh in range(1,win_hhs+1):
                index.append(ts_index-pd.DateOffset(hours=i_hh*0.5))
                index.append(ts_index+pd.DateOffset(hours=i_hh*0.5))
            #Add succeeding days
            ta_index=t_index_cor+pd.DateOffset(days=i_d)
            index.append(ta_index)
            for i_hh in range(1,win_hhs+1):
                index.append(ta_index-pd.DateOffset(hours=i_hh*0.5))
                index.append(ta_index+pd.DateOffset(hours=i_hh*0.5))

    # Attention: Mix of DatetimeIndex and Timestamp
    # Sort index and make sure it is DatetimeIndex (and not Timestamp)
    # For win_hhs > 23, unique() removes duplicates from index
    index = pd.to_datetime(index).sort_values().unique()

    if scenario == 'hhs':
        #Drop current half-hour
        index = index.drop(t_index_cor) #append() inplace but drop() not inplace?!
    elif scenario == 'days':
        #Drop current day
        t_start=t_index_cor.strftime('%Y-%m-%d 00:15:00')
        t_end=t_index_cor.strftime('%Y-%m-%d 23:45:00')
        index= index.drop(pd.date_range(t_start, t_end, freq='30min'), errors='ignore') #ignore since not all indices (of full day) might have been included

    # Correct time stamp back to end of half hour
    index = index + pd.DateOffset(hours=0.25)
    return index #DatetimeIndex object

"""
#TESTING
    i_g=64; data.index[i_g]
    gen_index(data.index[i_g], win_days=0, win_hhs=2, scenario='days')
    gen_index(data.index[i_g], 1, 2, 'days')
    gen_index(data.index[i_g], 0, 2, 'hhs')
    gen_index(data.index[i_g], 0, 2, 'none')
"""

#%% Define LUT techniques including MDA, MDC, FDA, WDM

def def_LUTs(var_1='none', range_1=np.nan, var_2='none', range_2=np.nan, var_3='none', range_3=np.nan, var_light='none', var_thres=np.nan):
    # def_LUTs():           Define dataframe with look-up tables (LUTs)
    # var_1, range_1, ...:  Variable names and float ranges for LUT
    # var_light, var_thres: Variable name and threshold for daylight / nighttime
    #
    # LUTs data frame:
    # Technique:            Name of technique
    # WinDayStart:          Window size of ± days to start with
    # WinDayStep:           Number of days to increase window size
    # WinHHs:               Number of half-hours to interpolate
    # CondIFs:              if-conditions to look-up-table or to interpolate for others
    #
    # TESTING: var_1='none'; range_1=np.nan; var_2='none'; range_2=np.nan; var_3='none'; range_3=np.nan; var_light='none'; var_thres=np.nan
    # TESTING: var_light='Rg'; var_thres=5

    LUTs = pd.DataFrame(columns=[], index=['Technique','WinDayStart','WinDayStep','WinHHs','CondIFs'])
    if (var_light != 'none') & np.isfinite(var_thres):
        LUTs['WDM'] = ['WDM', 0, 1, -1, '((data.'+var_light+'[i_g] > '+str(var_thres)+') & (df_window.'+var_light+' > '+str(var_thres)+')) | '+
            '((data.'+var_light+'[i_g] <= '+str(var_thres)+') & (df_window.'+var_light+' <= '+str(var_thres)+'))']
    # For FDA, the data is weighted by data is weighted day/night for the whole win_days window and only per day for win_days=0
    LUTs['FDA_hh6'] = ['FDA_hh6', 0, 1, -6, 'df_window.'+var_light+' != True'] #Dummy, but could be real condition
    LUTs['MDA_hh5'] = ['MDA_hh5', 0, 1, 2, 'df_window.'+var_light+' != True'] #Dummy, but could be real condition
    LUTs['MDC_d3'] = ['MDC_d3', 3, 3, 0, 'df_window.'+var_light+' != True'] #Dummy, but could be real condition
    LUTs['MDC_d7'] = ['MDC_d7', 7, 7, 0, 'df_window.'+var_light+' != True'] #Dummy, but could be real condition
    if (var_1 != 'none') & np.isfinite(range_1):
        LUTs['LUT_V1_d3'] = ['LUT_V1_d3', 3, 3, -1, '(abs(df_window.'+var_1+' - data.'+var_1+'[i_g]) <= '+str(range_1)+')']
        LUTs['LUT_V1_d7'] = ['LUT_V1_d7', 7, 7, -1, '(abs(df_window.'+var_1+' - data.'+var_1+'[i_g]) <= '+str(range_1)+') ']
        if (var_2 != 'none') & np.isfinite(range_2) :
            LUTs['LUT_V1V2_d3'] = ['LUT_V1V2_d3', 3, 3, -1, '(abs(df_window.'+var_1+' - data.'+var_1+'[i_g]) <= '+str(range_1)+') & '+
               '(abs(df_window.'+var_2+' - data.'+var_2+'[i_g]) <= '+str(range_2)+')']
            LUTs['LUT_V1V2_d7'] = ['LUT_V1V2_d7', 7, 7, -1, '(abs(df_window.'+var_1+' - data.'+var_1+'[i_g]) <= '+str(range_1)+') & '+
                   '(abs(df_window.'+var_2+' - data.'+var_2+'[i_g]) <= '+str(range_2)+')']
            if (var_3 != 'none') & np.isfinite(range_3):
                LUTs['LUT_V1V2V3_d3'] = ['LUT_V1V2V3_d3', 3, 3, -1, '(abs(df_window.'+var_1+'- data.'+var_1+'[i_g]) <= '+str(range_1)+') & '+
                       '(abs(df_window.'+var_2+' - data.'+var_2+'[i_g]) <= '+str(range_2)+') & '+
                       '(abs(df_window.'+var_3+' - data.'+var_3+'[i_g]) <= '+str(range_3)+')']
                LUTs['LUT_V1V2V3_d7'] = ['LUT_V1V2V3_d7', 7, 7, -1, '(abs(df_window.'+var_1+' - data.'+var_1+'[i_g]) <= '+str(range_1)+') & '+
                       '(abs(df_window.'+var_2+' - data.'+var_2+'[i_g]) <= '+str(range_2)+') & '+
                       '(abs(df_window.'+var_3+' - data.'+var_3+'[i_g]) <= '+str(range_3)+')']
    return LUTs

def def_LUT_MDS(var_1='none', range_1=np.nan, var_2='none', range_2=np.nan, var_3='none', range_3=np.nan):
    # def_LUT_MDS():    Same as above but for defining MDS LUT setup only

    LUTs = pd.DataFrame(columns=[], index=['Technique','WinDayStart','WinDayStep','WinHHs','CondIFs'])
    if (var_1 != 'none') & np.isfinite(range_1) & (var_2 != 'none') & np.isfinite(range_2) & (var_3 != 'none') & np.isfinite(range_3):
        LUTs['LUT_MDS_d7'] = ['LUT_MDS_d7', 7, 7, -1, '(abs(df_window.'+var_1+' - data.'+var_1+'[i_g]) <= '+str(range_1)+') & '+
                           '(abs(df_window.'+var_2+' - data.'+var_2+'[i_g]) <= '+str(range_2)+') & '+
                           '(abs(df_window.'+var_3+' - data.'+var_3+'[i_g]) <= '+str(range_3)+')']
    elif (var_1 != 'none') & np.isfinite(range_1) & (var_2 != 'none') & np.isfinite(range_2) & (var_3 == 'none'): #If 3rd variable (VPD) is missing as in NH3 dataset
        LUTs['LUT_MDS_d7'] = ['LUT_MDS_d7', 7, 7, -1, '(abs(df_window.'+var_1+' - data.'+var_1+'[i_g]) <= '+str(range_1)+') & '+
                           '(abs(df_window.'+var_2+' - data.'+var_2+'[i_g]) <= '+str(range_2)+')']
    return LUTs

def fill_LUT(data, flux_org, methLUT, scenario):
    # fill_LUT():       Function to fill gaps with LUT techniques
    # data:             Dataframe with timestamp index, flux measurements, and maybe meteo
    # flux_org:         Data column with original measured fluxes to be filled/used and rest set to nan
    # methLUT:          Gap filling technique as defined in the dataframe 'LUTs' above
    # scenario:         Scenario for gap filling ('hhs', 'days', or 'none', see also gen_index())
    #
    # TESTING: data=data_co2; ini=co2.Settings; flux_org = gen_fcol(data,ini); methLUT=LUTs['WDM']; scenario='hhs'

    flux_fm = methLUT.Technique +'_'+ scenario #Flux filled with fill-technique
    print('>>> Fill technique:', flux_fm, '   (', methLUT.CondIFs,')')

    # Write gap filled values into new column
    data[flux_fm] = np.nan
    win_days = methLUT.WinDayStart
    num_gaps = data[flux_fm].isnull().sum()
    num_gaps_org = data[flux_fm].isnull().sum()
    while num_gaps > 0:
        check_gaps = data[flux_fm].isnull()
        array_gaps = np.array(range(0,data.shape[0]))[np.array(check_gaps)]
        # Calculate LUT value
        #print('>', win_days, check_gaps.sum())
        for i_g in array_gaps:
        #TESTING: for i_g in array_gaps[0:5]:
            df_window = data[data.index.isin(gen_index(data.index[i_g], win_days=win_days, win_hhs=methLUT.WinHHs, scenario=scenario))]
            #--> Much faster to first reduce dataset to window BEFORE compound-if-query
            lut_entries = eval(methLUT.CondIFs)
            if (lut_entries.sum() >= 2): #if more than two flux values available (after MR only, difference to EF algorithm)
                data[flux_fm].iat[i_g] = np.mean(df_window[flux_org][lut_entries])

        # Increase window of days and break if too large
        #Print remaining gaps for comparison with full MDS algorithm
        num_gaps =  data[flux_fm].isnull().sum()
        if (methLUT.Technique == 'LUT_MDS_d7'):
           print('>LUT_MDS: window size:', win_days, ', remaining gaps:', num_gaps, ' of', num_gaps_org,', in percent: ', '{:04.2f}'.format(num_gaps/num_gaps_org*100))

        win_days = win_days + methLUT.WinDayStep
        tot_days = int(data.shape[0] / 48.0)
        if (win_days > 0.5 * tot_days) & (num_gaps > 0):
           print('>! ', num_gaps, 'remaining gaps. Break since window size: ', win_days, ' is larger than half of total days: ', 0.5*tot_days, '.)')
           break

 #%% Define interpolation techniques

def calc_IPs(data, flux_org, technique, scenario):
    # calc_IPs():       Calculate interpolation of gaps, two techniques implemented
    # data:             Dataframe with timestamp index, flux measurements, and maybe meteo
    # flux_org:         Data column with original measured fluxed to be filled/used and rest set to nan
    # technique:        Interpolation technique 'IP_lin' or 'IP_mov'
    # scenario:         Scenario for gap filling ('hhs', 'days', or 'none', see also gen_index())
    #
    # TESTING: flux_org=mgf.utl.name_flux(ini); technique='IP_mov'; scenario='hhs'; scenario='days';

    flux_fm = technique +'_'+ scenario #Flux filled with fill-technique
    print('>>> Fill technique:', flux_fm)
    data[flux_fm]=np.nan
    # For linear interpolation, all real gaps could be filled at once.
    # Loop only need to simulate artificial gaps, each at a time.
    # Limit interpolation to certain day window to speed up processing.
    win_days = 1 #must be at least ±1 for linear interpolation and also for days scenario
    gapl_S = mgf.utl.set_gapl_S() #pre-definded length of short gaps
    num_gaps = data[flux_fm].isnull().sum()
    while num_gaps > 0:
        check_gaps = data[flux_fm].isnull()
        array_gaps = np.array(range(0,data.shape[0]))[np.array(check_gaps)]
        # Calculate IP value
        #print('>', win_days, check_gaps.sum())
        for i_g in array_gaps:
        #TESTING: for i_g in array_gaps[0:20]: # for i_g in array_gaps[16696:(16696+20)]: # i_g = 0
            # limit = maximum number of consecutive values set to (win_days*48) to ensures that gap does not range to end of ±win_days period
            df_window = data[data.index.isin(gen_index(data.index[i_g], win_days=win_days, win_hhs=-1, scenario='none'))] #Continuous index without scenario
            entries = df_window.index.isin(gen_index(data.index[i_g], win_days=win_days, win_hhs=-1, scenario=scenario)) #Index with scenario
            df_window.insert(0,'data',df_window[flux_org].where(entries))
            df_window.insert(1,'gaps',mgf.utl.countGapsTot(df_window,'data'))
           # Linear interpolation in two steps
            # Attention: 'limit' does not limit gap size but only how many values are interpolated and 'inside','outside' did not work as expected...
            if technique == 'IP_lin':
                if (df_window['gaps'].loc[data.index[i_g]] <= gapl_S) : #Gaps are filled if short, at least start and end are available
                # Interpolate with pandas.interpolate()
                    data[flux_fm].iat[i_g] = df_window['data'].interpolate(technique='time', axis=0, limit=gapl_S, limit_direction='both').loc[data.index[i_g]] #So cool, single line routine!
#                elif (df_window['gaps'].loc[data.index[i_g]] < gapl_S) : #If within gapl_S interpolate with quarterly means
#                    df_window.is_copy = False
#                    # Calculate fixed 3h-mean and interpolate missing 3h-means
#                    df_window['fix_3h'] = df_window['data'].groupby(pd.Grouper(freq='180min',closed='right',label='left')).apply(lambda g: g.mean()).interpolate(technique='time', axis=0, limit=win_days, limit_direction='both')
#                    # Propagate NEXT valid observation backwards to fill gap (i.e. pure propagation, no further interpolation)
#                    data[flux_fm].iat[i_g] = df_window['fix_3h'].bfill().loc[data.index[i_g]]
                else: # Otherwise use daily means
                    df_win_days = df_window['data'].groupby(pd.Grouper(freq='1d',closed='right',label='left')).apply(lambda g: g.mean()).interpolate(technique='time', axis=0, limit=win_days, limit_direction='both')
                    data[flux_fm].iat[i_g] = df_win_days.loc[(data.index[i_g]- pd.DateOffset(hours=0.25)).strftime('%Y-%m-%d')] ## Use shifted time stamp to middle of half-hour to avoid midnight switch
            elif technique == 'IP_mov': #Moving average interpolation
                if (df_window['gaps'].loc[data.index[i_g]] <= gapl_S):   # Only up to gapl_S hh as for IP_lin
                # Rolling = moving window with increasing window size.
                # Moving average (= mean) to fill wholes in time series (= interpolation).
                    roll_start = 5
                    roll_step = 2
                    min_hh = 2 #Minimum of available half-hours
                    window = roll_start
                    data[flux_fm].iat[i_g] = df_window['data'].rolling(window, center=True, min_periods=min_hh).mean().loc[data.index[i_g]]
                    #while np.isnan(data[flux_fm].iat[i_g]) & ((window*0.5) < (win_days*48)): # Since center=True option, half the window critical
                    while np.isnan(data[flux_fm].iat[i_g]):
                        window = window + roll_step
                        data[flux_fm].iat[i_g] = df_window['data'].rolling(window, center=True, min_periods=min_hh).mean().loc[data.index[i_g]]
                else:
                    min_days=2 #At least 2 for center of three days
                    df_win_days = df_window['data'].groupby(pd.Grouper(freq='1d',closed='right',label='left')).apply(lambda g: g.mean()).rolling(window=min_days+win_days, center=True, min_periods=min_days).mean()
                    data[flux_fm].iat[i_g] = df_win_days.loc[(data.index[i_g]- pd.DateOffset(hours=0.25)).strftime('%Y-%m-%d')] ## Use shifted time stamp to middle of half-hour to avoid midnight switch
        # Increase window of days and break if too large
        win_days = win_days + 1
        num_gaps =  data[flux_fm].isnull().sum()
        if (win_days > 10) & (num_gaps > 0):
           print('>!', num_gaps, 'remaining gaps. Break since window size: ', win_days, ' is larger than ±10 for interpolations.)')
           break


import logging
import requests
import json
import sys
import config
from pathlib import Path
from abc import ABC, abstractmethod
from utils import convert_webp_webm, convert_webp_png, check_directories, progressBar

logger = logging.getLogger("run")

class BaseEmote(ABC):
    def __init__(self, url, all_variants: bool = False):
        self.url: str = url
        self.emote_id: str = ""
        self.name: str = ""
        self.animated: bool = False
        self.api_emote_url: str = ""
        self.cdn_url: str = ""
        self.cdn_file_name = ""
        self.filepath = None
        self.all_variants = all_variants

    @classmethod
    @abstractmethod
    def get_emote_details(cls) -> None:
        pass

    @classmethod
    @abstractmethod
    def download_emote(cls) -> None:
        pass

    @classmethod
    @abstractmethod
    def get_emote_id(cls):
        pass

    def process_emote(self):
        self.get_emote_id()
        self.get_emote_details()
        self.download_emote()
        if self.animated:
            convert_webp_webm(self.filepath, self.name)
        else:
            convert_webp_png(self.filepath, self.name)

class SevenTVEmote(BaseEmote):
    def __init__(self, url):
        super().__init__(url)
        self.api_emote_url = "https://7tv.io/v3/emotes/"
        self.cdn_url = "http://cdn.7tv.app/emote/"
        self.cdn_file_name = "4x.webp"

    def get_emote_id(self):
        self.emote_id = self.url.split("/")[-1]

    def get_emote_details(self):
        url = self.api_emote_url + self.emote_id
        response = requests.get(url)
        if response.status_code == 200:
            response_dict = json.loads(response.content.decode("utf-8"))
            self.name = response_dict["name"]
            self.animated = response_dict["animated"]
        else:
            logger.error(f"Failed to get emote name of id: {self.emote_id}. Status code: {response.status_code}")

    def download_emote(self):
        download_url = self.cdn_url + self.emote_id + "/" + self.cdn_file_name
        self.filepath = Path(config.TEMP_FOLDER + self.name + "_" + self.cdn_file_name)
        response = requests.get(download_url)
        if response.status_code == 200:
            with open(self.filepath, "wb") as file:
                file.write(response.content)
            logger.info(f"Emote file with id {self.emote_id} downloaded successfully.")
        else:
            logger.error(f"Failed to download emote file with id {self.emote_id}. Status code:", response.status_code)

class BetterTTVEmote(BaseEmote):
    def __init__(self, url):
        super().__init__(url)
        self.api_emote_url = "https://api.betterttv.net/3/emotes/"
        self.cdn_url = "https://cdn.betterttv.net/emote/"
        self.cdn_file_name = "3x.webp"

    def get_emote_details(self):
        url = self.api_emote_url + self.emote_id
        response = requests.get(url)
        if response.status_code == 200:
            response_dict = json.loads(response.content.decode("utf-8"))
            self.name = response_dict["code"]
            self.animated = response_dict["animated"]
        else:
            logger.error(f"Failed to get emote name of id: {self.emote_id}. Status code: {response.status_code}")

    def download_emote(self):
        download_url = self.cdn_url + self.emote_id + "/" + self.cdn_file_name
        self.filepath = Path(config.TEMP_FOLDER + self.name + "_" + self.cdn_file_name)
        response = requests.get(download_url)
        if response.status_code == 200:
            with open(self.filepath, "wb") as file:
                file.write(response.content)
            logger.info(f"Emote file with id {self.emote_id} downloaded successfully.")
        else:
            logger.error(f"Failed to download emote file with id {self.emote_id}. Status code:", response.status_code)

    def get_emote_id(self):
        self.emote_id = self.url.split("/")[-1]

class FrankFaseZEmote(BaseEmote):
    def __init__(self, url):
        super().__init__(url)
        self.api_emote_url = "https://api.frankerfacez.com/v2/emote/"
        self.cdn_url = "https://cdn.frankerfacez.com/emote/"
        self.cdn_file_name = ""

    def get_emote_details(self):
        url = self.api_emote_url + self.emote_id
        response = requests.get(url)
        if response.status_code == 200:
            response_dict = json.loads(response.content.decode("utf-8"))
            self.name = response_dict["emote"]["name"]
            self.animated = response_dict["emote"]["animated"]
        else:
            logger.error(f"Failed to get emote name of id: {self.emote_id}. Status code: {response.status_code}")

    def download_emote(self):
        if self.animated:
            download_url = self.cdn_url + self.emote_id + "/animated/4"
        else:
            download_url = self.cdn_url + self.emote_id + "/4"
        self.filepath = Path(config.TEMP_FOLDER + self.name + ".webp")
        response = requests.get(download_url)
        if response.status_code == 200:
            with open(self.filepath, "wb") as file:
                file.write(response.content)
            logger.info(f"Emote file with id {self.emote_id} downloaded successfully.")
        else:
            logger.error(f"Failed to download emote file with id {self.emote_id}. Status code:", response.status_code)

    def get_emote_id(self):
        self.emote_id = self.url.split("/")[-1].split("-")[0]

def main():

    urls = []
    urls_file = "emote_links.txt"
    with open(urls_file, "r") as f:
        urls.extend([line.strip() for line in f.readlines()])

    logger.info(f"Links: {urls}")
    if "--all" in sys.argv:
        config.ALL_VARIANTS = True
    if urls:
        check_directories()
    for url in progressBar(urls, prefix="Progress:", suffix="Complete", length=50):
        try:
            if "7tv.app" in url:
                emote = SevenTVEmote(url)
            elif "betterttv.com" in url:
                emote = BetterTTVEmote(url)
            elif "frankerfacez.com" in url:
                emote = FrankFaseZEmote(url)
            else:
                continue
            emote.process_emote()
        except Exception as e:
            logger.exception(f"Exception caught while processing {url}: {e}")

# TODO: add poetry
# TODO: update calling with args
if __name__ == "__main__":
    if sys.argv:
        logger.info(f"Args: {sys.argv}")
    main()


import sys
import time

from mercurial import util

def spacejoin(*args):
    return ' '.join(s for s in args if s)

def shouldprint(ui):
    return (getattr(sys.stderr, 'isatty', None) and
            (sys.stderr.isatty() or ui.configbool('progress', 'assume-tty')))

class progbar(object):
    def __init__(self, ui):
        self.ui = ui
        self.resetstate()

    def resetstate(self):
        self.topics = []
        self.printed = False
        self.lastprint = time.time() + float(self.ui.config(
            'progress', 'delay', default=3))
        self.indetcount = 0
        self.refresh = float(self.ui.config(
            'progress', 'refresh', default=0.1))
        self.order = self.ui.configlist(
            'progress', 'format',
            default=['topic', 'bar', 'number'])

    def show(self, topic, pos, item, unit, total):
        if not shouldprint(self.ui):
            return
        termwidth = self.width()
        self.printed = True
        head = ''
        needprogress = False
        tail = ''
        for indicator in self.order:
            add = ''
            if indicator == 'topic':
                add = topic
            elif indicator == 'number':
                if total:
                    add = ('% ' + str(len(str(total))) +
                           's/%s') % (pos, total)
                else:
                    add = str(pos)
            elif indicator.startswith('item') and item:
                slice = 'end'
                if '-' in indicator:
                    wid = int(indicator.split('-')[1])
                elif '+' in indicator:
                    slice = 'beginning'
                    wid = int(indicator.split('+')[1])
                else:
                    wid = 20
                if slice == 'end':
                    add = item[-wid:]
                else:
                    add = item[:wid]
                add += (wid - len(add)) * ' '
            elif indicator == 'bar':
                add = ''
                needprogress = True
            elif indicator == 'unit' and unit:
                add = unit
            if not needprogress:
                head = spacejoin(head, add)
            else:
                tail = spacejoin(add, tail)
        if needprogress:
            used = 0
            if head:
                used += len(head) + 1
            if tail:
                used += len(tail) + 1
            progwidth = termwidth - used - 3
            if total and pos <= total:
                amt = pos * progwidth // total
                bar = '=' * (amt - 1)
                if amt > 0:
                    bar += '>'
                bar += ' ' * (progwidth - amt)
            else:
                progwidth -= 3
                self.indetcount += 1
                # mod the count by twice the width so we can make the
                # cursor bounce between the right and left sides
                amt = self.indetcount % (2 * progwidth)
                amt -= progwidth
                bar = (' ' * int(progwidth - abs(amt)) + '<=>' +
                       ' ' * int(abs(amt)))
            prog = ''.join(('[', bar , ']'))
            out = spacejoin(head, prog, tail)
        else:
            out = spacejoin(head, tail)
        sys.stderr.write('\r' + out[:termwidth])
        sys.stderr.flush()

    def clear(self):
        if not shouldprint(self.ui):
            return
        sys.stderr.write('\r%s\r' % (' ' * self.width()))

    def complete(self):
        if not shouldprint(self.ui):
            return
        if self.ui.configbool('progress', 'clear-complete', default=True):
            self.clear()
        else:
            sys.stderr.write('\n')
        sys.stderr.flush()

    def width(self):
        tw = self.ui.termwidth()
        return min(int(self.ui.config('progress', 'width', default=tw)), tw)

    def progress(self, topic, pos, item='', unit='', total=None):
        if pos is None:
            if self.topics and self.topics[-1] == topic and self.printed:
                self.complete()
                self.resetstate()
        else:
            if topic not in self.topics:
                self.topics.append(topic)
            now = time.time()
            if (now - self.lastprint >= self.refresh
                and topic == self.topics[-1]):
                self.lastprint = now
                self.show(topic, pos, item, unit, total)

def uisetup(ui):
    class progressui(ui.__class__):
        _progbar = None

        def progress(self, *args, **opts):
            self._progbar.progress(*args, **opts)
            return super(progressui, self).progress(*args, **opts)

        def write(self, *args, **opts):
            if self._progbar.printed:
                self._progbar.clear()
            return super(progressui, self).write(*args, **opts)

        def write_err(self, *args, **opts):
            if self._progbar.printed:
                self._progbar.clear()
            return super(progressui, self).write_err(*args, **opts)

    # Apps that derive a class from ui.ui() can use
    # setconfig('progress', 'disable', 'True') to disable this extension
    if ui.configbool('progress', 'disable'):
        return
    if shouldprint(ui) and not ui.debugflag and not ui.quiet:
        ui.__class__ = progressui
        # we instantiate one globally shared progress bar to avoid
        # competing progress bars when multiple UI objects get created
        if not progressui._progbar:
            progressui._progbar = progbar(ui)

def reposetup(ui, repo):
    uisetup(repo.ui)