"""Default save class."""
import abc
import copy
import gc
import json
import logging
import os
from typing import Any, Dict, Optional, Union, Callable, Mapping

import numpy as np

from brain_pipe.pipeline.cache.base import (
    pickle_dump_wrapper,
    pickle_load_wrapper,
)
from brain_pipe.save.base import Save
from brain_pipe.utils.list import wrap_in_list
from brain_pipe.utils.multiprocess import MultiprocessingSingleton

# Shorthand interfaces.
CheckInterface = Callable[[Dict[str, Any], str, Dict[str, Any]], Union[str, bool]]
FilenameFnInterface = Callable[[Dict[str, Any], Optional[str], Optional[str]], str]
SaveFnInterface = Union[
    Callable[[Any, str], None], Mapping[str, Callable[[Any, str], None]], None
]
ReloadFnInterface = Union[
    Callable[[str], Any], Mapping[str, Callable[[str], Any]], None
]

def default_metadata_key_fn(data_dict: Dict[str, Any]) -> str:
    """Generate a key for the metadata.

    Parameters
    ----------
    data_dict: Dict[str, Any]
        The data dict containing the data to save.

    Returns
    -------
    str
        The key for the metadata.
    """
    if "data_path" in data_dict:
        return os.path.basename(data_dict["data_path"])

    if "stimulus_path" in data_dict and data_dict["stimulus_path"] is not None:
        return os.path.basename(data_dict["stimulus_path"])

    if "trigger_path" in data_dict and data_dict["trigger_path"] is not None:
        return os.path.basename(data_dict["trigger_path"])

    raise ValueError("No data_path or stimulus_path in data_dict.")

class DefaultFilenameFn(FilenameFnInterface):
    """Default filename function to create paths to save data."""

    SPLIT_CHAR = "/"

    def __init__(
        self,
        path_keys=("data_path", "stimulus_path"),
        other_keys=("event_info/snr",),
        separator="_-_",
        data_dict_extension=".data_dict",
        feature_extension=".npy",
    ):
        """Create a new DefaultFilenameFn instance.

        Parameters
        ----------
        path_keys: Sequence[str]
            The keys of the paths to include in the filename.
        other_keys: Sequence[str]
            The keys of other data to include in the filename.
        separator: str
            The separator to use between parts of the filename.
        data_dict_extension: str
            The extension to use when saving the entire data_dict.
        feature_extension: str
            The extension to use when saving a single feature.
        """
        self.path_keys = path_keys
        self.other_keys = other_keys
        self.separator = separator
        self.data_dict_extension = data_dict_extension
        self.feature_extension = feature_extension

    def __call__(self, data_dict, feature_name=None, set_name=None):
        """Generate a filename for the data_dict.

        Parameters
        ----------
        data_dict: Dict[str, Any]
            The data dict containing the data to save.
        feature_name: Optional[str]
            The name of the feature.
        set_name: Optional[str]
            The name of the set. If no set name is given, the set name is not
            included in the filename.

        Returns
        -------
        str
            The filename.
        """
        parts = []

        # Add indicated path keys to filename
        for path_key in self.path_keys:
            if path_key in data_dict:
                parts.append(
                    "".join(os.path.basename(data_dict[path_key]).rsplit(".")[0])
                )
        # If no feature name or set name is given, return the data_dict filename.
        if feature_name is None and set_name is None:
            return self.separator.join(parts) + self.data_dict_extension

        # Add indicated other keys to data
        for full_key in self.other_keys:
            item = data_dict
            append = True
            for key in full_key.split(self.SPLIT_CHAR):
                if key not in item:
                    append = False
                    break
                item = item[key]
            if append:
                # Has to be added as a string to be able to be used in filename
                parts.append(str(item))

        # Add the feature name as the last part of the filename, if given
        if feature_name is not None:
            parts.append(feature_name)
        # Add the set name as the first part of the filename, if given
        if set_name is not None:
            parts.insert(0, set_name)
        return self.separator.join(parts) + self.feature_extension

class AttachSave(abc.ABC):
    """Mixin class to attach a Save object."""

    def __init__(self, saver=None):
        """Initialize the AttachSaver.

        Parameters
        ----------
        saver: Optional[Save]
            The saver to use. Can be attached later with :meth:`attach_saver`, but
            that will create a new object.

        """
        self.saver = saver

    def attach_saver(self, saver):
        """Initialize a new object with the saver.

        Parameters
        ----------
        saver: Save
            The saver.
        """
        new_ = copy.deepcopy(self)
        new_.saver = saver
        return new_

class CheckFunctor(CheckInterface, AttachSave, abc.ABC):
    """Functor to use with DefaultSave to check something about of a metadata item."""

    @abc.abstractmethod
    def __call__(self, metadata_item, feature_name, data_dict):
        """Check something about the metadata item.

        Parameters
        ----------
        metadata_item: Dict[str, Any]
            The metadata item. Will have keys :attr:`DefaultSave.FILENAME_STR`,
            :attr:`DefaultSave.FEATURE_NAME_STR`, :attr:`DefaultSave.SET_NAME_STR` and
            optionally :attr:`DefaultSave.OLD_FORMAT_STR`
        feature_name: Optional[str]
            The name of the feature.
        data_dict: Dict[str, Any]
            The data dict containing the data to save.

        Returns
        -------
        Union[Any, bool, None]
            False if the data does not pass the check, otherwise a useful representation
            of the data (such as the path). If None is returned, the check was not
            applicable.
        """
        pass

class IsDoneCheck(CheckFunctor):
    """Check if data has already been saved."""

    def __call__(self, metadata_item, feature_name, data_dict):
        """Check if data has already been saved.

        Parameters
        ----------
        metadata_item: Dict[str, Any]
            The metadata item. Will have keys :attr:`DefaultSave.FILENAME_STR`,
            :attr:`DefaultSave.FEATURE_NAME_STR`, :attr:`DefaultSave.SET_NAME_STR` and
            optionally :attr:`DefaultSave.OLD_FORMAT_STR`
        feature_name: Optional[str]
            The name of the feature.

        Returns
        -------
        Union[str, bool, None]
            False if the data has not been saved.
            None if the check was not applicable for the given feature name.
            The path to the data otherwise

        """
        if metadata_item[self.saver.metadata.FEATURE_NAME_STR] == feature_name:
            path = os.path.join(
                self.saver.root_dir, metadata_item[self.saver.metadata.FILENAME_STR]
            )
            return path if os.path.exists(path) else False

class IsReloadableCheck(CheckFunctor):
    """Check if data has already been saved and is reloadable."""

    def __call__(
        self,
        metadata_item: Dict[str, Any],
        feature_name: Optional[str],
        data_dict: Dict[str, Any],
    ):
        """Check if data has already been saved and is reloadable.

        Parameters
        ----------
        metadata_item: Dict[str, Any]
            The metadata item. Will have keys :attr:`DefaultSave.FILENAME_STR`,
            :attr:`DefaultSave.FEATURE_NAME_STR`, :attr:`DefaultSave.SET_NAME_STR` and
            optionally :attr:`DefaultSave.OLD_FORMAT_STR`.
        feature_name: Optional[str]
            The name of the feature.
        data_dict: Dict[str, Any]
            The data dict containing the unprocessed data

        Returns
        -------
        Union[str, bool, None]
            False if the data is not reloadable.
            None if the check was not applicable for the given feature name.
            The path to the data otherwise.

        """
        if metadata_item[self.saver.metadata.FEATURE_NAME_STR] is None:
            is_old_format = self.saver.metadata.is_old_format(metadata_item)
            filename = metadata_item[self.saver.metadata.FILENAME_STR]
            expected_filename = os.path.relpath(
                self.saver.filename_fn(data_dict, None, None), self.saver.root_dir
            )
            # If old format and not the expected filename to be reloadable
            # then skip
            if is_old_format and filename != expected_filename:
                return False

            path = os.path.join(
                self.saver.root_dir, metadata_item[self.saver.metadata.FILENAME_STR]
            )
            return path if os.path.exists(path) else False

class SaveMetadata(AttachSave, abc.ABC):
    """Abstract class for metadata to use when saving/reloading."""

    FEATURE_NAME_STR = "feature_name"
    FILENAME_STR = "filename"
    SET_NAME_STR = "set_name"
    OLD_FORMAT_STR = "old_format"

    def __init__(
        self, key_fn: Callable[[Dict[str, Any]], str] = default_metadata_key_fn
    ):
        """Create a new SaveMetadata.

        Parameters
        ----------
        key_fn: Callable[[Dict[str, Any]], str]
            The function to use to get the key from the data dict.
        """
        super().__init__()
        self.key_fn = key_fn

    @abc.abstractmethod
    def clear(self):
        """Clear the metadata."""

    @abc.abstractmethod
    def add(
        self,
        data_dict: Dict[str, Any],
        filepath: str,
        feature_name: Optional[str],
        set_name: Optional[str],
    ):
        """Add a metadata entry.

        Parameters
        ----------
        data_dict: Dict[str, Any]
            The data dict containing the data to save.
        filepath: str
            The path to the data.
        feature_name: Optional[str]
            The name of the feature.
        set_name: Optional[str]
            The name of the set.
        """

    @abc.abstractmethod
    def __contains__(self, item):
        """Check if the metadata contains a certain item.

        Parameters
        ----------
        item: Any
            The item to check.

        Returns
        -------
        bool
            Whether the item is contained.
        """

    @abc.abstractmethod
    def __getitem__(self, key: Any):
        """Retrieve a metadata item.

        Parameters
        ----------
        key: Any
            The key to retrieve.

        Returns
        -------
        Dict[str, Any]
            The metadata item.
        """

    @classmethod
    def is_old_format(cls, metadata_item: Union[str, Dict[str, Any]]):
        """Check if the metadata item is in the old format.

        Parameters
        ----------
        metadata_item: Union[str, Dict[str, Any]]
            The metadata item to check.

        Returns
        -------
        bool
            Whether the metadata item is in the old format.
        """
        return isinstance(metadata_item, str) or cls.OLD_FORMAT_STR in metadata_item

class OldMetadataCompliant(abc.ABC):
    """Mixin class for metadata that is compliant with the old format."""

    @abc.abstractmethod
    def convert_old_format(self, metadata_item: str, data_dict: Dict[str, Any]):
        """Convert the metadata item from the old format.

        Parameters
        ----------
        metadata_item: str
            The metadata item to convert.

        data_dict: Dict[str, Any]
            The data dict containing the data to save.

        Returns
        -------
        Dict[str, Any]
            The converted metadata item.
        """

class DefaultSaveMetadata(OldMetadataCompliant, SaveMetadata):
    """Implementation of SaveMetadata to work with DefaultSave."""

    def __init__(
        self,
        key_fn: Callable[[Dict[str, Any]], str] = default_metadata_key_fn,
        filename: str = ".save_metadata.json",
    ):
        """Create a new DefaultSaveMetadata.

        Parameters
        ----------
        key_fn: Callable[[Dict[str, Any]], str]
            The function to use to get the key from the data dict.
        filename: str
            The filename to use for the metadata.
        """
        super().__init__(key_fn=key_fn)
        self.filename = filename
        self.saver = None

    def get_path(self):
        """Get the path to the metadata file.

        Returns
        -------
        str
            The path to the metadata file.
        """
        if self.saver is not None and isinstance(self.saver, DefaultSave):
            path = os.path.join(self.saver.root_dir, self.filename)
            return path
        return self.filename

    def get_relpath(self, path: str):
        """Construct a relative path with regard to save folder.

        Parameters
        ----------
        path: str
            The path to make relative. If no saver is attached, this is returned as is.

        Returns
        -------
        str
            The relative path.
        """
        if (
            self.saver is not None
            and isinstance(self.saver, DefaultSave)
            and os.path.isabs(path)
        ):
            return os.path.relpath(path, self.saver.root_dir)
        return path

    @property
    def lock(self):
        """Retrieve the lock to use for the metadata file.

        Returns
        -------
        multiprocessing.Lock
            The lock to use for the metadata file.
        """
        return MultiprocessingSingleton.get_lock(self.get_path())

    def clear(self):
        """Clear the metadata."""
        self.lock.acquire()
        metadata_path = self.get_path()
        if os.path.exists(metadata_path):
            os.remove(metadata_path)
        self.lock.release()

    def get_metadata_for_savepath(
        self,
        path: str,
        feature_name: Optional[str],
        set_name: Optional[str],
        from_old_format=False,
    ):
        """Get the metadata associated for path where data is saved.

        Parameters
        ----------
        path: str
            The path to the data.
        feature_name: Optional[str]
            The name of the feature.
        set_name: Optional[str]
            The name of the set.
        from_old_format: bool
            Whether the metadata is in the old format.

        Returns
        -------
        Dict[str, Any]
            The metadata associated with the path.
        """
        metadata = {
            self.FILENAME_STR: self.get_relpath(path),
            self.FEATURE_NAME_STR: feature_name,
            self.SET_NAME_STR: set_name,
        }
        if from_old_format:
            metadata[self.OLD_FORMAT_STR] = True
        return metadata

    def convert_old_format(self, metadata_item: str, data_dict: Dict[str, Any]):
        """Convert the metadata item from the old format.

        Parameters
        ----------
        metadata_item: str
            The metadata item to convert.
        data_dict: Dict[str, Any]
            The data dict containing the data to save.

        Returns
        -------
        Dict[str, Any]
            The converted metadata item.
        """
        return self.get_metadata_for_savepath(metadata_item, None, None, True)

    def get(self):
        """Load the metadata.

        Returns
        -------
        Dict[str, Any]
            The metadata.
        """
        metadata_path = self.get_path()
        if not os.path.exists(metadata_path):
            return {}
        self.lock.acquire()
        with open(metadata_path) as fp:
            metadata = json.load(fp)
        self.lock.release()
        return metadata

    def add(
        self,
        data_dict: Dict[str, Any],
        filepath: str,
        feature_name: Optional[str],
        set_name: Optional[str],
    ):
        """Add metadata for a file.

        Parameters
        ----------
        data_dict: Dict[str, Any]
            The data dictionary.
        filepath: str
            The path to the file.
        feature_name: Optional[str]
            The name of the feature.
        set_name: Optional[str]
            The name of the set.
        """
        metadata = self.get()
        key = self.key_fn(data_dict)
        if key not in metadata:
            metadata[key] = []
        all_filepaths = wrap_in_list(filepath)
        for path in all_filepaths:
            metadata_for_savepath = self.get_metadata_for_savepath(
                path, feature_name, set_name
            )
            if metadata_for_savepath not in metadata[key]:
                metadata[key] += [metadata_for_savepath]
        self.write(metadata)

    def write(self, metadata_dict: Dict[str, Any]):
        """Write the metadata to disk.

        Parameters
        ----------
        metadata_dict: Dict[str, Any]
            A dictionary containing the metadata.
        """
        self.lock.acquire()
        with open(self.get_path(), "w") as fp:
            json.dump(metadata_dict, fp)
        self.lock.release()

    def __contains__(self, item: Any):
        """Check if the metadata contains a certain item.

        Parameters
        ----------
        item: Any

        Returns
        -------
        bool
            Whether the item is contained.
        """
        return item in self.get()

    def __getitem__(self, key: Any):
        """Retrieve a metadata item.

        Parameters
        ----------
        key: Any
            The key to retrieve.

        Returns
        -------
        Any
            The metadata item.
        """
        return self.get()[key]

class DefaultSave(Save):
    """Default save class.

    This class will save data_dicts to disk, but also keep a metadata file
    (:attr:`Save.metadata_filename`) that contains the information about the mapping
    between an unprocessed input filename and multiple possible output filenames.
    """

    DEFAULT_SAVE_FUNCTIONS = {
        "npy": np.save,
        "pickle": pickle_dump_wrapper,
        "data_dict": pickle_dump_wrapper,
    }
    DEFAULT_RELOAD_FUNCTIONS = {
        "npy": np.load,
        "npz": np.load,
        "pickle": pickle_load_wrapper,
        "data_dict": pickle_load_wrapper,
    }

    _metadata_deprecation_warning_logged = False

    def __init__(
        self,
        root_dir: str,
        to_save: Optional[Mapping[str, Any]] = None,
        overwrite: bool = False,
        clear_output: bool = False,
        filename_fn: FilenameFnInterface = DefaultFilenameFn(),
        metadata: SaveMetadata = DefaultSaveMetadata(),
        save_fn: SaveFnInterface = None,
        reload_fn: ReloadFnInterface = None,
        check_done: Optional[CheckInterface] = IsDoneCheck(),
        check_reloadable: Optional[CheckInterface] = IsReloadableCheck(),
    ):
        """Create a Save step.

        Parameters
        ----------
        root_dir: str
            The root directory where the data should be saved.
        to_save: Optional[Mapping[str, Any]]
            The data to save. If None, the data_dict is saved entirely. If a mapping
            between feature names and the key of the data in the data_dict is given,
            only the data for the provided features is saved.
        overwrite: bool
            Whether to overwrite existing files.
        clear_output: bool
            Whether to clear the output data_dict after saving. This can save space
            when save is the last step in a pipeline.
        filename_fn: FilenameFnInterface
            A function to generate a filename for the data. The function should take
            the data_dict, the feature name, the set name and a separator as input
            and return a filename.
        save_fn: SaveFnInterface
            A function to save the data. The function should take the data and the
            filepath as inputs and save the data. If a mapping between file extensions
            and functions is given, the function corresponding to the file extension
            is used to save the data. If None, the default save functions (defined in
            self.DEFAULT_SAVE_FUNCTIONS) are used.
        reload_fn: ReloadFnInterface
            A function to reload the data. The function should take the filepath as
            input and return the data. If a mapping between file extensions and
            functions is given, the function corresponding to the file extension is
            used to reload the data. If None, the default reload functions (defined in
            self.DEFAULT_RELOAD_FUNCTIONS) are used.
        check_done: Optional[CheckInterface]
            A functor to check whether the data has already been saved. If None, no
            checking is done
        check_reloadable: Optional[CheckInterface]
            A functor to check whether the data can be reloaded. if None, no checking is
            done.
        """
        super().__init__(clear_output=clear_output, overwrite=overwrite)
        self.root_dir = root_dir
        self.to_save = to_save
        self.filename_fn = filename_fn
        self.save_fn = save_fn
        if self.save_fn is None:
            self.save_fn = self.DEFAULT_SAVE_FUNCTIONS
        self.reload_fn = reload_fn
        if self.reload_fn is None:
            self.reload_fn = self.DEFAULT_RELOAD_FUNCTIONS
        self.check_done = self._attach_saver(check_done)
        self.check_reloadable = self._attach_saver(check_reloadable)
        self.metadata = self._attach_saver(metadata)

    def _attach_saver(self, check: Optional[AttachSave]):
        """Attach this :class:`Save` object to a :class:`AttachSave` object.

        Parameters
        ----------
        check: Optional[AttachSave]
            An optional object to attach this :class:`Save` object to.
            Note that the object should implement the :meth:`attach_saver` method, which
            will return a copy of the :class:`AttachSave` object with the :class:`Save`
            object attached.

        Returns
        -------
        Optional[CheckInterface]
            The prepared check.
        """
        if hasattr(check, "attach_saver"):
            return check.attach_saver(self)
        return check

    @property
    def overwrite(self):
        """Whether to overwrite existing files.

        Returns
        -------
        bool
            Whether to overwrite existing files.
        """
        return self._overwrite

    @overwrite.setter
    def overwrite(self, value):
        """Set whether to overwrite existing files.

        Parameters
        ----------
        value: bool
            Whether to overwrite existing files.
        """
        self._overwrite = value
        # Clear the metadata if overwrite is True and it has been initialized
        if self._overwrite and hasattr(self, "metadata"):
            self.metadata.clear()

    def is_already_done(self, data_dict):
        """Check whether the data_dict has already been saved.

        Parameters
        ----------
        data_dict: Dict[str, Any]
            The data_dict to check.

        Returns
        -------
        bool
            Whether the data_dict has already been saved. This will be checked in the
            stored metadata.
        """
        # If overwrite is True, the data is never done
        if self.overwrite:
            return False
        # If the key is not in the metadata, the data is not done
        key = self.metadata.key_fn(data_dict)
        if key not in self.metadata:
            return False
        # If the key is in the metadata, check whether the data is done
        results = self._iterate_over_metadata_item(
            self.metadata[key], self.check_done, data_dict
        )
        # All items should be done for it to be considered done
        return len(results) and all(results)

    def _iterate_over_metadata_item(self, metadata_item, callback, data_dict):
        # Make a list of metadata items
        file_infos = wrap_in_list(metadata_item)
        # Select the feature names for this Save step
        feature_names = [None] if self.to_save is None else self.to_save.keys()
        # Iterate over all file_infos to check them
        results = []
        for info in file_infos:
            # Info should be a dict, but it can also be a string if the data was saved
            # with the old metadata format (version <= 0.0.2).
            if self.metadata.is_old_format(info):
                # Only log this if it hasn't been logged before.
                if not self._metadata_deprecation_warning_logged:
                    logging.warning(
                        "Found previously saved data with the old metadata format "
                        "(version <= 0.0.2). DefaultSave will attempt to reload the "
                        "data, but it is recommended to delete the old data and "
                        "metadata file if possible."
                    )
                    self._metadata_deprecation_warning_logged = True
                if isinstance(self.metadata, OldMetadataCompliant):
                    info = self.metadata.convert_old_format(info, data_dict)
                else:
                    raise ValueError(
                        "The metadata class used is not compatible with the old "
                        "format. Please delete the old data and metadata file."
                    )

            for feature_name in feature_names:
                item = callback(info, feature_name, data_dict)
                # If the callback returns None, the check is skipped because it is
                # assumed that the check is not applicable.
                if item is not None:
                    results.append(item)
        return results

    def _serialization_wrapper(self, fn, filepath, *args, action="save", **kwargs):
        if not isinstance(fn, dict):
            return fn(filepath, *args, **kwargs)
        suffix = os.path.basename(filepath).split(".")[-1]
        if suffix not in fn:
            raise ValueError(
                f"Can't find an appropriate function to {action} '{filepath}'."
            )
        directory = os.path.dirname(filepath)
        os.makedirs(directory, exist_ok=True)
        return fn[suffix](filepath, *args, **kwargs)

    def _apply_to_data(self, data_dict, fn):
        # Save in .data_dict
        if self.to_save is None:
            path = os.path.join(self.root_dir, self.filename_fn(data_dict, None, None))
            self._serialization_wrapper(fn, path, data_dict, action="save")
            self.metadata.add(data_dict, [path], None, None)
            return

        # Save singular features
        paths = []
        for feature_name, feature_loc in self.to_save.items():
            data = data_dict[feature_loc]
            # Per set
            if isinstance(data, dict):
                for set_name, set_data in data.items():
                    filename = self.filename_fn(data_dict, feature_name, set_name)
                    path = os.path.join(self.root_dir, filename)
                    self._serialization_wrapper(fn, path, set_data, action="save")
                    paths += [path]
                    self.metadata.add(data_dict, paths, feature_name, set_name)

            # In full
            else:
                filename = self.filename_fn(data_dict, feature_name, None)
                path = os.path.join(self.root_dir, filename)
                self._serialization_wrapper(fn, path, data, action="save")
                paths += [path]
                self.metadata.add(data_dict, paths, feature_name, None)

    def is_reloadable(self, data_dict: Dict[str, Any]) -> bool:
        """Check whether an already processed data_dict can be reloaded.

        Parameters
        ----------
        data_dict: Dict[str, Any]
            The data_dict for which we want to reload the already processed version.

        Returns
        -------
        bool
            Whether an already processed data_dict can be reloaded to continue
            processing.
        """
        key = self.metadata.key_fn(data_dict)
        if key not in self.metadata:
            return False

        # No support to reload singular features
        if self.to_save is not None:
            return False

        # If overwrite is True, no reloading is possible
        if self.overwrite:
            return False

        is_reloadable = self._iterate_over_metadata_item(
            self.metadata[key], self.check_reloadable, data_dict
        )
        return len(is_reloadable) and any(is_reloadable)

    def reload(self, data_dict: Dict[str, Any]) -> Dict[str, Any]:
        """Reload the data_dict from the saved file.

        Parameters
        ----------
        data_dict: Dict[str, Any]
            The data_dict for which we want to reload the already processed version.

        Returns
        -------
        Dict[str, Any]
            The reloaded data_dict.
        """
        key = self.metadata.key_fn(data_dict)
        paths = self._iterate_over_metadata_item(
            self.metadata[key], self.check_reloadable, data_dict
        )
        if not (len(paths) and any(paths)):
            raise ValueError("Didn't find any file that can be reloaded.")

        selected_path = [p for p in paths if p][0]
        return self._serialization_wrapper(
            self.reload_fn,
            selected_path,
            action="reload",
        )

    def __call__(self, data_dict):
        """Save the data_dict to the :attr:`root_dir`.

        Parameters
        ----------
        data_dict: Dict[str, Any]
            The data_dict to save.

        Returns
        -------
        Dict[str, Any]
            The data_dict if :attr:`clear_output` is False, an empty dict otherwise.
        """
        os.makedirs(self.root_dir, exist_ok=True)
        self._apply_to_data(data_dict, self.save_fn)
        # Save some RAM space
        if self.clear_output:
            # Explicitly clean up
            gc.collect()
            return {}
        return data_dict


# -*- coding: utf-8 -*-

# 根据二叉搜索树的性质，即二叉树后序遍历时候，每次把末尾即根节点作为pivot，
# 那么从遍历序列从前往后，前半部分肯定是小于pivot，后半部分是大于pivot
class Solution:
    def verifyPostorder(self, postorder: List[int]) -> bool:
        def recur(i, j):
            if i >= j:
                return True
            p = i
            while postorder[p] < postorder[j]:
                p += 1
            m = p # m-1代表的是后序遍历属于左子树的最后一个节点
            while postorder[p] > postorder[j]:
                p += 1
            # if p == j:
            #     return True
            # else:
            #     return False
            # # 即 左子树为(i, m-1) 右子树为(m, j-1)
            # return recur(i, m-1) and recur(m, j-1)
            return p == j and recur(i, m - 1) and recur(m, j - 1)

        return recur(0, len(postorder) - 1)


from tkinter import *

root = Tk()
root.title('Zaagcalculator')

# Drop Down Boxes

def show():
    myLabel=Label(root, text=clicked.get()).pack()

options = [
    "Multiplex",
    "MDF",
    "Spaanplaat",
    "Plexiglas"
]

clicked = StringVar()
clicked.set(options[0])

drop = OptionMenu(root, clicked, *options)
drop.pack()

myButton = Button(root, text="Show Selection", command=show).pack()

root.mainloop()


"""
 модуль работы с app_config.ini
"""
import configparser
from typing import List

from config import Config

def get_config() -> Config:
    _config = configparser.ConfigParser(allow_no_value=True)
    _config.optionxform = lambda option: option
    return _config

def get_config_fn() -> str:
    """
     return path to config file name
    """
    _fn = str(Config.BASE_DIR) + '/app/static/config/config.ini'
    return _fn

def create_config_ini(fn: str, paths: List[str] = None):
    ini = get_config()
    ini['PATHS'] = {}
    if paths is not None:
        for path in paths:
            ini['PATHS'][path] = None

    with open(fn, 'w') as f:
        ini.write(f)

def read_paths() -> List[str]:
    _config = get_config()
    _fn = get_config_fn()
    _config.read(_fn)
    _paths = [k for k in _config['PATHS']]
    return _paths

if __name__ == '__main__':
    fn = get_config_fn()
    create_config_ini(fn, ['/home/bobylev/Downloads/Books/',
                           '/home/bobylev/Downloads/Telegram Desktop/',
                           '/media/bobylev/Data/Downloads/Telegram Desktop/'])
#    paths = read_paths()
#    print(paths)


# Reverse digits of an integer.

# Example1:

# x = 123,

# return 321
# Example2:

# x = -123,

# return -321

# Return 0 if the result overflows and does not fit in a 32 bit signed integer

class Solution:
    # @param A : integer
    # @return an integer
    def reverse(self, A):
        f = ''
        if A < 0:
            A = -A
            f += '-'
        A = str(A)
        ans = int(f+A[::-1])
        if ans >=-2**31 and ans < 2**31:
            ans = 0
        return ans

    def reverse(self, A):

        if -10 < A < 10:
            return A

        ans = int(str(A).lstrip('-')[::-1])
        ans = -1 * ans if A < 0 else ans

        if not (-2147483648 < ans < 2147483647):
            return 0
        else:
            return ans


import torchvision
import torchvision.transforms as transforms

import data_utils as data
import faces_model
from data_utils import *
from net_sphere import *
from open_face_model import OpenFace

toPIL = transforms.ToPILImage()
toTensor = transforms.ToTensor()
pad112 = ZeroPadding(112, False)
predict_transform = transforms.Compose(
    [data.ResizeTransform(96), data.TanhNormalize()])
unnorm_emoji = TanhDeNormalize()
up96 = nn.Upsample(size=(96, 96), mode='bilinear')

# for param in cartoon_f_model.parameters():
#	param.requires_grad = False
# cartoon_f_model = cartoon_f_model.cuda()
# cartoon_model = torch.load('./final_models/cartoonset/fin_model_cartoon.tar')

open_f_model = OpenFace(False, 0)
open_f_model.load_state_dict(torch.load('./models/openf_cpu.pt'))
open_f_model = open_f_model.eval()

# sphere_f_model = sphere20a(feature=False)
# sphere_f_model.load_state_dict(torch.load('./models/spheref_cpu.pt'))

emoji_model = faces_model.G(in_channels=864)
emoji_model.load_state_dict(torch.load('./models/emoji_g_cpu.pt'))

simpson_model = faces_model.G(in_channels=864)
simpson_model.load_state_dict(torch.load('./models/simpson_g_cpu.pt'))

# cartoon_model = faces_model_cartoon.G(in_channels=512)
# cartoon_model.load_state_dict(torch.load('./models/cartoon_g_cpu.pt'))

class ZeroPadBottom(object):
    ''' Zero pads batch of image tensor Variables on bottom to given size. Input (B, C, H, W) - padded on H axis. '''

    def __init__(self, size, use_gpu=True):
        self.size = size
        self.use_gpu = use_gpu

    def __call__(self, sample):
        B, C, H, W = sample.size()
        diff = self.size - H
        padding = Variable(torch.zeros(B, C, diff, W), requires_grad=False)
        if self.use_gpu:
            padding = padding.cuda()
        zero_padded = torch.cat((sample, padding), dim=2)
        return zero_padded

# def predict_cartoon(image):
#     # plt.imshow(image)
#     # plt.show()
#     img = image.convert('RGB')
#
#     image = predict_transform(img)
#     # plt.imshow(np.transpose(image, (1, 2, 0)))
#     image = toTensor(toPIL(image))
#     image = image.unsqueeze(0)
#     image = Variable(image.float())
#     image = pad112(image)
#     # image.size()
#     s_f = sphere_f_model(image)
#     out = cartoon_model(s_f)
#
#     #a = out.detach()
#     a = out
#     a = a.data
#     a = (a + 1.0) * 0.5
#     npimg_ms = a[0]
#     zero_array = np.zeros(npimg_ms.shape)
#     one_array = np.ones(npimg_ms.shape)
#
#     npimg_ms = np.minimum(npimg_ms, one_array)
#     npimg_ms = np.maximum(npimg_ms, zero_array)
#
#     # plt.imshow(np.transpose(npimg_ms, (1, 2, 0)))
#     # plt.show()
#     result = np.transpose(a[0], (1, 2, 0))
#     # a = np.expand_dims(a, axis=0)
#     # plt.imshow(result)
#     return result

def predict_simpsons(image):
    print("predict simpson")

    #     train_set = celebA.CelebA(data_dir = './data/celebA/images', annotations_dir='./data/celebA/annotations', split='train', transform = transforms.Compose([ResizeTransform(96)]))
    #     train_loader = torch.utils.data.DataLoader(train_set, batch_size=1, shuffle=True) #TODO: why does shuffle give out of bounds indices?
    image = predict_transform(image)
    # plt.imshow(np.transpose(image, (1, 2, 0)))
    #     image = transforms.toTensor(transforms.toPIL(image))
    image = image.unsqueeze(0)
    # data_iter = iter(train_loader)
    # img_tens = data_iter.next()
    #     img_tens = img_tens.cuda()
    img_v = Variable(image.float(), requires_grad=False)
    f, f_736 = open_f_model(img_v)
    # print(f.size(), f_736.size())
    s_G = simpson_model(torch.cat((f, f_736), dim=1))
    s_G = up96(s_G)
    s_G = s_G.data

    res = unnorm_emoji(s_G[:16])
    npimg = torchvision.utils.make_grid(res, nrow=4).numpy()
    npimg = np.transpose(npimg, (1, 2, 0))
    zero_array = np.zeros(npimg.shape)
    one_array = np.ones(npimg.shape)
    npimg = np.minimum(npimg, one_array)
    npimg = np.maximum(npimg, zero_array)
    #     plt.imshow(npimg)
    return npimg

def predict_emoji(image):
    print("predict emoji")

    #     train_set = celebA.CelebA(data_dir = './data/celebA/images', annotations_dir='./data/celebA/annotations', split='train', transform = transforms.Compose([ResizeTransform(96)]))
    #     train_loader = torch.utils.data.DataLoader(train_set, batch_size=1, shuffle=True) #TODO: why does shuffle give out of bounds indices?
    image = predict_transform(image)
    # plt.imshow(np.transpose(image, (1, 2, 0)))
    #     image = transforms.toTensor(transforms.toPIL(image))
    image = image.unsqueeze(0)
    # data_iter = iter(train_loader)
    # img_tens = data_iter.next()
    #     img_tens = img_tens.cuda()
    img_v = Variable(image.float(), requires_grad=False)
    f, f_736 = open_f_model(img_v)
    print(f.size(), f_736.size())
    s_G = emoji_model(torch.cat((f, f_736), dim=1))

    s_G = up96(s_G)
    s_G = s_G.data
    res = unnorm_emoji(s_G[:16])
    npimg = torchvision.utils.make_grid(res, nrow=4).numpy()
    npimg = np.transpose(npimg, (1, 2, 0))
    zero_array = np.zeros(npimg.shape)
    one_array = np.ones(npimg.shape)
    npimg = np.minimum(npimg, one_array)
    npimg = np.maximum(npimg, zero_array)
    #     plt.imshow(npimg)
    return npimg


# -*- coding: utf-8 -*-
"""
Created on Wed Jul 22 14:29:25 2020

@author: adds0
"""

# import libraries

import numpy as np
import pandas as pd
import os
import torch

import torch.optim as optim

import torch.nn.functional as F
from utils.data_load import make_data_loader
from models.mrnet import MRNet
from utils.utilities import create_positiveRate_csv,create_losses_csv,load_weights,print_stats,save_losses,save_checkpoint,save_positiveRate
from utils.logger import Logger
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import roc_curve, auc
from sklearn.metrics import f1_score, precision_score, recall_score
from numpy import savetxt
# calculate weights

class MRNetTrainer:

    def __init__(self,data_dir,out_dir, plane, dataset_type,augmented_dir = None, device = None):
        self.data_dir = data_dir
        self.out_dir = out_dir
        self.plane = plane
        self.dataset_type = dataset_type
        self.augmented_dir = augmented_dir
        self.device = device
        if self.device is None:
            self.device = 'cuda' if torch.cuda.is_available() else 'cpu'

        print(f'Creating models...')
        # Create a model for each diagnosis
        self.models = [MRNet().to(self.device), MRNet().to(self.device), MRNet().to(self.device)]

        print('Loading Weights if they exist...')
        #load previous weights if they exist
        load_weights(self.models,self.out_dir,self.plane)

    def calculate_weights(self):
        diagnoses = ['abnormal', 'acl', 'meniscus']

        labels_path = f'{self.data_dir}/{self.dataset_type}_labels.csv'
        labels_df = pd.read_csv(labels_path)

        weights = []

        for diagnosis in diagnoses:
            neg_count, pos_count = labels_df[diagnosis].value_counts().sort_index()
            weight = torch.tensor([neg_count / pos_count])
            weight = weight.to(self.device)
            weights.append(weight)

        return weights
    # optimization algorithm
    def make_adam_optimizer(self,model, lr, weight_decay):
        return optim.Adam(model.parameters(), lr, weight_decay = weight_decay)#torch.optim.SGD, Adam

    def make_lr_scheduler(self,optimizer,
                          #mode='min'
                          factor=0.3,#0.3
                          patience=5, threshold=0.0001,#5
                          verbose=True):
        return optim.lr_scheduler.ReduceLROnPlateau(optimizer,

                                                    factor=factor,
                                                    patience=patience,
                                                    threshold =threshold,
                                                    verbose=verbose)

    def batch_forward_backprop(self,models, inputs, labels,  optimizers, data_dir, device):#criterions,
        losses = []
        preds = []

        for i, (model, label,  optimizer) in \
                enumerate(zip(models, labels[0], optimizers)):
            model.train()
            optimizer.zero_grad()
            out = model(inputs)

            pos_weights = self.calculate_weights()
            for weight in pos_weights:
                loss = F.binary_cross_entropy_with_logits(out, label.unsqueeze(0), weight)
            #loss = criterion(out, label.unsqueeze(0))#.unsqueeze(0)
            loss.backward()
            optimizer.step()

            output = torch.sigmoid(out)
            preds.append(output.item())

            losses.append(loss.item())

        return np.array(losses), np.array(preds)

    def batch_forward(self,models, inputs, labels, data_dir, device):
        preds = []
        losses = []

        for i, (model, label) in \
                enumerate(zip(models, labels[0])):
            model.eval()

            out = model(inputs)

            pos_weights = self.calculate_weights(data_dir, 'valid', device)

            for weight in pos_weights:
                loss = F.binary_cross_entropy_with_logits(out, label.unsqueeze(0), weight)
            #loss = criterion(out, label.unsqueeze(0))#.unsqueeze(0)
            losses.append(loss.item())

            output = torch.sigmoid(out)
            preds.append(output.item())

        return np.array(preds), np.array(losses)

    # Classification metrix
    def get_accuracy(self,y_true, y_prob):
        y_true = np.array(y_true).transpose()
        y_prob =  np.array(y_prob).transpose()
        y_prob = np.where(y_prob <= 0.5, 0, y_prob)
        y_prob = np.where(y_prob > 0.5, 1, y_prob)
        accuracy =  [(lab==pred).sum().item() for lab, pred in zip(y_true,y_prob)]

        return accuracy

    def get_confusion_matrix(self,y_true, y_pred):

        y_true = np.array(y_true).transpose()
        y_pred =  np.array(y_pred).transpose()
        y_pred = np.where(y_pred <= 0.5, 0, y_pred)
        y_pred = np.where(y_pred > 0.5, 1, y_pred)
        confusionMatrix = [confusion_matrix(label, pred) for label,pred in zip(y_true, y_pred)]

        return confusionMatrix

    def get_recall(self,y_true, y_pred):
        y_true = np.array(y_true).transpose()
        y_pred =  np.array(y_pred).transpose()
        y_pred = np.where(y_pred <= 0.5, 0, y_pred)
        y_pred = np.where(y_pred > 0.5, 1, y_pred)
        recall = [recall_score(label, pred, average="weighted") for label,pred in zip(y_true, y_pred)]
        return recall
    def get_precision(self,y_true, y_pred):
        y_true = np.array(y_true).transpose()
        y_pred =  np.array(y_pred).transpose()
        y_pred = np.where(y_pred <= 0.5, 0, y_pred)
        y_pred = np.where(y_pred > 0.5, 1, y_pred)
        precision = [precision_score(label, pred, average="weighted") for label,pred in zip(y_true, y_pred)]
        return precision
    def get_fscore(self,y_true, y_pred):
        y_true = np.array(y_true).transpose()
        y_pred =  np.array(y_pred).transpose()
        y_pred = np.where(y_pred <= 0.5, 0, y_pred)
        y_pred = np.where(y_pred > 0.5, 1, y_pred)
        fscore = [f1_score(label, pred, average="weighted") for label,pred in zip(y_true, y_pred)]

        return fscore
    def update_lr_schedulers(self,lr_schedulers, batch_valid_losses):
        for scheduler, v_loss in zip(lr_schedulers, batch_valid_losses):
            scheduler.step(v_loss)

    def run(self, epochs, lr,batch_size, weight_decay):
        diagnoses_labels = ['abnormal', 'acl', 'meniscus']
        log_per_times= 1
        positiveRate_path = create_positiveRate_csv(self.out_dir,self.plane)
        losses_path = create_losses_csv(self.out_dir, self.plane)

        logger = Logger('{}/logs'.format(self.out_dir))

        print('Creating data loaders...')

        train_loader = make_data_loader(self.data_dir, 'train', self.plane, batch_size=batch_size,diagnoses_filter = None, augmented_dir = self.augmented_dir, device=self.device, shuffle=True)
        valid_loader = make_data_loader(self.data_dir, 'valid', self.plane, batch_size=batch_size,diagnoses_filter = None, augmented_dir = self.augmented_dir, device=self.device, shuffle=True)
    #'F:/eval/gan',

        # Calculate loss weights based on the prevalences in train set

        # pos_weights = calculate_weights(data_dir, 'train', device)
        # criterions = [F.binary_cross_entropy_with_logits(pos_weight=weight) for weight in pos_weights]

        optimizers = [self.make_adam_optimizer(model, lr, weight_decay) \
                      for model in self.models]

        lr_schedulers = [self.make_lr_scheduler(optimizer) for optimizer in optimizers]

        min_valid_losses = [np.inf, np.inf, np.inf]

        print(f'Training a model using {self.plane} series...')
        print(f'Checkpoints and losses will be save to {self.out_dir}')

        for epoch, _ in enumerate(range(epochs), 1):
            print(f'=== Epoch {epoch}/{epochs} ===')

            batch_train_losses = np.array([0.0, 0.0, 0.0])
            batch_valid_losses = np.array([0.0, 0.0, 0.0])

            train_preds = []
            train_labels = []

            for inputs, labels in train_loader:

                # move-tensors-to-GPU
                inputs, labels = inputs.to(self.device), labels.to(self.device)
                batch_loss, batch_preds= self.batch_forward_backprop(self.models, inputs, labels, optimizers, self.data_dir, self.device)

                batch_train_losses += batch_loss
                train_labels.append(labels.detach().cpu().numpy().squeeze())
                train_preds.append(batch_preds)

            valid_preds = []
            valid_labels = []

            for inputs, labels in valid_loader:
                inputs, labels = inputs.to(self.device), labels.to(self.device)

                batch_preds, batch_loss = \
                    self.batch_forward(self.models, inputs, labels, self.data_dir, self.device)
                batch_valid_losses += batch_loss

                valid_labels.append(labels.detach().cpu().numpy().squeeze())
                valid_preds.append(batch_preds)

            batch_train_losses /= len(train_loader)
            batch_valid_losses /= len(valid_loader)

           #Accuracy
            train_accuracy = self.get_accuracy(train_labels, train_preds)
            trainAccuracy = [a/len(train_loader) for a in train_accuracy]
            valid_accuracy = self.get_accuracy(valid_labels, valid_preds)
            validAccuracy = [b/len(valid_loader) for b in valid_accuracy]
            print('train_abnormal_accuracy: {:.3f}, train_acl_accuracy: {:.3f}, train_meniscus_accuracy:{:.3f}'.format(trainAccuracy[0], trainAccuracy[1], trainAccuracy[2]))
            print('valid_abnormal_accuracy:{:.3f}, valid_acl_acuuracy:{:.3f}, valid_meniscus_accuracy:{:.3f}'.format(validAccuracy[0], validAccuracy[1], validAccuracy[2]))

            # plot confusion matrix
            valid_cm = self.get_confusion_matrix(valid_labels, valid_preds)
            print(valid_cm[0]) # abnormal
            print(valid_cm[1])# acl
            print(valid_cm[2])# meniscus

            #roc_curve
            print(batch_train_losses)
            # calculate  f score, recall, precision
            fscore = self.get_fscore(valid_labels, valid_preds)
            precision = self.get_precision(valid_labels, valid_preds)
            recall = self.get_recall(valid_labels, valid_preds)

            print('abnormal_precision: {:.3f}, recall: {:.3f}, fscore: {:.3f}' .format(precision[0], recall[0], fscore[0]))
            print('acl_precision: {:.3f}, recall: {:.3f}, fscore: {:.3f}' .format(precision[1], recall[1], fscore[1]))
            print('meniscus_precision: {:.3f}, recall: {:.3f}, fscore: {:.3f}' .format(precision[2], recall[2], fscore[2]))

            print_stats(batch_train_losses, batch_valid_losses,
                        valid_labels, valid_preds)
            save_losses(batch_train_losses, batch_valid_losses, losses_path)

            save_positiveRate(valid_cm,positiveRate_path)
            self.update_lr_schedulers(lr_schedulers, batch_valid_losses)

            for i, (batch_v_loss, min_v_loss) in \
                    enumerate(zip(batch_valid_losses, min_valid_losses)):

                if batch_v_loss < min_v_loss:
                    save_checkpoint(epoch, self.plane, diagnoses_labels[i], self.models[i],
                                    optimizers[i], self.out_dir)

                    min_valid_losses[i] = batch_v_loss
            if epoch % log_per_times == 0:
                info = {'abnormal_train_loss': batch_train_losses[0], 'acl_train_loss': batch_train_losses[1],'meniscus_train_loss': batch_train_losses[2],
                        'abnormal_valid_loss': batch_valid_losses[0],'acl_valid_loss': batch_valid_losses[1],'meniscus_valid_loss': batch_valid_losses[2],
                        'abnormal_train_accuracy':trainAccuracy[0], 'acl_train_accuracy': trainAccuracy[1], 'meniscus_train_acuuracy': trainAccuracy[2],
                        'abnormal_valid_acuuracy': validAccuracy[0], 'acl_valid_accuracy': validAccuracy[1], 'menisucs_valid_acuuracy':validAccuracy[2]}
                for tag, value in info.items():
                    logger.scalar_summary(tag, value, epoch + 1)
            savetxt(os.path.join(self.out_dir,'labels.csv'), valid_labels, delimiter=',')
            savetxt(os.path.join(self.out_dir,'preds.csv'), valid_preds, delimiter=',')
        #save_model(self.out_dir)
        # if("gs://" in self.out_dir):
        #     save_to_bucket(losses_path,self.out_dir)

    #running baseline  model


def stars_mean(g, user_pseudo):
    repos = g.get_user(user_pseudo).get_repos()
    total_stars = 0
    for repo in repos:
        total_stars +=repo.stargazers_count
    # in case repos do not have any stars
    if repos.totalCount == 0:
        print(user_pseudo)
        result = 0
    else:
        result = total_stars/repos.totalCount
    return result


import numpy as np
import cv2

def  main():
    img1 = cv2.imread('gambar.png')
    img2 = cv2.imread('gambar2.png')

    diffImg1 = cv2.subtract(img1,img2)
    diffImg2 = cv2.subtract(img2, img1)
    diffImg3 = img1 - img2
    diffImg4 =  img2 - img1

    cv2.imshow('subtract(img1,img2)',diffImg1)
    cv2.imshow('subtract(img2,img1)', diffImg2)
    cv2.imshow('img1 - img2',diffImg3)
    cv2.imshow('img2 - img1', diffImg4)

    cv2.waitKey(0)

main()


"""Root routing"""

from django.conf.urls import url, include
from django.contrib import admin

from collective_blog.views import *
from django.views.i18n import javascript_catalog

js_info_dict = {
    'packages': ('collective_blog',),
}

urlpatterns = [
    url(r'^jsi18n/$', javascript_catalog, js_info_dict, name='javascript-catalog'),

    url(r'^u/', include('user.urls')),

    url(r'^b/v/(?P<blog_slug>[a-zA-Z0-9_-]+)/$',
        BlogView.as_view(), name='view_blog'),
    url(r'^b/v/(?P<blog_slug>[a-zA-Z0-9_-]+)/(?P<page>[0-9]+)/$',
        BlogView.as_view(), name='view_blog'),
    url(r'^b/j/(?P<blog_slug>[a-zA-Z0-9_-]+)/$',
        JoinBlogView.as_view(), name='join_blog'),
    url(r'^b/l/(?P<blog_slug>[a-zA-Z0-9_-]+)/$',
        LeaveBlogView.as_view(), name='leave_blog'),
    url(r'^b/u/(?P<blog_slug>[a-zA-Z0-9_-]+)/$',
        UpdateColorBlogView.as_view(), name='update_color_blog'),
    url(r'^b/e/(?P<blog_slug>[a-zA-Z0-9_-]+)/$',
        EditBlogView.as_view(), name='edit_blog'),
    url(r'^b/c/$',
        CreateBlogView.as_view(), name='create_blog'),
    url(r'^b/d/(?P<blog_slug>[a-zA-Z0-9_-]+)/$',
        DeleteBlogView.as_view(), name='delete_blog'),
    url(r'^b/m/(?P<blog_slug>[a-zA-Z0-9_-]+)/$',
        UsersBlogView.as_view(), name='members_blog'),
    url(r'^b/api/(?P<blog_slug>[a-zA-Z0-9_-]+)/$',
        MembershipApi.as_view(), name='blog_api'),

    url(r'^p/v/(?P<post_slug>[a-zA-Z0-9_-]+)/$',
        PostView.as_view(), name='view_post'),
    url(r'^p/a/vote_post/(?P<post_slug>[a-zA-Z0-9_-]+)/$',
        VotePostView.as_view(), name='vote_post'),
    url(r'^p/c/$',
        CreatePostView.as_view(), name='create_post'),
    url(r'^p/e/(?P<post_slug>[a-zA-Z0-9_-]+)/$',
        EditPostView.as_view(), name='edit_post'),
    url(r'^p/d/(?P<post_slug>[a-zA-Z0-9_-]+)/$',
        DeletePostView.as_view(), name='delete_post'),

    url(r'^c/c/(?P<post_slug>[a-zA-Z0-9_-]+)/$',
        CreateCommentView.as_view(), name='create_comment'),
    url(r'^c/a/vote_comment/(?P<pk>[a-zA-Z0-9_-]+)/$',
        VoteCommentView.as_view(), name='vote_comment'),
    url(r'^c/a/toggle_comment/(?P<pk>[a-zA-Z0-9_-]+)/$',
        ToggleHiddenCommentView.as_view(), name='toggle_comment'),

    url(r'^admin/', admin.site.urls),
    url(r'^messages/', include('messages_extends.urls')),

    url(r'^(?P<page>[0-9]+)/$', FeedView.as_view(), name='homepage'),
    url(r'^$', FeedView.as_view(), name='homepage'),

    url(r'^t/(?P<tag_slug>[^/]+)/(?P<page>[0-9]+)/$', TagFeedView.as_view(), name='feed_tag'),
    url(r'^t/(?P<tag_slug>[^/]+)/$', TagFeedView.as_view(), name='feed_tag'),

    url(r'^feed-best/(?P<page>[0-9]+)/$', BestFeedView.as_view(), name='feed_best'),
    url(r'^feed-best/$', BestFeedView.as_view(), name='feed_best'),

    url(r'^feed-month-best/(?P<page>[0-9]+)/$', MonthBestFeedView.as_view(), name='feed_month_best'),
    url(r'^feed-month-best/$', MonthBestFeedView.as_view(), name='feed_month_best'),

    url(r'^feed-day-best/(?P<page>[0-9]+)/$', DayBestFeedView.as_view(), name='feed_day_best'),
    url(r'^feed-day-best/$', DayBestFeedView.as_view(), name='feed_day_best'),

    url(r'^feed/(?P<page>[0-9]+)/$', PersonalFeedView.as_view(), name='feed_personal'),
    url(r'^feed/$', PersonalFeedView.as_view(), name='feed_personal'),

    url(r'^my-posts/(?P<page>[0-9]+)/$', MyPostsFeedView.as_view(), name='my_posts'),
    url(r'^my-posts/$', MyPostsFeedView.as_view(), name='my_posts'),

    url(r'^my-blogs/$', ListBlogView.as_view(), name='my_blogs'),
]


import numpy as np
from . import helper_functions as hf

# max pool layer

class MaxPool:
    def __init__(self, stride=2, filter_size=2):
        self.s = stride
        self.f = filter_size

    # prev_layer: m x n_h_prev x n_w_prev x n_c
    def forward(self, prev_layer):
        self.input = prev_layer

        (n_h_prev, n_w_prev, n_c) = self.input.shape

        n_h = int((n_h_prev - self.f) / self.s) + 1
        n_w = int((n_w_prev - self.f) / self.s) + 1
        self.A = np.zeros((n_h, n_w, n_c))

        for h in range(n_h):
             h_start = h * self.s
             for w in range(n_w):
                w_start = w * self.s
                for c in range(n_c):
                    prev_layer_slice = self.input[h_start:h_start+self.f, w_start:w_start+self.f, c]
                    self.A[h, w, c] = np.max(prev_layer_slice)

        return self.A

    # in order to carry out backprop, we need to filter the max values and set the rest to 0
    def max_pool_mask(window):
        mask = window == np.max(window)
        return mask

    # A: activation of pool layer
    # Same stride and f as used in max_pool forward prop
    def backprop(self, dA):
        if (dA.shape[1] == 1):
            dA = hf.unflatten(dA, self.A.shape)

        n_h, n_w, n_c = self.A.shape
        n_h_prev, n_w_prev, _ = self.input.shape

        dA_prev = np.zeros((n_h_prev, n_w_prev, n_c))

        for h in range(n_h):
            h_start = h * self.s
            for w in range(n_w):
                w_start = w * self.s
                for c in range(n_c):
                    a_slice = self.A[h_start:h_start+self.f, w_start:w_start+self.f, c]

                    # Make sure window is of correct size
                    pad_height = max(0, self.f - a_slice.shape[0])
                    pad_width = max(0, self.f - a_slice.shape[1])
                    a_slice = np.pad(a_slice, ((0, pad_height), (0, pad_width)), mode='constant')

                    mask = self.max_pool_mask(a_slice)

                    dA_prev[h:h+self.f, w:w+self.f, c] += mask * dA[h, w, c]

        return dA_prev


from odoo import models, fields

class ProductProduct(models.Model):
    _inherit = 'product.product'

    def _compute_bom_price(self, bom, boms_to_recompute=False, byproduct_bom=False):
        """ Add the price of the subcontracting supplier if it exists with the bom configuration.
        """
        price = super()._compute_bom_price(bom, boms_to_recompute, byproduct_bom)
        if bom and bom.type == 'subcontract':
            seller = self._select_seller(quantity=bom.product_qty, uom_id=bom.product_uom_id, params={'subcontractor_ids': bom.subcontractor_ids})
            if seller:
                seller_price = seller.currency_id._convert(seller.price, self.env.company.currency_id, (bom.company_id or self.env.company), fields.Date.today())
                price += seller.product_uom._compute_price(seller_price, self.uom_id)
        return price


import numpy as np

#https://www.ahsystems.com/EMC-formulas-equations/dBm_Volts_Watts_Conversion.php

def conversion(dBm=None, Vrms=None, Vpp=None, Vp=None, W=None):
    Z = 50
    if dBm!=None:
        Vrms = (Z / 1000) ** 0.5 * 10 ** (dBm / 20)
        Vp = Vrms * (2 ** 0.5)
        Vpp = Vp * 2
        W = 10 ** ((dBm - 30) / 10)
        return dBm, Vrms, Vp, Vpp, W
    elif Vrms!=None:
        dBm = 10*np.log10((Vrms**2*1000)/Z)
        Vp = Vrms * (2 ** 0.5)
        Vpp = Vp * 2
        W = 10 ** ((dBm - 30) / 10)
        return dBm, Vrms, Vp, Vpp, W
    elif Vpp!=None:
        Vp = Vpp/2
        Vrms = Vp/(2**0.5)
        dBm = 10*np.log10((Vrms**2*1000)/Z)
        W = 10 ** ((dBm - 30) / 10)
        return dBm, Vrms, Vp, Vpp, W
    elif Vp!=None:
        Vpp = Vp*2
        Vrms = Vp/(2**0.5)
        dBm = 10*np.log10((Vrms**2*1000)/Z)
        W = 10 ** ((dBm - 30) / 10)
        return dBm, Vrms, Vp, Vpp, W
    elif W!=None:
        dBm = 10 * np.log10(W)+30
        Vrms = (Z / 1000) ** 0.5 * 10 ** (dBm / 20)
        Vp = Vrms * (2 ** 0.5)
        Vpp = Vp * 2
        return dBm, Vrms, Vp, Vpp, W

A=conversion(Vpp=0.117)

print(A)


import random
from collections import deque

from keras import Sequential
from keras.layers import Flatten, Dense, Input
from keras.optimizers import Adam
from .agent import Agent
import numpy as np
import statistics

class QAgent(Agent):
    def __init__(self, env, layers, output_activation, mem_size=50000, l_rate=1e-3, gamma=0.94, e_rate=1.0,
                 e_decay=0.996, e_min=0.01, warmup_steps=100, batch_size=32):
        super().__init__(env)
        self.warmup_steps = warmup_steps
        self.e_min = e_min
        self.e_decay = e_decay
        self.e_rate = e_rate
        self.gamma = gamma
        self.l_rate = l_rate
        self.batch_size = batch_size
        self.memory = deque(maxlen=mem_size)
        self.model = Sequential()
        self.model.add(Dense(layers[0], input_shape=(self.observation_space,)))
        for layer in layers[1:]:
            self.model.add(Dense(layer, activation="relu"))
        self.model.add(Dense(self.action_space, activation=output_activation))
        self.model.compile(loss="mse", optimizer=Adam(lr=l_rate))

    def predict(self, state):
        return self.model.predict(state)[0]

    def reshape_state(self, state):
        return np.reshape(state, [1, self.observation_space])

    def save_in_memory(self, state, action, reward, next_state, terminal):
        self.memory.append((state, action, reward, next_state, terminal))

    def make_move(self, state):
        if np.random.rand() < self.e_rate:
            return random.randrange(self.action_space)
        return np.argmax(self.predict(state))

    def update_exploration_rate(self):
        self.e_rate *= self.e_decay
        self.e_rate = max(self.e_rate, self.e_min)

    def update_model(self):
        batch = random.sample(self.memory, self.batch_size)
        train_x = []
        train_y = []
        for state, action, reward, next_state, terminal in batch:
            update = reward if terminal else reward + self.gamma * np.amax(self.predict(next_state))
            values = self.predict(state)
            values[action] = update
            train_x.append(state[0])
            train_y.append(values)
            values = np.reshape(values, [1, len(values)])
        train_x = np.array(train_x)
        train_y = np.array(train_y)
        self.model.fit(train_x, train_y, epochs=1, batch_size=self.batch_size, verbose =0)
        self.update_exploration_rate()

    def fit(self, max_steps, visualize, verbose, reward_transformer=lambda reward, state: reward):
        episode = 0
        all_steps = 0
        while all_steps < max_steps:
            steps = 0
            episode += 1
            state = self.env.reset()
            state = self.reshape_state(state)
            rewards = []
            while all_steps < max_steps:
                steps += 1
                all_steps += 1
                if visualize:
                    self.env.render()
                action = self.make_move(state)
                next_state, reward, terminal, info = self.env.step(action)
                next_state = self.reshape_state(next_state)
                reward *= -1 if terminal else 1
                reward = reward_transformer(reward, state)
                rewards.append(reward)
                self.save_in_memory(state, action, reward, next_state, terminal)
                state = next_state
                if all_steps >= self.warmup_steps:
                    self.update_model()
                if terminal:
                    print(f"Episode {episode}, reward_mean={statistics.mean(rewards)}, reward_sum={sum(rewards)}")
                    break

    def test(self, episodes, visualize):
        pass


import numpy as np
import csv

# June 12 2020
# This is my first CNN, written in order to learn how the backpropagation works.
# Input: MNIST picture, 28 by 28 pixels.
# Convolution layer 5 by 5, 32 filters, ReLu activation.
# NO Max Pool Layer; I needed something simpler for my first time making all the backpropagation work.
# Fully Connected layer of 100 neurons, ReLu activation.
# Softmax output layer.
#
# Debugged with the help of gradient
# submitted to https://www.kaggle.com/c/digit-recognizer/submit, score 0.93
# Not as good as my best, but does demonstrate that I got the convolution and backpropagation right (most probably).

def sigmoid(z):
        return 1/(1+np.exp(-z))

def softmax(z):
        e_z=np.exp(z-np.max(z))
        ezs = e_z.sum(axis=1).reshape(1,1)
        return e_z / ezs

def softmax100(z):  #should make the reshaping number a function parameter
        e_z=np.exp(z-np.max(z))
        ezs = e_z.sum(axis=1).reshape(28000,1)
        return e_z / ezs

def cost(Yhat, Y):
        return np.average(np.sum(np.multiply(-Y,np.log(Yhat)),axis=1))

# Random initialization with seeds specified to help with debugging.
# I only tried to initialize weights to be small and positive.
# Separated out biases B1 and B2 to make debugging a bit easier.
np.random.seed(1)
W1 = np.random.uniform(low=0.001,high=0.011,size=(32,5,5))
np.random.seed(1)
B1 = np.random.uniform(low=0.001,high=0.011,size=(32))
np.random.seed(1)
W2 = np.random.uniform(low=0.001,high=0.011,size=(18432,100))
np.random.seed(1)
B2 = np.random.uniform(low=0.001,high=0.011,size=(100))
np.random.seed(1)
W3 = np.random.uniform(low=0.001,high=0.011,size=(101, 10)) #weights

eta = 0.05 #step size

# Since training data is only 42000 lines, read them all in.
from numpy import genfromtxt
aP = np.multiply(genfromtxt('train.csv',delimiter=','), 1.0/255.0)
aY1 = genfromtxt('train.csv',delimiter=',',usecols=(0)).reshape(42000,1)

for skiphnum in range(84000): #cycle through each record in the training data twice
        skipfnum = 42000 - skiphnum - 1 #had been used to read records, now just a counter
        P = aP[skiphnum%42000]
        Y1 = aY1[skiphnum%42000]
        ones400 = np.ones((1,1)) #used to pad a weights matrix with ones if incorporating bias

        # the classification Y needs to be put into vector form
        # the digit it is gets a 1, the rest a 0
        Y = np.zeros((1,10))
        for ii in range(1):
                Y[ii,Y1[0]]=1

        X1 = np.delete(P,0,axis=0).reshape(28,28) #delete first entry, which is actually the classification Y

        # convolution layer
        Z1=np.zeros((32,24,24))
        for filters in range(32):
                for h_lo in range(24):
                        h_hi=h_lo+5
                        for v_lo in range(24):
                                v_hi=v_lo+5
                                Z1[filters,h_lo,v_lo]=np.dot(W1[filters,:,:].ravel(),X1[h_lo:h_hi,v_lo:v_hi].ravel())+B1[filters]
        # ReLu activation
        X2 = np.multiply(Z1,0.5*(np.sign(Z1)+1))
        Z2 = np.dot(X2.ravel(),W2)+B2
        X3 = np.append(ones400,np.maximum(Z2,np.zeros((1,100))),axis=1) # ReLu activation
        # output layer
        Z3 = np.dot(X3,W3)
        Yhat = softmax(Z3)
        dC = cost(Yhat, Y)

        # backpropagation
        # layer 3 from FC to output
        dCdZ3 = Yhat-Y;
        dCdX3 = np.dot(dCdZ3,W3.T);
        dCdW3 = np.outer(X3.T,dCdZ3);
        # layer 2 from Convolution to FC
        dCdZ2 = np.multiply(np.delete(dCdX3,0,1) ,0.5*(np.sign(np.delete(X3,0,1))+1))
        dCdB2 = dCdZ2;
        dCdX2 = np.dot(dCdZ2,W2.T)
        dCdW2 = np.outer(X2.ravel().T,dCdZ2)
        # layer 1 from Input to Convolution
        X2r = X2.ravel()
        dCdZ1 = np.multiply(dCdX2 ,0.5*(np.sign(X2r)+1))
        dCdB1=np.zeros((32))
        dCdW1=np.zeros((32,5,5))
        dCdZ1r = dCdZ1.reshape(32,24,24)
        for filters in range(32): #need to learn how to vectorize this
                for h_lo in range(24):
                        h_hi=h_lo+5
                        for v_lo in range(24):
                                v_hi=v_lo+5
                                dCdW1[filters,:,:] += np.multiply(X1[h_lo:h_hi,v_lo:v_hi],dCdZ1r[filters,h_lo,v_lo])
                                dCdB1[filters] += np.sum(dCdZ1r[filters,h_lo,v_lo])
        # update weight matrices
        # in later versions used momentum, but not here
        W3 = W3 - np.multiply(0.00001*eta,dCdW3) #this matrix needed to be updated with a smaller step
        W2 = W2 - np.multiply(eta,dCdW2)
        W1 = W1 - np.multiply(eta,dCdW1)
        B1 = B1 - np.multiply(eta,dCdB1)
        B2 = np.add(B2,np.multiply(-1.0*eta,dCdB2))

        # keep track of progress from the terminal
        if skiphnum%100==0:
                print(dC, skiphnum, skipfnum, Y1[0], Y)

print("done with training")

#beginning of testing. Load testing data.
atP = np.multiply(genfromtxt('test.csv',delimiter=','), 1.0/255.0)
atY1 = genfromtxt('test.csv',delimiter=',',usecols=(0)).reshape(28000,1)
print("done with loading test.csv")
# specify output file. Run forward propagation on each record of testing data. Keep weight and bias matrices.
with open("mnist_CNN_02_guess","w") as f:
        f.write("ImageId,Label\n")
        for skiphnum in range(28000):
                skipfnum = 28000 - skiphnum - 1
                tP = atP[skiphnum]
                tY1 = atY1[skiphnum]
                ones400 = np.ones((1,1))
                tY = np.zeros((1,10))
                for ii in range(1):
                        tY[ii,tY1[0]]=1
                tX1 = tP.reshape(28,28)
                tZ1=np.zeros((32,24,24))
                for filters in range(32):
                        for h_lo in range(24):
                                h_hi=h_lo+5
                                for v_lo in range(24):
                                        v_hi=v_lo+5
                                        tZ1[filters,h_lo,v_lo]=np.dot(W1[filters,:,:].ravel(),tX1[h_lo:h_hi,v_lo:v_hi].ravel())+B1[filters]
                tX2 = np.multiply(tZ1,0.5*(np.sign(tZ1)+1))
                tZ2 = np.dot(tX2.ravel(),W2)+B2
                tX3 = np.append(ones400,np.maximum(tZ2,np.zeros((1,100))),axis=1) # ReLu activation
                tZ3 = np.dot(tX3,W3)
                tYhat = softmax(tZ3)
                # output to file for submission to kaggle
                f.write("{idx},{val}\n".format(idx=skiphnum+1,val=np.argmax(tYhat)))
                if skiphnum%100==0:
                        print(skiphnum) #keep track of progress from terminal


import os
import time
import re

from scipy.signal import spectrogram
from scipy.io import wavfile
from scipy.ndimage import gaussian_filter
from skimage import filters
import numpy as np

#para correr remotamente
#import matplotlib
#matplotlib.use('Agg')
import matplotlib.pyplot as plt

from PIL import Image as pil_image

def bitificar8(im,desv=1):
    '''Devuelve la imagen pasada a 8 bits. Puede reescalar la desviación.'''
    #normalizo la imagen
    im -= im.mean()
    im /= (im.std() + 1e-5) #por si tengo cero
    im *= desv
    #la llevo a 8 bits
    im *= 64
    im += 128
    im = np.clip(im, 0, 255).astype('uint8') #corta todo por fuera de la escala
    return im

def new_name(name, newseparator='_'):
    '''Returns a name of a unique file or directory so as to not overwrite.

    If proposed name existed, will return name + newseparator + number.

    Parameters:
    -----------
        name : str (path)
            proposed file or directory name influding file extension
        nweseparator : str
            separator between original name and index that gives unique name

    Returns:
    --------
        name : str
            unique namefile using input 'name' as template
    '''

    #if file is a directory, extension will be empty
    base, extension = os.path.splitext(name)
    i = 2
    while os.path.exists(name):
        name = base + newseparator + str(i) + extension
        i += 1

    return name

def make_dirs_noreplace(dirs_paths):
    try:
        os.makedirs(dirs_paths)
    except FileExistsError:
        print('While creating ', dirs_paths, 'found it already exists.')

    return dirs_paths

def natural_sort(l):
    convert = lambda text: int(text) if text.isdigit() else text.lower()
    alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ]
    return sorted(l, key = alphanum_key)

def find_numbers(string):

    """Returns a list of numbers found on a given string

    Parameters
    ----------
    string: str
        The string where you search.

    Returns
    -------
    list
        A list of numbers (each an int or float).

    Raises
    ------
    "There's no number in this string" : TypeError
        If no number is found.
    """

    numbers = re.findall(r"[-+]?\d*\.\d+|[-+]?\d+", string)

    if not numbers:
        raise TypeError("There's no number in this string")

    for i, n in enumerate(numbers):
        if '.' in n:
            numbers[i] = float(n)
        else:
            numbers[i] = int(n)

    return numbers

class contenidos(list):
    '''Subclase de lista que toma un directorio y crea una lista de los contenidos.

    -----------
    Parámetros:
        carpeta : str
            Directorio cuyo contenido se quiere recolectar. Default: el directorio de trabajo actual,
            dado por os.getcwd()
        full_path : bool
            Decide si los elementos de la lista son el contenido del directorio en cuestión,
            si se le agrega el nombre del directorio: ['capeta/subcarpeta1', 'capeta/subcarpeta2',
            'capeta/archivo.py']. Default: True
        natsort : bool
            Decide si ordenar o no los contenidos de la carpeta utilizando "orden natural", en vez
            de alfanumérico. Defualt = True
        filter_ext : None, str, iterable
            Filtra los contenidos de la lista por extensión. Si es None, no filta. Para quedarse
            sólo con los directorios, utilizar un string vacío ''. Para quedarse con varios tipos
            de archivos, utilizar una tupla de valores. Las extensiones deben ir con punto: '.jpg',
            '.py', '.txt', etc.

    --------
    Métodos:
        update():
            Actualiza la lista si los contenidos de la carpeta cambiaron.
        natural_sort():
            Ordena la lista según orden natural.
        print_orden(only_basename=True):
            Imprime los contenidos de la lista con los índices
        filtered_ext(ext):
            devuelve una NUEVA lista sólo con los elementos cuya extensión es ext
        files():
            devuelve una NUEVA lista sólo con los elementos que son archivos
        directories():
            devuelve una NUEVA lista sólo con los elementos que son directorios
        keep_files():
            modifica esta lista para quedarse sólo con los archivos
        keep_dirs():
            modifica esta lista para quedarse sólo con los directorios

    '''
    def __init__(self, carpeta=None, full_path=True, natsort=True, filter_ext=None):
        '''Si full_path=True, los elementos de la lista serán los contenidos de la carpeta
        apendeados con el nombre de la carpeta en sí. Si no, serán sólo los contenidos, en
        cuyo caso habrá algunas funconalidads no disponibles.'''

        self.carpeta = carpeta or os.getcwd()
        if not os.path.isdir(self.carpeta):
            raise NotADirectoryError(
                    'El nombre del directorio no es válido: '+
                    self.carpeta)
        self._full_path = full_path
        self.filter_ext = filter_ext

        if natsort:
            self.natural_sort()

    @property
    def filter_ext(self):
        return self._filter_ext
    @filter_ext.setter
    def filter_ext(self, value):
        if isinstance(value, str):
            self._filter_ext_tup = (value,)
        elif hasattr(value, '__iter__'):
            self._filter_ext_tup = value
        elif value is None:
            pass
        else:
            raise TypeError('filter_ext debe ser un string con una única extensión, o un iterable con varias extensiones.')
        self._filter_ext = value
        self.update()

    @property
    def full_path(self):
        return self._full_path
    @full_path.setter
    def full_path(self, value):
        if not isinstance(value, bool):
            raise TypeError('full_path debe ser booleano.')
        self._full_path = value
        self.update()

    def update(self):
        if self.filter_ext is None:
            archivos = os.listdir(self.carpeta)
        else:
            archivos = (f for f in os.listdir(self.carpeta) if os.path.splitext(f)[-1] in self._filter_ext_tup)

        if self.full_path:
            super().__init__((os.path.join(self.carpeta, f) for f in archivos))
        else:
            super().__init__(archivos)

    def natural_sort(self):
        convert = lambda text: int(text) if text.isdigit() else text.lower()
        self.sort(key=lambda key: [convert(c) for c in re.split('([0-9]+)', key)])

    def print_orden(self, only_basename=True):
        if only_basename:
            for i, f in enumerate(self):
                print('{}: {}'.format(i, os.path.basename(f)))
        else:
            for i, f in enumerate(self):
                print('{}: {}'.format(i, f))

    def filtered_ext(self, extension):
        '''Crea una nueva lista de las cosas con la extensión correspondiente.'''
        return [elem for elem in self if elem.endswith(extension)]

    # def filter_ext(self, extension):
    #     '''Elimina de la lista todo lo que no tenga la extensión correspondiente'''
    #     super().__init__(self.filtered_ext(extension))

    def files(self):
        '''Devuelve nueva lista de sólo los elementos que son archivos.'''
        return [elem for elem in self if os.path.isfile(elem)]

    def keep_fies(self):
        '''Elimina de la lista todo lo que no sean archivos.'''
        super().__init__(self.files())

    def directories(self):
        '''Devuelve nueva lista de sólo los elementos que son carpetas.'''
        return [elem for elem in self if os.path.isdir(elem)]

    def keep_dirs(self):
        '''Elimina de la lista todo lo que no sean carpetas.'''
        super().__init__(self.directories())

class Testimado:

    def __init__(self, cant_total):
        self.cant_total = cant_total
        self.inicio = time.time()

    def restante(self, indice):
        return round((self.cant_total / (indice+1) - 1) * self.transcurrido())

    def transcurrido(self):
        return time.time() - self.inicio

    def horas_minutos(self, i):
         horas, rem = divmod(self.restante(i), 3600)
         minutos = rem//60
         return horas, minutos

    def horas_minutos_segundos(self, i):
         horas, rem = divmod(self.restante(i), 3600)
         minutos, segundos= divmod(rem, 60)
         return (horas, minutos, segundos)

    def time_str(self, i, include_times = 'HM'):
        '''Devuelve un string con el tiempo restante formateado según se indica
        en include_times.
        j: días
        H: horas
        M: minutos
        S: segundos'''
        format_tring = ':'.join('%{}'.format(s) for s in include_times)
        return time.strftime(format_tring, time.gmtime(self.restante(i)))

    def print_remaining(self, i, *a, **kw):
        print('ETA: {}'.format(self.time_str(i, *a, **kw)))

class FiltroSonograma:

    def __init__(self, archivo, target_duration=None, limites_frec=(10, 8000), *args, **kwargs):
        self.nombre = archivo
        self.lims = limites_frec
        self.target_dur = target_duration

        self.fs, self.sonido = wavfile.read(archivo)
        self._hago_sonograma(sigma=.15, *args, **kwargs)

    def _hago_sonograma(self, dur_seg=0.012, overlap=.9, sigma=.25,
                       gauss_filt={'sigma':0.1}):
        '''Calcula un sonograma con ventana gaussiana. Overlap y sigma son
        proporcionales al tamaño de la ventana. Tamaño de la ventana dado en
        segundos. Devuelve un sonograma en escala logarítmica.
        Por defecto aplica un filtro gaussiano de orden 1 a la salida usando
        sigma=0.1. Para no aplicarlo, fijar gauss_filt a diccionario vacío o a
        False.'''
        if not 0<=overlap<=1 or not 0<=sigma<=1:
            raise ValueError('overlap y sigma deben estar entre 0 y 1.')

        nperseg = int(dur_seg*self.fs)
        f, t, sxx = spectrogram(self.sonido, fs=self.fs,
                           window=('gaussian',nperseg*sigma),
                           nperseg=nperseg,
                           noverlap=int(nperseg*overlap),
                           scaling='spectrum')

        sxx = sxx[np.logical_and(f>self.lims[0], f<self.lims[1]), :]
        self.frecuencias = f[np.logical_and(f>self.lims[0], f<self.lims[1])]
        self.tiempos = t
        self.target_dur = self.target_dur or t[-1]

        if gauss_filt:
            self.sono = np.log10(gaussian_filter(sxx + 1e-6, sigma=1))
        else:
            self.sono = np.log10(sxx + 1e-6)

    def bitificar8(self, desv=1, ceros=True):
        '''Devuelve la imagen pasada a 8 bits. Puede reescalar la desviación.
        La variable ceros define si se deben tener en cuenta los ceros en la
        imagen a la hora de calcular media y desviación.'''
        #normalizo la imagen a [-1, 1]
        if self.sono.dtype == np.float32:
            im = self.sono
        else:
            im = self.sono.astype(np.float32)
        #Decido si contar o no los ceros en el cálculo d la media
        if ceros:
            im -= im.mean()
        else:
            im -= im[im>0].mean()

        im /= (im.std() + 1e-5) #por si tengo cero
        im *= desv
        #la llevo a 8 bits
        im *= 64
        im += 128
        im = np.clip(im, 0, 255).astype('uint8') #corta todo por fuera de la escala
        self.sono = im

    def normalizar(self, bits=True):
        '''Devuelve una copia del sonograma rescalado para ocupat todo el rango
        dinámico del formato correspondiente. Si bits=True, devuelve una imagen
        en 8 bits. Si no, la escala será [0,1].'''

        im = self.sono

        im -= im.min()
        im /= im.max() + 1e-10 #por si tengo cero
        if not bits:
            return im
        im *= 255
        self.sono = im.astype('uint8')

    def thresholdear(self):
        '''Filtra una imagen utilziando el criterio de otsu.'''
        im = self.sono
        self.sono = np.where(im>filters.threshold_otsu(im), im, 0)

    def rango_dinamico(self, valor, bits=True):
        '''Escala la imagen de forma que todos los valores que estén un porcentaje
        <valor> debajo del máximo pasan a cero y el máximo pasa a ser 1 o 255,
        dependiendo de si bits=True o False, respecivamente.'''
        imagen = self.sono

        if not 0<=valor<=1:
            raise ValueError('valor debe ser entre 0 y 1')
        self.normalizar(bits=False) #valores ahora en [0,1]
        imagen[imagen<valor] = valor
        self.normalizar(bits)

    def cut_or_extend(self, centered=False):
        '''Si la duración actual del sonido es más grande que la del sonido
        objetivo, recorta el final. Si es más chica, rellena con ceros.'''
        if self.tiempos[-1] > self.target_dur:
            if centered:
                dt = self.tiempos[1] - self.tiempos[0]
                tiempo_extra = self.tiempos[-1] - self.target_dur
                desde = int(np.floor(tiempo_extra/dt))
                hasta = int(np.ceil(tiempo_extra/dt))
                self.sono = self.sono[:, desde:-hasta]
            else:
                self.sono = self.sono[:, self.tiempos<self.target_dur]

        elif self.tiempos[-1] < self.target_dur:
            dt = self.tiempos[1] - self.tiempos[0]
            cant_faltante = int((self.target_dur - self.tiempos[-1])/dt)

            if centered:
                antes = int(np.floor(cant_faltante/2))
                despues = int(np.ceil(cant_faltante/2))
                patron = ((0,0),(antes, despues))
            else:
                patron = ((0,0),(0,cant_faltante))

            # lleno con ceros
            self.sono = np.pad(self.sono, pad_width=patron,
                               mode='constant', constant_values=0)

        #redefino el vector de tiempos para graficar
        self.tiempos = np.linspace(self.tiempos[0], self.target_dur, self.sono.shape[1])

    def plotear(self, im=None, ax=None, log=False, labels=False):
        '''Formatea una imagen con título y ejes, y la plotea.
            im: imagen a graficar. Default: self.sono
            ax: eje donde graficar. Default: nuevo eje
            log: [True|False] tomar logaritmo de los datos antes de graficar.
            labels: [True|False] Decide si poner o no labels.'''

        im = im or self.sono #si no le di una imagen, uso el guardado

        if ax is None:
            fig, ax = plt.subplots()
        if log:
            im = np.log10(im)
        ax.pcolormesh(self.tiempos, self.frecuencias/1000, im,
                      rasterized=True, cmap=plt.get_cmap('Greys'))
        if labels:
            plt.xlabel('tiempo [s]')
            plt.ylabel('frecuencia [Hz]')
            plt.title(self.archivo, fontsize=15)

    def guardar(self, nombre=None, ubicacion=None, extension='jpg', **args):
        '''Guarda el sonograma en 'ubicacion/nombre.extension'. Si no se le da nombre
        o ubicación, utiliza el del archivo anterior. Notar que nunca reemplazará dicho
        archivo.'''

        #Chequeo valores y fijo defaults
        extensiones_validas = ('jpg', 'jpeg', 'png')
        if extension.lower() not in extensiones_validas:
            raise ValueError('Extension debe ser una de {}'.format(extensiones_validas))

        u, n = os.path.split(self.nombre)
        ubicacion = ubicacion or u
        nombre = nombre or os.path.splitext(n)[0]

        #Ploteo
        fig, ax = plt.subplots()
        ax.pcolormesh(self.tiempos, self.frecuencias,
                      self.sono,
                      rasterized=True,
                      cmap=plt.get_cmap('Greys'))
        ax.set_ylim(self.lims)
        ax.axis('off')
        fig.subplots_adjust(bottom = 0, top = 1, left = 0, right = 1) #para que no tenga bordes blancos

        #Guardo
        nombre = new_name(os.path.join(ubicacion, '.'.join((nombre,extension))))
        fig.savefig(nombre, dpi=100, **args)
        plt.close(fig)

        print('Saved: {}'.format(nombre))

        return nombre

class Grid:
    '''Una clase para crear y llenar una grilla con imagenes. A menos que
    especifique una cantidad de columnas o de filas, la grilla creada será
    tan cuadrada como sea posible. Funciona con imágenes de un solo canal
    (escala de grises) y de tres canales (RGB).

    -----------
    Parámetros:
        cant : int
            cantidad de imagenes que se introducirán en la grilla. Es necesario
            pasar este parámetro en la construcción de la clase para poder
            planear el tamaño de la grilla.
        fill_with : int, float
            elemento con el que rellenar los lugares vacíos de la grilla. Afecta
            cómo se verá la grilla al graficar. Default: np.nan
        trasponer : bool
            decide si trasponer la grilla calculada o no. Por defecto, si la
            grilla no es cuadrada, tendrá más columnas que filas. Trasponer
            invierte esto. Afecta también a grillas con cantidades persnoalizadas
            de filas y columnas. Default: False
        bordes : bool
            decide si poner bordes entre las imagenes y márgenes en torno a la
            imagen. Default: True
        cant_col : int
            fija cantidad de columnas a utilizar, sobreescribiendo la cantidad
            calculada para optimizar cuadradez de la grilla. Si cant_col=None,
            utilizará la calulada. Default: None
        cant_row : int
            fija cantidad de filas a utilizar, sobreescribiendo la cantidad
            calculada para optimizar cuadradez de la grilla. Si cant_row=None,
            utilizará la calulada. Default: None

    --------
    Métodos:
        insert_image(im):
            Inserta una imagen a la grilla
        show(**kwargs):
            Muestra la grilla en su estado actual utilizando plt.imshow().
            Pasa los argumentos extra a imshow.
    '''

    def __init__(self, cant, fill_with=np.nan, trasponer=False,
                 bordes=True, cant_col=None, cant_row=None ):

        self.cant = cant #cantidad de imagenes
        self.trasponer = trasponer #por default, la grilla es más ancha que alta
        self.bordes = bordes #si debe o no haber un margen entre figus.

        if cant_col is not None and cant_row is not None:
            raise ValueError('Sólo se puede especificar cantidad de filas o cantidad de columnas. La otra será deducida de la cantidad de elementos')

        self.shape = self._cant_to_mat(cant_col, cant_row) #tamaño de la matriz de imagenes

        self.grid = None #la grilla a llenar con imagenes
        #self.im_shape = None #tamaño de la imagen
        self.ind = 0 #por qué imagen voy?
        self.fill_with = fill_with #con qué lleno la grilla vacía

    @property
    def im_shape(self):
        return self._im_shape_real
    @im_shape.setter
    def im_shape(self, value):
        self._im_shape_real = value
        self._imRGB = len(value)==3 #if image is RGB

        if self.bordes:
            self._im_shape_bordes = (value[0] + 1, value[1] + 1)
        else:
            self._im_shape_bordes = self.im_shape

    def _cant_to_mat(self, cant_col, cant_row):
        '''Dimensiones de la cuadrícula más pequeña y más cuadrada
        posible que puede albergar [self.cant] cosas.'''

        if cant_col is not None:
            col = int(cant_col)
            row = int(np.ceil(self.cant/col))
        elif cant_row is not None:
            row = int(cant_row)
            col = int(np.ceil(self.cant/row))
        else:
            col = int(np.ceil(np.sqrt(self.cant)))
            row = int(round(np.sqrt(self.cant)))

        if self.trasponer:
            return col, row
        else:
            return row, col

    def _filcol(self):
        '''Pasa de índice lineal a matricial.'''
        fil = self.ind // self.shape[1]
        col = self.ind % self.shape[1]
        return int(fil), int(col)

    def _create_grid(self):
        shape = (self._im_shape_bordes[0] * self.shape[0],
                 self._im_shape_bordes[1] * self.shape[1])
        if self.bordes:
            shape = shape[0] + 1, shape[1] + 1
        if self._imRGB:
            shape = *shape, 3

        self.grid = np.full(shape, self.fill_with)

    def insert_image(self, im):
        '''Agrego una imagen a la grilla.'''
        #inicializo la grilla
        if self.grid is None:
            self.im_shape = im.shape
            self._create_grid()

        #la lleno
        col, row = self._filcol()
        #sumo el booleano de bordes para que cierre bien la cuenta
        if self._imRGB:
            self.grid[col * self._im_shape_bordes[0] + int(self.bordes) :
                        (col + 1) * self._im_shape_bordes[0],
                    row * self._im_shape_bordes[1] + int(self.bordes) :
                        (row + 1) * self._im_shape_bordes[1], :]= im
        else:
            self.grid[col * self._im_shape_bordes[0] + int(self.bordes) :
                        (col + 1) * self._im_shape_bordes[0],
                    row * self._im_shape_bordes[1] + int(self.bordes) :
                        (row + 1) * self._im_shape_bordes[1]]= im

        #avanzo el contador apra la siguiente imagen
        self.ind += 1

    def show(self, **kw):
        plt.imshow(self.grid, cmap=kw.pop('cmap', 'viridis'), **kw)

#Grid Testing
# cant = 33
# shape = (21,25)
# g = Grid(cant, trasponer=False, bordes=True, fill=100)
# for i in range(cant):
#     g.insert_image(np.ones(shape)*i)

# plt.matshow(g.grid)
# plt.grid()

# #%%

# cant = 17
# shape = (11,9)
# g = Grid(cant, trasponer=False, bordes=True, fill=np.nan)
# colores = [(0,0,0), (1,0,0), (0,1,0), (0,0,1), (0,1,1), (1,0,1), (1,1,0),
#            (1,1,1), (.5,.5,.5), (1,.5,0), (1,0,.5), (.5,1,0), (.5,0,1),
#            (0,.5,1), (0,1,.5), (.5,0,0), (0,.5,0), (0,0,.5)]
# imagenes = []
# for c in colores:
#     liso = np.ones((*shape,3))
#     for i in range(3):
#         liso[:,:,i] *= c[i]
#     imagenes.append(liso)

# for i in range(cant):
#     g.insert_image(imagenes[i])

# plt.imshow(g.grid)
# plt.grid()

if pil_image is not None:
    _PIL_INTERPOLATION_METHODS = {
        'nearest': pil_image.NEAREST,
        'bilinear': pil_image.BILINEAR,
        'bicubic': pil_image.BICUBIC,
    }
    # These methods were only introduced in version 3.4.0 (2016).
    if hasattr(pil_image, 'HAMMING'):
        _PIL_INTERPOLATION_METHODS['hamming'] = pil_image.HAMMING
    if hasattr(pil_image, 'BOX'):
        _PIL_INTERPOLATION_METHODS['box'] = pil_image.BOX
    # This method is new in version 1.1.3 (2013).
    if hasattr(pil_image, 'LANCZOS'):
        _PIL_INTERPOLATION_METHODS['lanczos'] = pil_image.LANCZOS

def load_img(path, grayscale=False, color_mode='rgb', target_size=None,
             interpolation='nearest'):
    """Loads an image into PIL format.
    # Arguments
        path: Path to image file.
        grayscale: DEPRECATED use `color_mode="grayscale"`.
        color_mode: One of "grayscale", "rgb", "rgba". Default: "rgb".
            The desired image format.
        target_size: Either `None` (default to original size)
            or tuple of ints `(img_height, img_width)`.
        interpolation: Interpolation method used to resample the image if the
            target size is different from that of the loaded image.
            Supported methods are "nearest", "bilinear", and "bicubic".
            If PIL version 1.1.3 or newer is installed, "lanczos" is also
            supported. If PIL version 3.4.0 or newer is installed, "box" and
            "hamming" are also supported. By default, "nearest" is used.
    # Returns
        A PIL Image instance.
    # Raises
        ImportError: if PIL is not available.
        ValueError: if interpolation method is not supported.
    """
    if grayscale is True:
        import warnings
        warnings.warn('grayscale is deprecated. Please use '
                      'color_mode = "grayscale"')
        color_mode = 'grayscale'
    if pil_image is None:
        raise ImportError('Could not import PIL.Image. '
                          'The use of `load_img` requires PIL.')
    img = pil_image.open(path)
    if color_mode == 'grayscale':
        if img.mode != 'L':
            img = img.convert('L')
    elif color_mode == 'rgba':
        if img.mode != 'RGBA':
            img = img.convert('RGBA')
    elif color_mode == 'rgb':
        if img.mode != 'RGB':
            img = img.convert('RGB')
    else:
        raise ValueError('color_mode must be "grayscale", "rgb", or "rgba"')
    if target_size is not None:
        width_height_tuple = (target_size[1], target_size[0])
        if img.size != width_height_tuple:
            if interpolation not in _PIL_INTERPOLATION_METHODS:
                raise ValueError(
                    'Invalid interpolation method {} specified. Supported '
                    'methods are {}'.format(
                        interpolation,
                        ", ".join(_PIL_INTERPOLATION_METHODS.keys())))
            resample = _PIL_INTERPOLATION_METHODS[interpolation]
            img = img.resize(width_height_tuple, resample)
    return img