import sys
import nltk
nltk.download(['punkt', 'wordnet'])
import re
import numpy as np
import pandas as pd
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.tree import DecisionTreeClassifier
from sklearn.multioutput import MultiOutputClassifier
from sklearn.metrics import classification_report
from sqlalchemy import create_engine
import pickle

def load_data(database_filepath):
    """Load the file and return the X,Y data"""
    name = 'sqlite:///' + database_filepath
    engine = create_engine(name)
    df = pd.read_sql('select * from InsertTableName', engine)
    X = df['message']
    Y = df.drop(columns=['message','original','genre','id'])
    category_names = Y.columns

    return X,Y, category_names

def tokenize(text):
    """Transform the input text by tokenization and cleaing"""
    url_regex = 'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
    detected_urls = re.findall(url_regex, text)
    for url in detected_urls:
        text = text.replace(url, "urlplaceholder")

    tokens = word_tokenize(text)
    lemmatizer = WordNetLemmatizer()

    clean_tokens = []
    for tok in tokens:
        clean_tok = lemmatizer.lemmatize(tok).lower().strip()
        clean_tokens.append(clean_tok)

    return clean_tokens

def build_model():
    """Builds Pipeline with Grid Search model and classifier"""

    pipeline = Pipeline([
    ('vect', CountVectorizer(tokenizer=tokenize)),
    ('tfidf', TfidfTransformer()),
    ('clf', MultiOutputClassifier(RandomForestClassifier())),
    ])

    parameters = {'clf__estimator__max_depth': [10, 50, None],
              'clf__estimator__min_samples_leaf':[2, 5, 10]}

    cv = GridSearchCV(pipeline, param_grid=parameters)
    return cv

def evaluate_model(model, X_test, Y_test, category_names):
    """Print the evaluation results of the model"""
    y_pred = model.predict(X_test)
    for i, col in enumerate(Y_test.columns):
        print (classification_report(Y_test[col], y_pred.astype(int)[:,i],
            target_names = category_names))

def save_model(model, model_filepath):
    """Save model"""
    pickle.dump(model, open(model_filepath, 'wb'))

def main():
    if len(sys.argv) == 3:
        database_filepath, model_filepath = sys.argv[1:]
        print('Loading data...\n    DATABASE: {}'.format(database_filepath))
        X, Y, category_names = load_data(database_filepath)
        X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)

        print('Building model...')
        model = build_model()

        print('Training model...')
        model.fit(X_train, Y_train) # if not working try X_train.as_matrix(),Y_train.as_matrix()

        print('Evaluating model...')
        evaluate_model(model, X_test, Y_test, category_names)

        print('Saving model...\n    MODEL: {}'.format(model_filepath))
        save_model(model, model_filepath)

        print('Trained model saved!')

    else:
        print('Please provide the filepath of the disaster messages database '\
              'as the first argument and the filepath of the pickle file to '\
              'save the model to as the second argument. \n\nExample: python '\
              'train_classifier.py ../data/DisasterResponse.db classifier.pkl')

if __name__ == '__main__':
    main()

"""Below is the model with the best parameters"""
"""pipeline = Pipeline([
    ('vect', CountVectorizer()),
    ('tfidf', TfidfTransformer()),
    ('clf', MultiOutputClassifier(RandomForestClassifier(max_depth= 10, min_samples_leaf= 5))),
])
"""


from gcsa.google_calendar import GoogleCalendar
from datetime import datetime
from tkinter import *
import webbrowser

root = Tk()
root.title("Google Calendar")
root.geometry("1000x700")
calendar = GoogleCalendar('yourgmail@gmail.com')

y = 10
for event in calendar:
    y += 1
    x = 0
    s = StringVar()
    s = event.start.strftime('%A')
    if datetime.now().strftime('%A') != s:
        break
    date = event.start.strftime('  %m/%d/%y')
    time = event.start.strftime("%I:%M%p")

    print(s, date, '-->', time, event.description)
    label_frame = LabelFrame(root, text=time)

    label_frame.pack(padx=10, pady=10)
    L1 = Label(label_frame, text=s)
    L1.grid(row=y, column=x, padx=10, pady=30)
    x += 10
    L2 = Label(label_frame, text=date)
    L2.grid(row=y, column=x, padx=10, pady=30)
    x += 10
    L3 = Label(label_frame, text=time)
    L3.grid(row=y, column=x, padx=10, pady=30)
    x += 10
    L4 = Label(label_frame, text=event.description)
    L4.grid(row=y, column=x, padx=10, pady=30)
    location = str(event.location)
    print(len(location))
    new = 1

    def OpenUrl(url):
        print(url)
        webbrowser.open(url, new=new)

    if len(location) != 4:
        Btn = Button(root, text="Open Google Maps", command=lambda aurl=location: OpenUrl(aurl)())
        Btn.pack()
root.mainloop()


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import helpers  # Import test helpers module.
import mozunit

helpers.setup()

from GenerateWebIDLBindings import load_and_parse_JSONSchema

def test_all_jsonschema_load_and_parse_smoketest():
    """Make sure it can load and parse all JSONSchema files successfully"""
    schemas = load_and_parse_JSONSchema()
    assert schemas
    assert len(schemas.json_schemas) > 0
    assert len(schemas.api_namespaces) > 0

if __name__ == "__main__":
    mozunit.main()


import logging

from collections import defaultdict
from os.path import abspath
from typing import Any, Callable, Optional

from zelos.api.memory_api import MemoryApi
from zelos.api.regs_api import RegsApi
from zelos.breakpoints import BreakState
from zelos.config_gen import generate_config, generate_config_from_cmdline
from zelos.engine import Engine
from zelos.hooks import HookInfo, HookType
from zelos.plugin import ParsedBinary, Plugins
from zelos.processes import Process
from zelos.threads import Thread

class Zelos:
    """
    Class that provides access to the core APIs. These core APIs
    are event hooking, debugging, memory access, register access, and
    emulation context.

    Args:
        filename: Specifies the name of the file to emulate.
        cmdline_args: Arguments that are passed to the emulated binary.
        flags: Parameters for zelos. To see the list of all flags, refer
            to :ref:`flag-label`

    Example:
        .. code-block:: python

            from zelos import Zelos

            # initialize zelos with binary name, 2 cmdline args, and
            # turn on the inst feed to print instructions.
            z = Zelos(
                "binary_to_emulate"
                "ARG1",
                "ARG2",
                inst=True,
            )
    """

    def __init__(self, filename, *cmdline_args, **flags):
        config = generate_config(filename, *cmdline_args, **flags)
        self._setup(config)

    def _setup(self, config):
        self.config = config
        self._regs = RegsApi(self)
        self._memory = MemoryApi(self)

        self._breakpoints = {}
        self._watchpoints = defaultdict(dict)

        # If you need to access data that is not exposed through the api
        # yet, access the internal_engine representation at your own
        # risk.
        e = Engine(config=config, api=self)
        self.plugins = Plugins(self, ["plugins"])
        self.plugins.initialize()
        self.internal_engine = e  # Used to inform what type this is.
        self.internal_engine.plugins = self.plugins

        # After setup allow MEMORY.INTERNAL_* hooks to run.
        e.hook_manager._enable_internal_memory_hooks()

    # **** Memory API ****
    @property
    def memory(self):
        """
        Returns the :py:class:`~zelos.api.memory_api.MemoryApi` object.
        """
        return self._memory

    # **** Registers API ****
    @property
    def regs(self):
        """
        Returns the :py:class:`~zelos.api.regs_api.RegsApi` object.
        """
        return self._regs

    @property
    def logger(self):
        """
        Returns a logger that will behave in accordance to the --log
        config/cmdline option.
        """
        return logging.getLogger(__name__)

    # **** Begin Hook API ****
    def hook_memory(
        self,
        hook_type: HookType.MEMORY,
        callback: Callable[["Zelos", int, int, int, int], Any],
        mem_low: Optional[int] = None,
        mem_high: Optional[int] = None,
        name: Optional[str] = None,
        end_condition: Optional[Callable[[], bool]] = None,
    ) -> HookInfo:
        """
        Registers a hook on memory. Executes callback every time the
        specified event happens in memory.

        The hook will only trigger when the event occurs at an address
        between mem_low and mem_high, if either of them are specified.

        The hook will continue to trigger until the end_condition
        specified evaluates to True.

        Args:
            hook_type: Specifies the event in memory that should trigger
                the callback to be executed. Options can be found in
                :py:class:`zelos.HookType.MEMORY`
            callback: The code that should be executed when the
                specified event occurs. The function should accept the
                following inputs: (zelos, access, address, size, value).
                The return value of "callback" is ignored.
            mem_low: If specified, only executes callback if the
                event occurs at an address greater than or equal to
                this.
            mem_high: If specified, only executes callback if the
                event occurs at an address less than or equal to this.
            name: An identifier for this hook. Used for debugging.
            end_condition: If specified, executes after the callback. If
                the function returns True, this hook is deleted.

        Returns:
            Information regarding the hook.

        Example:
            .. code-block:: python

                from zelos import Zelos, HookType

                # Print every write to memory
                def memory_hook(zelos, access, address, size, value):
                    print(value)

                z = Zelos("binary_to_emulate")
                z.hook_memory(
                    HookType.MEMORY.WRITE,
                    memory_hook
                )
                z.start()
        """

        return self.internal_engine.hook_manager.register_mem_hook(
            hook_type,
            callback,
            mem_low=mem_low,
            mem_high=mem_high,
            name=name,
            end_condition=end_condition,
        )

    def hook_execution(
        self,
        hook_type: HookType.EXEC,
        callback: Callable[["Zelos", int, int], Any],
        ip_low: Optional[int] = None,
        ip_high: Optional[int] = None,
        name: Optional[str] = None,
        end_condition: Optional[Callable[[], bool]] = None,
    ) -> HookInfo:
        """
        Registers a hook that executes when code is executed. This is
        either for every instruction that is executed, or every block.

        The hook will only trigger when the event occurs at an address
        between ip_low and ip_high, if either of them are specified.

        The hook will continue to trigger until the end_condition
        specified evaluates to True.

        Args:
            hook_type: Specifies whether the callback should be
                triggered every instruction, or every block. Options can
                be found in :py:class:`zelos.HookType.EXEC`
            callback: The code that should be executed when the
                specified event occurs. The function should accept the
                following inputs: (zelos, address, size).
                The return value of "callback" is ignored.
            ip_low: If specified, only executes callback if the
                event occurs at an address greater than or equal to
                this.
            ip_high: If specified, only executes callback if the
                event occurs at an address less than or equal to this.
            name: An identifier for this hook. Used for debugging.
            end_condition: If specified, executes after the callback. If
                the function returns True, this hook is deleted.

        Returns:
            Information regarding the hook.

        Example:
            .. code-block:: python

                from zelos import Zelos, HookType

                # Print the first address of every block
                def exec_hook(zelos, address, size):
                    print(address)

                z = Zelos("binary_to_emulate")
                z.hook_execution(
                    HookType.EXEC.BLOCK, exec_hook
                )
                z.start()
        """
        return self.internal_engine.hook_manager.register_exec_hook(
            hook_type,
            callback,
            ip_low=ip_low,
            ip_high=ip_high,
            name=name,
            end_condition=end_condition,
        )

    def hook_close(self, callback: Callable[[], Any]) -> HookInfo:
        """
        Registers a callback that is called when
        :py:meth:`zelos.Engine.close()` is called.

        Args:
            callback: Called when zelos is closed. Does not take any
                arguments. The return value of `callback` is ignored

        Example:
            .. code-block:: python

                from zelos import Zelos

                # Close a file you are using with zelos
                file = open("testfile", "r")
                def close_cleanup():
                    file.close()

                z = Zelos("binary_to_emulate")
                z.hook_close(close_cleanup)
                z.start()

                # Hooks are run at this point
                z.close()
        """
        return self.internal_engine.hook_manager.register_close_hook(callback)

    def hook_syscalls(
        self,
        syscall_hook_type: HookType.SYSCALL,
        callback: Callable[["Zelos", str, "Args", int], Any],
        name: str = None,
        syscall_name: str = None,
    ) -> HookInfo:
        """
        Registers a callback that is called when a syscall is invoked.

        Args:
            syscall_hook_type: Decides when the hook should be triggered
                in relation to the execution of the syscall. Options can
                be found in :py:class:`zelos.HookType.SYSCALL`
            callback: The code that should be executed when the
                specified event occurs. The function should accept the
                following inputs:
                (zelos, syscall_name, args, return_value)
                The return value of "callback" is ignored.
            syscall_name: Restricts calls of the hook to syscalls that
                match the given name.
            name: An identifier for this hook. Used for debugging.

        Example:
            .. code-block:: python

                from zelos import Zelos, HookType

                # Keep track of the syscall return values
                syscall_return_values = []
                def syscall_hook(zelos, sys_name, args, ret_val):
                    syscall_return_values.append((sys_name, ret_val))

                z = Zelos("binary_to_emulate")
                z.hook_syscalls(
                    HookType.SYSCALL.AFTER, syscall_hook
                )
                z.start()

        """
        return self.internal_engine.hook_manager.register_syscall_hook(
            syscall_hook_type, callback, name=name, syscall_name=syscall_name
        )

    def hook_zml(
        self, zml_string: str, callback: Callable[[], Any]
    ) -> HookInfo:
        """
        Register a callback that triggers when a given ZML string is
        satisfied. For more information on ZML, view
        :py:class:`zelos.zml.ZmlParser`.

        Args:
            zml_string: Specifies condition to call callback.
            callback: The code that should be executed when the specified
                event occurs.
        """
        return self.internal_engine.hook_manager.register_zml_hook(
            zml_string, callback
        )

    def delete_hook(self, hook_info: HookInfo) -> None:
        self.internal_engine.hook_manager.delete_hook(hook_info)

    # **** Begin Debugging API ****

    def start(self, timeout: float = 0) -> Optional[BreakState]:
        """
        Begin emulation. When called for the first time, begins
        execution at the binary entry point. If the emulation is
        stopped (for example, after calling :py:meth:`stop()`) this
        will resume execution from the current IP.

        Args:
            timeout: Stops execution after `timeout` seconds.

        Example:
            .. code-block:: python

                from zelos import Zelos

                z = Zelos("binary_to_emulate")

                # Start execution from the entry point
                z.start()

        """
        return self.internal_engine.start(timeout=timeout)

    def step(self, count=1) -> None:
        """
        Begin emulation, stopping after executing `count` instructions.

        Args:
            count: Maximum number of instructions to execute before
                stopping
        """
        return self.internal_engine.step(count=count)

    def next(self, count=1) -> None:
        """
        Begin emulation, executing `count` instructions not including
        instructions inside function calls.

        Args:
            count: Maximum number of instructions to execute before
                stopping.
        """
        return self.internal_engine.step_over(count=count)

    def stop(self, reason: str = "plugin"):
        """
        Stop the Zelos run loop. After a call to
        :py:meth:`stop()`, execution can be resumed from the
        current IP with a call to :py:meth:`start()`.

        Args:
            reason: An optional identifier that specifies a reason for
                stopping execution. Upon calling stop, the reason will
                be printed to stdout after Zelos exits the run loop.
                This is useful for debugging when log level is set to
                'debug' or 'spam'.
        """
        self.internal_engine.scheduler.stop(reason)

    def close(self):
        """
        Closes Zelos and runs cleanup functions.
        """
        self.internal_engine.close()

    def end_thread(self):
        """
        End the current thread. Marks current thread as successfully
        completed and swaps to the next available thread, if one exists.
        """
        self.internal_engine.thread_manager.complete_current_thread()

    def swap_thread(self, reason: str = "thread swap"):
        """
        Swap the running thread with the next scheduled thread.

        Args:
            reason: An optional identifier that specifies a reason for
                swapping threads. Upon calling swap_thread, the reason
                will be printed to stdout after Zelos has successfully
                swapped to the next scheduled thread. This is useful
                for debugging when log level is set to 'spam'.
        """
        self.internal_engine.scheduler.stop_and_exec(
            reason, self.internal_engine.processes.schedule_next
        )

    def set_breakpoint(self, address: int, temporary: bool = False):
        """
        Set a breakpoint at a particular address.

        Args:
            address: Target address of breakpoint.
            temporary: Determines whether or not the breakpoint is
                temporary. A temporary breakpoint will be automatically
                removed after use.

        Example:
            .. code-block:: python

                from zelos import Zelos

                z = Zelos("binary_to_emulate")

                z.set_breakpoint(0xdeadbeef)

                z.start()

        """
        self.internal_engine.breakpoints.set_breakpoint(
            address, temporary=temporary
        )

    def remove_breakpoint(self, address: int):
        """
        Remove a previously set breakpoint.

        Args:
            address: Target address of breakpoint to remove.

        Example:
            .. code-block:: python

                from zelos import Zelos

                z = Zelos("binary_to_emulate")

                z.set_breakpoint(0xdeadbeef)

                z.remove_breakpoint(0xdeadbeef)

                z.start()

        """
        self.internal_engine.breakpoints.remove_breakpoint(address)

    def set_syscall_breakpoint(self, syscall_name: str):
        """
        Set a breakpoint at all syscalls of a specified name.

        Args:
            syscall_name: Target syscall set breakpoint at.

        Example:
            .. code-block:: python

                from zelos import Zelos

                z = Zelos("binary_to_emulate")

                z.set_syscall_breakpoint("write")

                z.start()

        """
        self.internal_engine.kernel.set_breakpoint(syscall_name)

    def remove_syscall_breakpoint(self, syscall_name: str):
        """
        Remove a previously set syscall breakpoint specified by name.

        Args:
            syscall_name: Target syscall to remove breakpoints from.

        Example:
            .. code-block:: python

                from zelos import Zelos

                z = Zelos("binary_to_emulate")

                z.set_syscall_breakpoint("write")

                z.remove_syscall_breakpoint("write")

                z.start()

        """
        self.internal_engine.kernel.remove_breakpoint(syscall_name)

    def set_watchpoint(
        self, address: int, read: bool, write: bool, temporary: bool = False
    ):
        """
        Set a watchpoint on a particular memory address.

        Args:
            address: Target address of watchpoint.
            read: Determines whether to watch for reads to the target
                memory address.
            write: Determines whether to watch for writes to the target
                memory address.

        Example:
            .. code-block:: python

                from zelos import Zelos

                z = Zelos("binary_to_emulate")

                # Break at any read or write to memory address 0xdeadbeef
                z.set_watchpoint(0xdeadbeef, True, True)

                # Break only at writes to memory address 0xfeedf00d
                z.set_watchpoint(0xfeedf00d, False, True)

                # Break only at reads to memory address 0xb0bad00d
                z.set_watchpoint(0xb0bad00d, True, False)

                z.start()

        """

        def hook(zelos, access, address, size, value):
            zelos.stop("watchpoint")

        if read:
            read_hook_info = self.hook_memory(
                HookType.MEMORY.READ,
                hook,
                name=f"read_watchpoint_{address:x}",
                mem_low=address,
                mem_high=address,
                end_condition=lambda: temporary,
            )
            self._watchpoints[address]["read"] = read_hook_info
        if write:
            write_hook_info = self.hook_memory(
                HookType.MEMORY.WRITE,
                hook,
                name=f"write_watchpoint_{address:x}",
                mem_low=address,
                mem_high=address,
                end_condition=lambda: temporary,
            )
            self._watchpoints[address]["write"] = write_hook_info

        return True

    def remove_watchpoint(self, address: int):
        """
        Remove a previously set watchpoint.

        Args:
            address: Target address of watchpoint to remove.

        Example:
            .. code-block:: python

                from zelos import Zelos

                z = Zelos("binary_to_emulate")

                z.set_watchpoint(0xdeadbeef, True, True)

                z.remove_watchpoint(0xdeadbeef)

                z.start()

        """
        for hook_info in self._watchpoints[address].values():
            self.internal_engine.hook_manager.delete_hook(hook_info)
        del self._watchpoints[address]

        return True

    # **** Begin Context API ****

    @property
    def date(self):
        """
        Returns the date string used internally during emulation. The
        date string format is `YYYY-MM-DD`.

        :getter: Returns the date string in YYYY-MM-DD format.
        :setter: Sets the date string. Input must be YYYY-MM-DD format.
        :type: str
        """
        return self.internal_engine.date

    @date.setter
    def date(self, date_str: str):
        """
        Set the date to be used for emulation. This affects the linux
        system calls sys_time, sys_gettimeofday, and sys_clock_gettime.

        Args:
            date_str: A string of the form `YYYY-MM-DD` specifying the
                target date.

        Example:
            .. code-block:: python

                from zelos import Zelos

                z = Zelos("binary_to_emulate")

                z.date = "2020-03-04"

                z.start()

        """
        self.internal_engine.date = date_str

    @property
    def process(self) -> Process:
        """
        Returns the currently active process.

        :type: Process

        """
        return self.internal_engine.current_process

    @property
    def thread(self) -> Thread:
        """
        Returns the currently active thread.

        :type: Thread

        """
        return self.internal_engine.current_process.current_thread

    @property
    def main_binary(self) -> Optional[ParsedBinary]:
        """
        Returns the parsed main binary, if it exists, otherwise returns
        None.
        Note that the "main" binary denotes the binary that is loaded by
        Zelos during emulation, not necessarily the binary that
        is specified as input (the "target" binary). When the specified
        input binary ("target") is statically linked, it is also the
        "main" binary. However, when the specified input binary
        ("target") is dynamically linked, the "main" binary instead
        refers to the dynamic linker/loader.

        :type: :py:class:`zelos.plugin.ParsedBinary`

        """
        return self.internal_engine.main_module

    @property
    def main_binary_path(self) -> Optional[str]:
        """
        Returns the absolute path to the main binary, if it exists, otherwise
        returns None.
        Note that the "main" binary denotes the binary that is loaded by
        Zelos during emulation, not necessarily the binary that
        is specified as input (the "target" binary). When the specified
        input binary ("target") is statically linked, it is also the "main"
        binary. However, when the specified input binary ("target") is
        dynamically linked, the "main" binary instead refers to the dynamic
        linker/loader.

        :type: str

        """
        if self.internal_engine.main_module_name:
            return abspath(self.internal_engine.main_module_name)
        return None

    @property
    def target_binary_path(self) -> Optional[str]:
        """
        Returns the absolute path to the target binary, if it exists,
        otherwise returns None. Note that the "target" binary denotes
        the binary specified as input to Zelos.

        :type: str

        """
        if self.internal_engine.target_binary_path:
            return abspath(self.internal_engine.target_binary_path)
        return None

class ZelosCmdline(Zelos):
    """
    A workaround for allowing `Zelos` to be initialized with a
    commandline string while also allowing the main `Zelos` to maintain
    its simple/intuitive constructor based on commandline arguments and
    flags.
    """

    def __init__(self, cmdline_args):
        config = generate_config_from_cmdline(cmdline_args)
        self._setup(config)


# Basic lambda expression
add = lambda x, y: x + y

result = add(3, 5)
print(result)  # Output: 8

# Lambda functions as arguments
numbers = [5, 2, 8, 1, 9]
sorted_numbers = sorted(numbers, key=lambda x: x % 3)

print(sorted_numbers)  # Output: [9, 1, 5, 2, 8]

# Lambda functions as arguments
pairs = [(1, 'one'), (2, 'two'), (3, 'three'), (4, 'four')]
pairs.sort(key=lambda pair: pair[1])

print(pairs)  # Output: [(4, 'four'), (1, 'one'), (3, 'three'), (2, 'two')]

# Lambda functions for mapping
numbers = [1, 2, 3, 4, 5]
squared_numbers = list(map(lambda x: x ** 2, numbers))

print(squared_numbers)  # Output: [1, 4, 9, 16, 25]

# Lambda functions for filtering
numbers = [1, 2, 3, 4, 5]
even_numbers = list(filter(lambda x: x % 2 == 0, numbers))

print(even_numbers)  # Output: [2, 4]


import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
from sklearn import metrics
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split

from src.data.preprocess.hatespeech.preprocess_rdforest_hatespeech import (
    generate_tfidf_vectors,
)
from src.features.hatespeech.featureseng_helperfunc_hatespeech import (
    combine_feature_space,
)
from src.features.hatespeech.featureseng_rdforest_hatespeech import (
    contains_not_sexist_but,
)
from src.utils.preprocess_text_helpers import normalizer

DATA = pd.read_csv("../../../DATA/external/hatespeech/hs_data.csv")

DATA["normalized"] = normalizer(DATA["text"])
X = generate_tfidf_vectors(DATA["normalized"].to_list())["vectors"]

DATA["contains_not_sexist_but"] = DATA["text"].apply(contains_not_sexist_but)
X = combine_feature_space(X, DATA["contains_not_sexist_but"].values)

Y = pd.get_dummies(DATA["annotation"])["misogynistic"]
X_TRAIN, X_TEST, Y_TRAIN, Y_TEST = train_test_split(X, Y, test_size=0.3)

CLF = RandomForestClassifier(
    bootstrap=True,
    class_weight=None,
    criterion="gini",
    max_depth=None,
    max_features="auto",
    max_leaf_nodes=None,
    min_impurity_decrease=0.0,
    min_impurity_split=None,
    min_samples_leaf=1,
    min_samples_split=2,
    min_weight_fraction_leaf=0.0,
    n_estimators=100,
    n_jobs=1,
    oob_score=False,
    random_state=None,
    verbose=0,
    warm_start=False,
)

CLF.fit(X_TRAIN, Y_TRAIN)

FEATURE_NAMES = generate_tfidf_vectors(DATA["normalized"].to_list())[
    "FEATURE_NAMES"
] + ["contains_not_sexist_but"]
FEATURE_IMPORTANCE = pd.Series(
    CLF.feature_importances_, index=FEATURE_NAMES
).sort_values(ascending=False)

Y_PRED = CLF.predict(X_TEST)

# Model Accuracy, how often is the classifier correct?
print("Accuracy:", metrics.accuracy_score(Y_TEST, Y_PRED))

# Creating a bar plot
sns.barplot(x=FEATURE_IMPORTANCE[0:10], y=FEATURE_IMPORTANCE.index[0:10])
# Add labels to your graph
plt.xlabel("Feature Importance Score")
plt.ylabel("Features")
plt.title("Visualizing Important Features")
plt.legend()
plt.show()


#find([1,2,3,4,5,6], lambda x: x>4)
# lst = [1,2,3,4,5,6]
# print(list(i for i in lst if i > 4)[0])

# res = next(x for x, val in enumerate(lst) if val > 4)
# print(res)

# res1 = list(filter(lambda i: i > 4, lst))[0]
# print(res1)

# res4 = list(filter(lambda x: x%2 == 0, lst))
# print(res4)

# print(set(lst) - set(res4))

students = [
         {'first_name':  'Michael', 'last_name' : 'Jordan'},
         {'first_name' : 'John', 'last_name' : 'Rosales'},
         {'first_name' : 'Mark', 'last_name' : 'Guillen'},
         {'first_name' : 'KB', 'last_name' : 'Tonel'}
    ]

# should output: (it's okay if each key-value pair ends up on 2 separate lines;
# bonus to get them to appear exactly as below!)
# first_name - Michael, last_name - Jordan
# first_name - John, last_name - Rosales
# first_name - Mark, last_name - Guillen
# first_name - KB, last_name - Tonel

def iterateDictionary(students) :
    for x in students:

        #result = " ".join(str(key) + " - " + str(value) for key, value in item.items())
        print({key: value for key, value in x.items()})

iterateDictionary(students)

dict1 = {'a': 1, 'b': 2, 'c': 3, 'd': 4}
# Put all keys of `dict1` in a list and returns the list
print(dict1.keys())
print(dict1.values())

print(dict1)
dict1_keys = {k : v for (k,v) in dict1.items()}
print(*dict1_keys)


from django.shortcuts import render,redirect

from django.http import JsonResponse

# Create your views here.

import json

# 導入 model
from .models import *

# 用戶查詢,訂單資料
def api_query_user_order_view(request):
    if request.method == 'POST':
        if 'user_name' in request.session and 'uid' in request.session:
            uid = request.session['uid'][0]
            user_order_number = request.POST['order_number']
            user_order_date_begin = request.POST['order_date_begin']
            user_order_date_end = request.POST['order_date_end']

            #  根據前端發送的值做判斷查詢(如果沒有 user_order_number ,則直接查詢日期的區間)
            if user_order_number == '':
                #  1.多筆查詢
                user_order_msg = Order_info.objects.filter(user_id=uid, order_time__range=(user_order_date_begin, user_order_date_end)).order_by('-id')
            else:
                #  2.查詢單筆
                # 如果有 user_order_number 則只要查詢 訂單編號即可
                user_order_msg = Order_info.objects.filter(user_id=uid, order_number=user_order_number)

            # 組數據給前端設置頁面
            user_order_msg_dict = {
                'user_order_status': 0,
            }

            # 設置 user_order_msg_list 來組 訂單數據格式
            user_order_msg_list = []

            # 如果 user_order_msg 存在,則表示有數據
            if user_order_msg:
                #  將 user_order_msg 狀態 改為 1
                user_order_msg_dict['user_order_status'] = 1

                for i in user_order_msg:
                    user_order_info_dic = {
                        'recipient_name': i.recipient_name,
                        'recipient_phone_number': i.recipient_phone_number,
                        'recipient_address': i.recipient_address,
                        'order_total_price': i.order_total_price,
                        'order_number': i.order_number,
                        'order_time': i.order_time,
                        'order_status': i.order_status,
                        'order_check_time': i.order_check_time,
                        'order_pay_chose_pay': i.pay.chose_pay,
                        'order_pay_send_price': i.pay.send_price,
                        'user_order_info_detail': [],
                    }
                    for j in i.order_info_detail_set.filter(orders_id=i.id):
                        user_order_info_dic['user_order_info_detail'].append({
                            'goods_id': j.goods_id,
                            'goods_name': j.goods.title,
                            'goods_img_url': j.goods_img_url,
                            'goods_color': j.goods_color,
                            'goods_color_img': str(Goods_color.objects.filter(goods_id=j.goods_id, goods_color=j.goods_color)[0].goods_color_img),
                            'goods_size': j.goods_size,
                            'goods_amount': j.goods_amount,
                            'goods_price': j.goods_price,
                            'goods_singo_total_price': j.goods_total_price,
                        })

                    user_order_msg_list.append(user_order_info_dic)

                user_order_msg_dict['user_order_info_msg'] = user_order_msg_list
            return JsonResponse(user_order_msg_dict)

        else:
            return redirect('/')
    else:
        return redirect('/')


from marvin.tools.maps import Maps
from marvin.tools.cube import Cube
import matplotlib.pyplot as plt

maps = Maps(plateifu='8714-12703')
print(maps)
# get an emission line map
haflux = maps.emline_gflux_ha_6564
values = haflux.value
ivar = haflux.ivar
mask = haflux.mask
haflux.plot()

cube = Cube(plateifu='8714-12703')
# get a spaxel by slicing cube[i,j]
spaxel = cube[16, 16]
flux = spaxel.flux
wave = flux.wavelength
ivar = flux.ivar
mask = flux.mask
flux.plot()

# plt.show()


from rest_framework import serializers
from datasets.models import DatasetLabel
from datasets.models.dataset import Dataset

class DatasetLabelSerializer(serializers.ModelSerializer):
    class Meta:
        model = DatasetLabel
        exclude = ['dataset']

    def create(self, validated_data):
        dataset_id = self.context.get('view').kwargs.get('dataset_pk')
        dataset = Dataset.objects.get(id=dataset_id)
        validated_data['dataset'] = dataset
        return super().create(validated_data)


"""Defines URL patterns for Project_logs"""

from django.conf.urls import url
from . import views

urlpatterns = [

	url(r'^$', views.index, name='index'),
    url(r'^all-posts/$', views.contents, name='contents'),
    url(r'^post/(?P<slug>[-\w]+)/$', views.post, name='post'),
    url(r'^new-post/$', views.new_post, name='new_post'),
    url(r'^add-images/(?P<slug>[-\w]+)/$', views.ImageView.as_view(), name='add_images'),
]


from ipaGenerator import IPAGenerator

class SuggestionGenerator:

    def __init__(self, data, ipaOfWord):
        self.data = data
        self.ipaOfWord = ipaOfWord
        for i in range(len(self.data)):
            self.data[i]['ed'] = 0

    # butil = BengaliUtilities()
    # #print(butil.isPunctuationMark("?"))
    # wp = BengaliWordProcessor("আমার?তোমার")
    # print(wp.removePunctuationMarks())
# str1ipa = IPAGenerator(string1).getIPA()
# print(string1, end=" ")
# print(str1ipa)

# editDistance(str1, data[i]['words'], len(str1), len(data[i]['words']))

    def editDistance(self, str1, str2, m, n):
        # If first string is empty, the only option is to
        # insert all characters of second string into first
        if m == 0:
            return n

            # If second string is empty, the only option is to
        # remove all characters of first string
        if n == 0:
            return m

            # If last characters of two strings are same, nothing
        # much to do. Ignore last characters and get count for
        # remaining strings.
        if str1[m - 1] == str2[n - 1]:
            return self.editDistance(str1, str2, m - 1, n - 1)

            # If last characters are not same, consider all three
        # operations on last character of first string, recursively
        # compute minimum cost for all three operations and take
        # minimum of three values.
        return 1 + min(self.editDistance(str1, str2, m, n - 1),  # Insert
                       self.editDistance(str1, str2, m - 1, n),  # Remove
                       self.editDistance(str1, str2, m - 1, n - 1)  # Replace
                       )

    def editDistanceGenerator(self):
        #most expensive code
        for i in range(len(self.data)):
            difference = (len(self.ipaOfWord) - len(self.data[i]['ipa']))
            if( (difference < 5 and difference > 5) or (difference > -5 and difference < 5) ):
                self.data[i]['ed'] = self.editDistance(self.ipaOfWord, self.data[i]['ipa'], len(self.ipaOfWord), len(self.data[i]['ipa']))
            else:
                self.data[i]['ed'] = 100
        # most expensive code

        self.data.sort(key=lambda x: x['ed'], reverse=False)
        count = 0
        for i in range(len(self.data)):
            if self.data[i]['ed'] < 10:
                count = count+1
        return self.data[0:5]


import pandas as pd
import re
from sklearn.model_selection import train_test_split
import torch
from transformers import T5ForConditionalGeneration, T5Tokenizer, Seq2SeqTrainer, Seq2SeqTrainingArguments

#load data
data = pd.read_csv('train.csv')

#function to replace number0 (etc) values with actual numbers
def replace_number_placeholders(row):
    def replace_number(match):
        number_index = int(match.group(1))  #extract number index
        return str(row['Numbers'].split(' ')[number_index])

    return re.sub(r'number(\d+)', replace_number, row['Question'])

#ensure 'ques' is a string
data['Question'] = data['Question'].astype(str)

#replace number placeholders
data['Processed_Question'] = data.apply(replace_number_placeholders, axis=1)

#train/val split
train_data, val_data = train_test_split(data, test_size=0.2, random_state=42)

train_questions = train_data['Processed_Question'].tolist()
train_answers = train_data['Answer'].apply(lambda x: str(x)).tolist()

val_questions = val_data['Processed_Question'].tolist()
val_answers = val_data['Answer'].apply(lambda x: str(x)).tolist()

print(val_data)

#initializing tokenizer
tokenizer = T5Tokenizer.from_pretrained('t5-small')

#tokenize input and output sequences
tokenized_train_inputs = tokenizer(train_questions, return_tensors='pt', padding=True, truncation=True)
tokenized_train_outputs = tokenizer(train_answers, return_tensors='pt', padding=True, truncation=True)

tokenized_val_inputs = tokenizer(val_questions, return_tensors='pt', padding=True, truncation=True)
tokenized_val_outputs = tokenizer(val_answers, return_tensors='pt', padding=True, truncation=True)

device = 'cuda' if torch.cuda.is_available() else 'cpu'

train_dataset = [
    {
        'input_ids': tokenized_train_inputs['input_ids'][i].to(device),
        'attention_mask': tokenized_train_inputs['attention_mask'][i].to(device),
        'decoder_input_ids': tokenized_train_outputs['input_ids'][i].to(device),
        'decoder_attention_mask': tokenized_train_outputs['attention_mask'][i].to(device),
        'labels': tokenized_train_outputs['input_ids'][i].to(device).clone()
    }
    for i in range(len(tokenized_train_inputs['input_ids']))
]

val_dataset = [
    {
        'input_ids': tokenized_val_inputs['input_ids'][i].to(device),
        'attention_mask': tokenized_val_inputs['attention_mask'][i].to(device),
        'decoder_input_ids': tokenized_val_outputs['input_ids'][i].to(device),
        'decoder_attention_mask': tokenized_val_outputs['attention_mask'][i].to(device),
        'labels': tokenized_val_outputs['input_ids'][i].to(device).clone()
    }
    for i in range(len(tokenized_val_inputs['input_ids']))
]

#define training arguments
training_args = Seq2SeqTrainingArguments(
    output_dir='./results',
    per_device_train_batch_size=32,
    per_device_eval_batch_size=32,
    logging_dir='./logs',
    logging_steps=100,
    save_steps=1000,
    evaluation_strategy='epoch',
    eval_steps=500,
    num_train_epochs=100,
    predict_with_generate=True
)

#define encoder-decoder model
model = T5ForConditionalGeneration.from_pretrained('t5-small')

#define optimizer and instantiate Seq2SeqTrainer
optimizer = torch.optim.Adam(model.parameters(), lr=1e-5)
trainer = Seq2SeqTrainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    eval_dataset=val_dataset,
    tokenizer=tokenizer,
    optimizers=(optimizer, None)
)

#train model
trainer.train()

#display training metrics
train_metrics = trainer.evaluate()
print(f"Training metrics: {train_metrics}")

def preprocess_word_problem(problem_text):
    #tokenize the problem text
    tokenized_problem = tokenizer(problem_text, return_tensors='pt', padding=True, truncation=True)
    return tokenized_problem

#sample word problem
word_problem = "Paul has 3 books. He gives 1 book to Amelia. How many books does Paul have now?"

print("Word problem:", word_problem)

#preprocess word problem
device = 'cuda:0'
tokenized_input = preprocess_word_problem(word_problem)
tokenized_input = {key: tensor.to(device) for key, tensor in tokenized_input.items()}
#print("tokenized input:",tokenized_input)

#generate answer
output = model.generate(input_ids=tokenized_input['input_ids'],
                        attention_mask=tokenized_input['attention_mask'],
                        max_length=200)

#decode the generated output tokens to text
prediction = tokenizer.decode(output[0], skip_special_tokens=True)

print("Answer:", prediction)


"""base module for Importing python files and manipulating the data structures before integrating into  Object"""

import pkgutil
import os
from datetime import datetime
from importlib.machinery import SourceFileLoader
from myfunctions.listmods import to_list
import importlib
from importlib.abc import Loader
import glob

class ImportRepository:
    """Main class that contains importing relating functions and methods"""

    def __init__(self):
        self.generic_dictionary = {}

    def load_execute_modules(
            self,
            executed_function,
            input_package,
            *kwargs,
            given_module=None):
        """Load the Variables from a package and all contained submodules and classes,
        Define a specific package or A module (file) from the package
        """
        input_package = to_list(input_package)

        for package in input_package:
            detected_modules = self.detect_submodules(package)
            if given_module is None:
                for module_name in detected_modules:
                    full_module_name = package + '.' + module_name
                    self.generic_dictionary.update(
                        self._execute_function(
                            executed_function,
                            full_module_name,
                            module_name,
                            *kwargs))
            else:
                full_module_name = input_package[0] + '.' + given_module
                self.generic_dictionary.update(
                    self._execute_function(
                        executed_function,
                        full_module_name,
                        given_module,
                        *kwargs))
            return self.generic_dictionary

    @staticmethod
    def _execute_function(functionname, *args):
        return functionname(*args)

    @staticmethod
    def rpo(object_name):
        """Ignores an object that is an internal or private variable and any functions"""
        if not object_name.startswith('__') and not object_name.startswith(
                '_') and not callable(object_name):
            return object_name
        return None

    @staticmethod
    def path_finder(object_path, start_path=None, walk=True):
        """Smart path finder that searches from a start path to find nearest match and continues to search up to the root directory"""
        start_time = datetime.now()
        user_base, user_ext = os.path.splitext(object_path)
        unicode_path_base = user_base.replace('/', '\\')
        full_name = None
        if os.path.exists(object_path) is True:
            return os.path.abspath(object_path)
        if start_path:
            dir_path = start_path
        else:
            dir_path = os.path.dirname(os.path.realpath(__file__))
        split_paths = dir_path.split('\\')
        directory = split_paths[0]
        for i in range((len(split_paths)), 0, -1):
            crawler = split_paths[:i]
            joiner = '\\'.join(crawler)
            if walk is False:
                joiner = dir_path
            for root, dirs, files in os.walk(joiner):
                if user_ext is None or user_ext == '':
                    for name in dirs:
                        full_name = os.path.join(root, name)
                        if unicode_path_base == name:
                            if full_name:
                                return str(os.path.realpath(full_name))
                else:
                    for name in files:
                        (base, ext) = os.path.splitext(
                            name)  # split base and extension
                        if ext == user_ext:  # check the extension
                            if base in unicode_path_base:
                                full_name = os.path.join(root, name)
                                if unicode_path_base in full_name:
                                    if full_name:
                                        return str(os.path.realpath(full_name))
        timediff = (datetime.now() - start_time)
        time_s = (timediff.seconds + ((timediff.microseconds) / (1000000)))
        msg = (
            f'The system was searched recursively up to the {directory} drive, yet no file or folder path containing {object_path} was found. Check your provided file path and extension.\n (Processing time: {time_s} seconds) ')
        raise ValueError(msg)

    def filename(self, path):
        return os.path.basename(os.path.splitext(path)[0])

    def locate_package(self, packagename, walk=True, init=True):
        """This method verifies the existence of a package and returns the full path
        arguments:
        filename: The suspected name of the file being searched for
            ex: "path_manipulation.py"
        walk: walk is a boolean argument. A value of "True" will allow the file finder to search
              recursively on the current drive.
         """
        initstring = r'\__init__.py'
        if not isinstance(walk, bool):
            raise TypeError(
                'walk is a boolean argument, must be either True or False')
        if init is True:
            packagename = packagename + initstring
        return self.path_finder(packagename, walk=walk)

    def detect_submodules(self, package, init=True):
        """Import all submodules of a module, recursively, including subpackages"""
        package_path = self.locate_package(package, walk=True, init=init)
        found_package = SourceFileLoader(
            package, package_path).load_module(package)
        results = []
        for loader, name, is_pkg in pkgutil.walk_packages(
                found_package.__path__):
            results.append(name)
        return results

    def locate_python_subfiles(self, dir_path):
        dir_path = self.path_finder(dir_path)
        return [
            os.path.join(
                dir_path,
                f) for f in os.listdir(dir_path) if os.path.isfile(
                os.path.join(
                    dir_path,
                    f)) and os.path.splitext(f)[1] == '.py']

    def import_class(self, module_path, class_name=None):
        try:
            assert os.path.isfile(module_path) is True
        except AttributeError:
            raise ImportError(
                "Module file does not exist, ensure full path has been given")
        module_name = self.filename(module_path)
        if not class_name:
            class_name = module_name
        spec = importlib.util.spec_from_file_location(module_name, module_path)
        module = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(module)
        try:
            klass = getattr(module, class_name)
            return klass, class_name
        except AttributeError as abe:
            raise ImportError(
                f"A module named {module_name}.py was found but it did not have a class named {class_name}") from abe

    def find_files(self, path, patterns):
        filelist = []
        for pattern in patterns:
            filelist = filelist + \
                glob.glob(f"{path}/**/{pattern}", recursive=True)
        return filelist


import data_preparation_functions as prepfunctions
import mlxtend.frequent_patterns as pm
import matplotlib.pyplot as plt
import ds_functions as ds
import pandas as pd
import os

graphsDir = './Results/Pattern Mining/'
if not os.path.exists(graphsDir):
    os.makedirs(graphsDir)

bin_strategies = ['Uniform', 'Quantile', 'Kmeans']
n_bins = [3, 5, 10]

values = {
    'with 3 bins': [18200, 20500, 20000],
    'with 5 bins': [18200, 12000, 18000],
    'with 10 bins': [12500, 5400, 10500],
}

plt.figure(figsize=(7,7))
ds.multiple_bar_chart(bin_strategies, values, ylabel='Number of Patterns')
plt.suptitle('Number of Patterns for a support of 0.01')
plt.savefig(graphsDir + 'HFCR Number of Patterns')

values = {
    'with 3 bins': [30, 59, 29],
    'with 5 bins': [45, 81, 40],
    'with 10 bins': [52, 50, 40],
}

plt.figure(figsize=(7,7))
ds.multiple_bar_chart(bin_strategies, values, ylabel='Lift Score')
plt.suptitle('Lift of the top 10% for a support of 0.01')
plt.savefig(graphsDir + 'HFCR Lift Score')

bin_strategies = ['0.25', '0.57']


import logging, math, torch
import torch.optim as optim
from torch import optim
from torch.optim.lr_scheduler import LambdaLR

logger = logging.getLogger(__name__)

class ConstantLRSchedule(LambdaLR):
    """ Constant learning rate schedule.
    """
    def __init__(self, optimizer, last_epoch=-1):
        super(ConstantLRSchedule, self).__init__(optimizer, lambda _: 1.0, last_epoch=last_epoch)

class WarmupConstantSchedule(LambdaLR):
    """ Linear warmup and then constant.
        Linearly increases learning rate schedule from 0 to 1 over `warmup_steps` training steps.
        Keeps learning rate schedule equal to 1. after warmup_steps.
    """
    def __init__(self, optimizer, warmup_steps, last_epoch=-1):
        self.warmup_steps = warmup_steps
        super(WarmupConstantSchedule, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch)

    def lr_lambda(self, step):
        if step < self.warmup_steps:
            return float(step) / float(max(1.0, self.warmup_steps))
        return 1.

class WarmupLinearSchedule(LambdaLR):
    """ Linear warmup and then linear decay.
        Linearly increases learning rate from 0 to 1 over `warmup_steps` training steps.
        Linearly decreases learning rate from 1. to 0. over remaining `t_total - warmup_steps` steps.
    """

    def __init__(self, optimizer, warmup_steps, t_total, last_epoch=-1):
        self.warmup_steps = warmup_steps
        self.t_total = t_total
        super(WarmupLinearSchedule, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch)

    def lr_lambda(self, step):
        if step < self.warmup_steps:
            return float(step) / float(max(1, self.warmup_steps))
        return max(0.0, float(self.t_total - step) / float(max(1.0, self.t_total - self.warmup_steps)))

class WarmupCosineSchedule(LambdaLR):
    """ Linear warmup and then cosine decay.
        Linearly increases learning rate from 0 to 1 over `warmup_steps` training steps.
        Decreases learning rate from 1. to 0. over remaining `t_total - warmup_steps` steps following a cosine curve.
        If `cycles` (default=0.5) is different from default, learning rate follows cosine function after warmup.
    """
    def __init__(self, optimizer, warmup_steps, t_total, cycles=.5, last_epoch=-1):
        self.warmup_steps = warmup_steps
        self.t_total = t_total
        self.cycles = cycles
        super(WarmupCosineSchedule, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch)

    def lr_lambda(self, step):
        if step < self.warmup_steps:
            return float(step) / float(max(1.0, self.warmup_steps))
        # progress after warmup
        progress = float(step - self.warmup_steps) / float(max(1, self.t_total - self.warmup_steps))
        return max(0.0, 0.5 * (1. + math.cos(math.pi * float(self.cycles) * 2.0 * progress)))

class WarmupCosineWithHardRestartsSchedule(LambdaLR):
    """ Linear warmup and then cosine cycles with hard restarts.
        Linearly increases learning rate from 0 to 1 over `warmup_steps` training steps.
        If `cycles` (default=1.) is different from default, learning rate follows `cycles` times a cosine decaying
        learning rate (with hard restarts).
    """

    def __init__(self, optimizer, warmup_steps, t_total, cycles=1., last_epoch=-1):
        self.warmup_steps = warmup_steps
        self.t_total = t_total
        self.cycles = cycles
        super(WarmupCosineWithHardRestartsSchedule, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch)

    def lr_lambda(self, step):
        if step < self.warmup_steps:
            return float(step) / float(max(1, self.warmup_steps))
        # progress after warmup
        progress = float(step - self.warmup_steps) / float(max(1, self.t_total - self.warmup_steps))
        if progress >= 1.0:
            return 0.0
        return max(0.0, 0.5 * (1. + math.cos(math.pi * ((float(self.cycles) * progress) % 1.0))))

def defineLRScheduler(args, optimizer, length_train_dataloader):
    warmup_steps = int(length_train_dataloader * 0.15)

    # 파이토치 기본제공 learning rate scheduler
    if args.lr_scheduler.lower() == 'steplr':
        scheduler = optim.plr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)
    elif args.lr_scheduler.lower() == 'multisteplr':
        '''
        - milestones : 해당 epoch마다 lr을 *gamma배만큼 decay
        - gamma : Multiplicative factor of learning rate decay. Default: 0.1.
        '''
        scheduler = optim.plr_scheduler.MultiStepLR(optimizer=optimizer, milestones=[2, 5, 10, 15, 30], gamma=0.5)
    elif args.lr_scheduler.lower() == 'reducelronplateau':
        scheduler = optim.plr_scheduler.ReduceLROnPlateau(optimizer=optimizer, mode='min', factor=0.5, patience=3, )
    elif args.lr_scheduler.lower() == 'cosineschedule':
        scheduler = optim.plr_scheduler.CosineAnnealingLR(optimizer=optimizer, T_max=10, eta_min=0)

    # 파이토치 기본제공 learning rate의 수정버전
    elif args.lr_scheduler.lower() == 'warmupcosineschedule':
        # warmup_steps동안 0~1까지 warmup 후 1~0까지 t_total-warmup_steps의 step동안 cosine을 그리며 decay
        scheduler = WarmupCosineSchedule(optimizer=optimizer, warmup_steps=warmup_steps, t_total=length_train_dataloader, cycles=0.5, last_epoch=-1)
        print('Warm-up during 0 to ', warmup_steps, 'steps, Cosine decay during ', warmup_steps, ' to ', length_train_dataloader, 'steps.')
    elif args.scheduler == 'WarmupCosineWithHardRestartsSchedule':
        scheduler = WarmupCosineWithHardRestartsSchedule(optimizer=optimizer, warmup_steps=warmup_steps, t_total=length_train_dataloader, cycles=1.0, last_epoch=-1)
    elif args.scheduler == 'WarmupLinearSchedule': # Constant learning rate scheduling.
        scheduler = ConstantLRSchedule(optimizer, last_epoch=-1)
    else:
        print("Invalid args input.")
        raise ValueError('Invalid scheduler')

    print('optimizer : ', type(optimizer), ' lr scheduler : ', type(scheduler))
    return scheduler


#!/usr/bin/python -tt

# To mount to drive E: in WSL/Ubuntu
# sudo mount -t drvfs E: /mnt/d

import sys
import os
import time
import re
import subprocess
import shutil

SPECIALDIR_CANONMSC = 'CANONMSC'
SPECIALFILE_SYNCME = 'SYNC.ME'

def main():

  # 1) Try to mount either Canon or Olympus SD card.
  # 2) Read contents of SYNC.ME
  # 3) Sync files (Olympus) or directories (Canon), except those named in the SYNC.ME file.
  # 4) Update contents of SYNC.ME file and unmount SD card.

  # Where to save photos to.
  trg = "/media/data/photos/"
  # Canon source paths.
  srcCanon = "/media/tv/CANON_DC/"
  syncFilePathCanon = srcCanon + SPECIALFILE_SYNCME
  # Olympus source paths.
  srcOlympus = "/media/tv/OLYMPUS/DCIM/100OLYMP/"
  syncFilePathOlympus = srcOlympus + SPECIALFILE_SYNCME

  # wait for udev rules to mount sd card
  mountTries = 30
  while (mountTries > 0) & (not os.path.exists(syncFilePathCanon)) & (not os.path.exists(syncFilePathOlympus)):
    time.sleep(2)
    mountTries -= 1

  if os.path.exists(syncFilePathCanon):
    srcImagePath = srcCanon + "DCIM/"
    doSyncCanon(srcImagePath, syncFilePathCanon, trg)
    umountArgs = ['umount', srcCanon]
    subprocess.call(umountArgs)
  elif os.path.exists(syncFilePathOlympus):
    doSyncOlympus(srcOlympus, trg)
    umountArgs = ['umount', srcOlympus]
    subprocess.call(umountArgs)
  else:
    sys.exit()

def doSyncCanon(srcImagePath, syncFilePathCanon, trg):
  copiedDirs = readSyncMeFile(syncFilePathCanon)

  # Include in sync the last folder named in SYNC.ME (could
  #    have changed since last sync)
  #    So remove last entry in copiedDirs
  if len(copiedDirs) > 0:
    copiedDirs.pop()
  # Also want to ignore the 'CANONMSC' directory
  copiedDirs.insert(0, SPECIALDIR_CANONMSC)

  # Also sync all folders on card that aren't named in SYNC.ME
  cardDirs = os.listdir(srcImagePath)
  toCopyDirs = [item for item in cardDirs if not item in copiedDirs]

  # Do copy
  for dir in toCopyDirs:
    fullSrcDir = srcImagePath + dir + "/"
    fullTrgDir = getFullTargetDirCanon(fullSrcDir, trg)
    copyDir(fullSrcDir, fullTrgDir)

  # Sort before writing to SYNC.ME, most recent last
  cardDirs.sort()

  # After sync remove lines in SYNC.ME if that folder
  # no longer exists to keep size of SYNC.ME file down
  syncFile = open(syncFilePathCanon, 'w')
  syncFile.truncate(0)
  for dir in cardDirs:
    if dir != SPECIALDIR_CANONMSC:
      syncFile.write(dir + '\n')
  syncFile.flush()
  syncFile.close()

def doSyncOlympus(srcImagePath, trg):
  syncFilePath = srcImagePath + SPECIALFILE_SYNCME

  copiedFiles = readSyncMeFile(syncFilePath)
  # Also want to ignore the 'SYNC.ME' file
  copiedFiles.insert(0, SPECIALFILE_SYNCME)

  # Sync all files on card that aren't named in SYNC.ME
  cardFiles = os.listdir(srcImagePath)
  toCopyFiles = [item for item in cardFiles if not item in copiedFiles and not item.endswith('.ORF') and not item.endswith('.pp3')]

  # Do copy
  for file in toCopyFiles:
    fullSrcPath = srcImagePath + file
    fullTrgPath = getFullTargetDirOlympus(fullSrcPath, trg)
    copyFile(fullSrcPath, fullTrgPath)

  # Sort before writing to SYNC.ME, most recent last
  cardFiles.sort()

  # After sync remove lines in SYNC.ME if that folder
  # no longer exists to keep size of SYNC.ME file down
  syncFile = open(syncFilePath, 'w')
  syncFile.truncate(0)
  for file in cardFiles:
    if file != SPECIALFILE_SYNCME:
      syncFile.write(file + '\n')
  syncFile.flush()
  syncFile.close()

def readSyncMeFile(syncFilePath):
  # 1) read contents of SYNC.ME
  if os.path.exists(syncFilePath):
    syncFile = open(syncFilePath, 'r')
    copiedDirs = syncFile.readlines()
    copiedDirs = [item.replace('\n','') for item in copiedDirs]
    syncFile.close()
  else:
    copiedDirs = []

  return copiedDirs

def copyDir(src, trg):
  if not os.path.exists(trg):
    os.makedirs(trg)
  # -arW = archive, recursive, Whole (not delta copy)
  rsyncArgs = ['rsync','-arW', src, trg]
  subprocess.call(rsyncArgs)

def copyFile(src, trg):
  trgDir = os.path.dirname(trg)
  if not os.path.exists(trgDir):
    os.makedirs(trgDir)
  shutil.copy2(src, trg)

# expands to /trg/2011/04/16/ based on file's modified date.
def getFullTargetDirOlympus(fullSrcFilePath, targetRoot):
  fileDateTime = time.localtime(os.path.getmtime(fullSrcFilePath))
  day = str(fileDateTime.tm_mday).zfill(2)
  month = str(fileDateTime.tm_mon).zfill(2)
  year = str(fileDateTime.tm_year)
  return targetRoot + year + "/" + month + "/" + day + "/"

# expands /src/139_1604/ to /trg/2011/04/16/
def getFullTargetDirCanon(fullSrcDir, targetRoot):
  date = getDate(fullSrcDir)
  day = date[0]
  month = date[1]
  year = time.localtime(os.path.getctime(fullSrcDir)).tm_year
  return targetRoot + str(year) + "/" + str(month) + "/" + str(day) + "/"

# gets day and month from src dir name
def getDate(src):
  # src is like '/some_path/139_1604', want [16, 04]
  m = re.search('([0-9]{2})([0-9]{2})', src[src.rfind('_')+1:])
  print (src)
  return [m.group(1), m.group(2)]

# Standard boilerplate to call the main() function.
if __name__ == '__main__':
  main()


import bpy, bmesh
from bpy import context as C
from mathutils import Vector

def newobj(bm, name):
    me = bpy.data.meshes.new(name)
    bm.to_mesh(me)
    ob = bpy.data.objects.new(name,me)
    C.scene.objects.link(ob)
    return ob

# https://blender.stackexchange.com/questions/32283/what-are-all-values-in-bound-box
def bounds(obj, local=False):
    local_coords = obj.bound_box[:]
    om = obj.matrix_world
    if not local:
        worldify = lambda p: om * Vector(p[:])
        coords = [worldify(p).to_tuple() for p in local_coords]
    else:
        coords = [p[:] for p in local_coords]
    rotated = zip(*coords[::-1])
    push_axis = []
    for (axis, _list) in zip('xyz', rotated):
        info = lambda: None
        info.max = max(_list)
        info.min = min(_list)
        info.distance = info.max - info.min
        push_axis.append(info)
    import collections
    originals = dict(zip(['x', 'y', 'z'], push_axis))
    o_details = collections.namedtuple('object_details', 'x y z')
    return o_details(**originals)

object_details = bounds(C.object, True)
bpy.ops.object.mode_set(mode='OBJECT')

cut_index = 0
step_size = 5

bisection_outer = bmesh.new()
bisection_outer.from_mesh(C.object.data)

# split the model into parts along y-axis, with steps of step_size in local coordinate space (this ignores the scale property, so make sure to apply it before)
for i in range(int(round(object_details.y.min)), int(round(object_details.y.max+step_size)), step_size):
        bisection_inner = bisection_outer.copy()
        bmesh.ops.bisect_plane(bisection_outer, geom=bisection_outer.verts[:]+bisection_outer.edges[:]+bisection_outer.faces[:], plane_co=(0,i,0), plane_no=(0,1,0), clear_inner=False)
        bmesh.ops.bisect_plane(bisection_inner, geom=bisection_inner.verts[:]+bisection_inner.edges[:]+bisection_inner.faces[:], plane_co=(0,i,0), plane_no=(0,1,0), clear_outer=True)
        newobj(bisection_inner, "bisect-"+str(cut_index))
        bisection_inner.free()  # free and prevent further access
        cut_index+=1

bisection_outer.free()  # free and prevent further access
bpy.data.objects.remove(C.object, True)


import numpy as np

from numba import jit

def fdconv(y1,y2,yfd1,yfd2):
    """Perform the convolution with the Fermi Dirac function"""
    n = len(y1) # number of points
    nc = (n-1)//2 # central site
    out = np.array([0.0 for i in range(n)])
    return fdconv_jit(y1,y2,yfd1,yfd2,out)

@jit(nopython=True)
def fdconv_jit(y1,y2,yfd1,yfd2,out):
    n = len(y1) # number of points
    nc = (n-1)//2 # central site
    for i in range(n):
      for j in range(-nc,2*n-nc):
          wp = j-i+nc # distance to zero frequency
          if wp<0: yi = y2[0] ; di = yfd2[0]
          elif wp>=n: yi = y2[n-1] ; di = yfd2[n-1]
          else: yi = y2[wp] ; di = yfd2[wp]
          if j>=n: yj = y1[n-1] ; dj = yfd1[n-1]
          elif j<0: yj = y1[0] ; dj = yfd1[0]
          else: yj = y1[j] ; dj = yfd1[j]
          out[i] = out[i] + yj*yi*(-dj+di)
    return out

@jit
def conv(y1,y2):
    """Perform the convolution with the Fermi Dirac function"""
    n = len(y1) # number of points
    nc = (n-1)//2 # central site
    out = [0.0 for i in range(n)]
    for i in range(n):
      for j in range(-nc,2*n-nc):
          wp = j-i+nc # distance to zero frequency
#          if not 0<=wp<n: continue
#          if not 0<=(j+wp)<n: continue
          if wp<0: yi = y2[0]
          elif wp>=n: yi = y2[n-1]
          else: yi = y2[wp]
          if j>=n: yj = y1[n-1]
          elif j<0: yj = y1[0]
          else: yj = y1[j]
          out[i] = out[i] + yj*yi
    return np.array(out)

def fdconvolution(x,y1,y2,fd1=None,fd2=None):
    """Fermi Dirac convolution"""
    if fd1 is None and fd2 is None: # no function given
        return np.convolve(y1,y2,mode="same")/len(y1)
        f1 = interpolate(x,y1) # interpolate
        f2 = interpolate(x,y2) # interpolate
        dx = max(x) - min(x)
        xs = np.linspace(min(x)-dx,max(x)+dx,len(x)*20) # as many points
        y3 = np.convolve(f1(xs),f2(xs),mode="same")
        return interpolate(xs,y3)(x)
    else: # assume that there is a Fermi Dirac distribution as input
        if callable(fd1): fd1x = fd1(x) # call function
        else: fd1x = fd1 # assume it is an array
        if callable(fd2): fd2x = fd2(x) # call function
        else: fd2x = fd2 # assume it is an array
        return fdconv(y1,y2,fd1x,fd2x)/len(x)

def plain_convolution(x,y1,y2):
    dx = (np.max(x) - np.min(x))/len(x)
    return np.convolve(y1,y2,mode="same")*dx


students = [
        {'name': 'Rezso', 'age': 9.5, 'candies': 2},
        {'name': 'Gerzson', 'age': 10, 'candies': 1},
        {'name': 'Aurel', 'age': 7, 'candies': 3},
        {'name': 'Zsombor', 'age': 12, 'candies': 5},
        {'name': 'Olaf', 'age': 12, 'candies': 7},
        {'name': 'Teodor', 'age': 3, 'candies': 2}
]

# create a function that counts the students that
# has more than 4 candies

# version no.1

def candies(kids):
    candy_kings = []
    for i in range(len(kids)):
        if kids[i]['candies'] > 4:
            candy_kings.append(kids[i]['name'])
    return candy_kings

print (candies(students))

# version no.2

def candies2(kids):
    candy_kings = 0
    for i in range(len(kids)):
        if kids[i]['candies'] > 4:
            candy_kings += 1
    return candy_kings

print (candies2(students))


from linked_list.linked_list import LinkedList, Node
def __contains__(self,value):
    """
    This method search in the linked list for a given value , and
        return True if the value exist and return False not exist
    """
    if self.head==None:
        return False
    else:
        x= self.head
        while x != None:
            if x.value==value:
                return True
            x=x.next
        return False
LinkedList.__contains__=__contains__

def append(self,value):
    """
    this method to add nodes in the end of the list
    """
    node=Node(value)
    currentNode=self.head
    if currentNode:
        while currentNode.next:
            currentNode=currentNode.next
        currentNode.next=node
    else:
        self.head=node
LinkedList.append=append

def insert_before(self,value, newValue):
    """
    adds a new node with the given new value immediately before the first node that has the value specified
    """
    if self.head is None:
        print("List has no element")
        return

    if value == self.head.value:
        new_node = Node(newValue)
        new_node.next = self.head
        self.head = new_node
        return

    n = self.head
    print(n.next)
    while n.next is not None:
        if n.next.value == value:
            break
        n = n.next
    if n.next is None:
        print("item not in the list")
    else:
        new_node = Node(newValue)
        new_node.next = n.next
        n.next = new_node

LinkedList.insert_before=insert_before
def insert_after(self,value,newValue):
    """
    adds a new node with the given new value immediately after the first node that has the value specified
    """
    if self.head is None:
        print("List has no element")
        return
    n = self.head
    print(n.next)
    while n is not None:
        if n.value == value:
            break
        n = n.next
    if n is None:
        print("item not in the list")
    else:
        new_node = Node(newValue)
        new_node.next = n.next
        n.next = new_node
LinkedList.insert_after=insert_after
if __name__ =="__main__":
    print("Abood")
    ll =LinkedList()

    ll2 =LinkedList()
    ll.append(1)
    ll.append(3)
    ll.append(5)
    ll.append(7)
    print(ll.__str__())
    ll2.insert(2)
    ll2.insert(4)
    ll2.insert(6)
    ll2.insert(8)
    ll2.insert(10)
    ll2.insert(12)
    ll.insert_after(3,7)
    print(ll.__str__())
    ll.insert_before(3,5)
    print(ll.__str__())


"""
Associative Compression Network based on https://arxiv.org/pdf/1804.02476v2.pdf

Strongly referenced ACN implementation and blog post from:
http://jalexvig.github.io/blog/associative-compression-networks/

Base VAE referenced from pytorch examples:
https://github.com/pytorch/examples/blob/master/vae/main.py
"""

# TODO conv
# TODO load function
# daydream function
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import os
import sys
import time
import numpy as np
from copy import deepcopy, copy
from sklearn.neighbors import KNeighborsClassifier
from torch import nn, optim
import torch
from torch.nn import functional as F
from torchvision import datasets, transforms
from torch.utils.data import Dataset, DataLoader
from torchvision.utils import save_image
from IPython import embed
from pixel_cnn import GatedPixelCNN

class ConvEncoder(nn.Module):
    def __init__(self, code_len, input_size=1, encoder_output_size=1000):
        super(ConvEncoder, self).__init__()
        self.code_len = code_len
        self.encoder = nn.Sequential(
            nn.Conv2d(in_channels=input_size,
                      out_channels=16,
                      kernel_size=4,
                      stride=2, padding=1),
            nn.BatchNorm2d(16),
            nn.ReLU(True),
            nn.Conv2d(in_channels=16,
                      out_channels=16,
                      kernel_size=4,
                      stride=2, padding=1),
            nn.BatchNorm2d(16),
            nn.ReLU(True),
            nn.Conv2d(in_channels=16,
                      out_channels=16,
                      kernel_size=2,
                      stride=2, padding=1),
            nn.BatchNorm2d(16),
            nn.ReLU(True),
            nn.Conv2d(in_channels=16,
                      out_channels=code_len*2,
                      kernel_size=1,
                      stride=1, padding=0),
            nn.BatchNorm2d(code_len*2),
            nn.ReLU(True),
           )
        # found via experimentation - 3 for mnist
        # input_image == 28 -> eo=7
        self.eo = eo = encoder_output_size
        self.fc21 = nn.Linear(eo, code_len)
        self.fc22 = nn.Linear(eo, code_len)

    def encode(self, x):
        o = self.encoder(x)
        ol = o.view(o.shape[0], o.shape[1]*o.shape[2]*o.shape[3])
        return self.fc21(ol), self.fc22(ol)

    def reparameterize(self, mu, logvar):
        if self.training:
            std = torch.exp(0.5*logvar)
            eps = torch.randn_like(std)
            o = eps.mul(std).add_(mu)
            return o
        else:
            return mu

    def forward(self, x):
        mu, logvar = self.encode(x)
        z = self.reparameterize(mu, logvar)
        return z, mu, logvar

class PriorNetwork(nn.Module):
    def __init__(self, size_training_set, code_length, n_hidden=512, k=5, random_seed=4543):
        super(PriorNetwork, self).__init__()
        self.rdn = np.random.RandomState(random_seed)
        self.k = k
        self.size_training_set = size_training_set
        self.code_length = code_length
        self.fc1 = nn.Linear(self.code_length, n_hidden)
        self.fc2_u = nn.Linear(n_hidden, self.code_length)
        self.fc2_s = nn.Linear(n_hidden, self.code_length)

        self.knn = KNeighborsClassifier(n_neighbors=self.k, n_jobs=-1)
        # codes are initialized randomly - Alg 1: initialize C: c(x)~N(0,1)
        codes = self.rdn.standard_normal((self.size_training_set, self.code_length))
        self.fit_knn(codes)

    def fit_knn(self, codes):
        ''' will reset the knn  given an nd array
        '''
        self.codes = codes
        y = np.zeros((len(self.codes)))
        self.knn.fit(self.codes, y)

    def batch_pick_close_neighbor(self, codes):
        '''
        :code latent activation of training example as np
        '''
        neighbor_distances, neighbor_indexes = self.knn.kneighbors(codes, n_neighbors=self.k, return_distance=True)
        bsize = neighbor_indexes.shape[0]
        if self.training:
            # randomly choose neighbor index from top k
            chosen_neighbor_index = self.rdn.randint(0,neighbor_indexes.shape[1],size=bsize)
        else:
            chosen_neighbor_index = np.zeros((bsize), dtype=np.int)
        return self.codes[neighbor_indexes[np.arange(bsize), chosen_neighbor_index]]

    def forward(self, codes):
        device = codes.device
        np_codes = codes.cpu().detach().numpy()
        previous_codes = self.batch_pick_close_neighbor(np_codes)
        previous_codes = torch.FloatTensor(previous_codes).to(device)
        return self.encode(previous_codes)

    def encode(self, prev_code):
        h1 = F.relu(self.fc1(prev_code))
        mu = self.fc2_u(h1)
        logstd = self.fc2_s(h1)
        return mu, logstd

def set_model_mode(model_dict, phase):
    for name, model in model_dict.items():
        print('setting', name, phase)
        if name != 'opt':
            if phase == 'valid':
                model_dict[name].eval()
            else:
                model_dict[name].train()
    return model_dict

def run_acn(train_cnt, model_dict, data_dict, phase, device):
    st = time.time()
    running_loss = 0.0
    data_loader = data_dict[phase]
    model_dict = set_model_mode(model_dict, phase)
    for batch_idx, (data, label, data_index) in enumerate(data_loader):
        target = data = data.to(device)
        bs = data.shape[0]
        model_dict['opt'].zero_grad()
        z, u_q, s_q = model_dict['encoder_model'](data)
        # add the predicted codes to the input
        yhat_batch = torch.sigmoid(model_dict['pcnn_decoder'](x=data, float_condition=z))
        model_dict['prior_model'].codes[data_index] = u_q.detach().cpu().numpy()
        model_dict['prior_model'].fit_knn(model_dict['prior_model'].codes)
        u_p, s_p = model_dict['prior_model'](u_q)
        kl = kl_loss_function(u_q, s_q, u_p, s_p)
        rec_loss = F.binary_cross_entropy(yhat_batch, target, reduction='mean')
        loss = kl+rec_loss
        if phase == 'train':
            loss.backward()
            model_dict['opt'].step()
        running_loss+= loss.item()
        # add batch size because it hasn't been added to train cnt yet
        if phase == 'train':
            train_cnt+=bs
    example = {'data':data, 'target':target, 'yhat':yhat_batch}
    avg_loss = running_loss/bs
    print("finished %s after %s secs at cnt %s loss %s"%(phase, time.time()-st, train_cnt, avg_loss))
    return model_dict, data_dict, avg_loss, example

def train_acn(train_cnt, epoch_cnt, model_dict, data_dict, info):
    base_filepath = info['base_filepath']
    base_filename = os.path.split(info['base_filepath'])[1]
    while train_cnt < info['num_examples_to_train']:
        print('starting epoch %s on %s'%(epoch_cnt, info['device']))
        model_dict, data_dict, avg_train_loss, train_example = run_acn(train_cnt, model_dict, data_dict, phase='train', device=info['device'])
        epoch_cnt +=1
        train_cnt +=info['size_training_set']
        if not epoch_cnt % info['save_every_epochs']:
            # make a checkpoint
            print('starting valid phase')
            model_dict, data_dict, avg_valid_loss, valid_example = run_acn(train_cnt, model_dict, data_dict, phase='valid', device=info['device'])
            state_dict = {}
            for key, model in model_dict.items():
                state_dict[key+'_state_dict'] = model.state_dict()
            info['train_losses'].append(avg_train_loss)
            info['train_cnts'].append(train_cnt)
            info['valid_losses'].append(avg_valid_loss)
            info['epoch_cnt'] = epoch_cnt
            state_dict['info'] = info

            ckpt_filepath = os.path.join(base_filepath, "%ss_%010dex.pt"%(base_filename, train_cnt))
            train_img_filepath = os.path.join(base_filepath,"%s_%010d_train_rec.png"%(base_filename, train_cnt))
            valid_img_filepath = os.path.join(base_filepath, "%s_%010d_valid_rec.png"%(base_filename, train_cnt))
            plot_filepath = os.path.join(base_filepath, "%s_%010dloss.png"%(base_filename, train_cnt))

            plot_example(train_img_filepath, train_example, num_plot=5)
            plot_example(valid_img_filepath, valid_example, num_plot=5)
            save_checkpoint(state_dict, filename=ckpt_filepath)

            plot_losses(info['train_cnts'],
                        info['train_losses'],
                        info['train_cnts'],
                        info['valid_losses'], name=plot_filepath, rolling_length=1)

def call_tsne_plot(model_dict, data_dict, info):
    from make_tsne_plot import tsne_plot
    # always be in eval mode
    model_dict = set_model_mode(model_dict, 'valid')
    with torch.no_grad():
        for phase in ['valid', 'train']:
            data_loader = data_dict[phase]
            with torch.no_grad():
                for batch_idx, (data, label, data_index) in enumerate(data_loader):
                    target = data = data.to(info['device'])
                    # yhat_batch is bt 0-1
                    z, u_q, s_q = model_dict['encoder_model'](data)
                    u_p, s_p = model_dict['prior_model'](u_q)
                    yhat_batch = torch.sigmoid(model_dict['pcnn_decoder'](x=target, float_condition=z))
                    html_path = info['model_loadpath'].replace('.pt', '.html')
                    X = u_q.cpu().numpy()
                    images = np.round(yhat_batch.cpu().numpy()[:,0], 0).astype(np.int32)
                    #images = target[:,0].cpu().numpy()
                    tsne_plot(X=X, images=images, color=batch_idx, perplexity=info['perplexity'],
                              html_out_path=html_path)

def sample(model_dict, data_dict, info):
    from skvideo.io import vwrite
    model_dict = set_model_mode(model_dict, 'valid')
    output_savepath = args.model_loadpath.replace('.pt', '')
    with torch.no_grad():
        for phase in ['train', 'valid']:
            data_loader = data_dict[phase]
            with torch.no_grad():
                for batch_idx, (data, label, data_index) in enumerate(data_loader):
                    target = data = data.to(info['device'])
                    bs = data.shape[0]
                    z, u_q, s_q = model_dict['encoder_model'](data)
                    # teacher forced version
                    yhat_batch = torch.sigmoid(model_dict['pcnn_decoder'](x=target, float_condition=z))
                    # create blank canvas for autoregressive sampling
                    canvas = torch.zeros_like(target)
                    building_canvas = []
                    for i in range(canvas.shape[1]):
                        for j in range(canvas.shape[2]):
                            print('sampling row: %s'%j)
                            for k in range(canvas.shape[3]):
                                output = torch.sigmoid(model_dict['pcnn_decoder'](x=canvas, float_condition=z))
                                canvas[:,i,j,k] = output[:,i,j,k].detach()
                                # add frames for video
                                if not k%5:
                                    building_canvas.append(deepcopy(canvas[0].detach().cpu().numpy()))

                    f,ax = plt.subplots(bs, 3, sharex=True, sharey=True, figsize=(3,bs))
                    nptarget = target.detach().cpu().numpy()
                    npoutput = output.detach().cpu().numpy()
                    npyhat = yhat_batch.detach().cpu().numpy()
                    for idx in range(bs):
                        ax[idx,0].imshow(nptarget[idx,0], cmap=plt.cm.viridis)
                        ax[idx,0].set_title('true')
                        ax[idx,1].imshow(npyhat[idx,0], cmap=plt.cm.viridis)
                        ax[idx,1].set_title('tf')
                        ax[idx,2].imshow(npoutput[idx,0], cmap=plt.cm.viridis)
                        ax[idx,2].set_title('sam')
                        ax[idx,0].axis('off')
                        ax[idx,1].axis('off')
                        ax[idx,2].axis('off')
                    iname = output_savepath + '_sample_%s.png'%phase
                    print('plotting %s'%iname)
                    plt.savefig(iname)
                    plt.close()

                    # make movie
                    building_canvas = (np.array(building_canvas)*255).astype(np.uint8)
                    print('writing building movie')
                    mname = output_savepath + '_build_%s.mp4'%phase
                    vwrite(mname, building_canvas)
                    print('finished %s'%mname)
                    # only do one batch
                    break

class IndexedDataset(Dataset):
    def __init__(self, dataset_function, path, train=True, download=True, transform=transforms.ToTensor()):
        """ class to provide indexes into the data
        """
        self.indexed_dataset = dataset_function(path,
                             download=download,
                             train=train,
                             transform=transform)

    def __getitem__(self, index):
        data, target = self.indexed_dataset[index]
        return data, target, index

    def __len__(self):
        return len(self.indexed_dataset)

def save_checkpoint(state, filename='model.pt'):
    print("starting save of model %s" %filename)
    torch.save(state, filename)
    print("finished save of model %s" %filename)

def create_mnist_datasets(dataset_name, base_datadir, batch_size):
    dataset = eval('datasets.'+dataset_name)
    datadir = os.path.join(base_datadir, dataset_name)
    train_data = IndexedDataset(dataset, path=datadir,
                                train=True, download=True,
                                transform=transforms.ToTensor())
    train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True)
    valid_data = IndexedDataset(dataset, path=datadir,
                               train=False, download=True,
                               transform=transforms.ToTensor())
    valid_loader = DataLoader(valid_data, batch_size=batch_size, shuffle=True)
    data_dict = {'train':train_loader, 'valid':valid_loader}
    nchans,hsize,wsize = data_dict['train'].dataset[0][0].shape
    size_training_set = len(train_data)
    return data_dict, size_training_set, nchans, hsize, wsize

def create_new_info_dict(arg_dict, size_training_set, base_filepath):
    info = {'train_cnts':[],
            'train_losses':[],
            'valid_losses':[],
            'save_times':[],
            'args':[args],
            'last_save':0,
            'last_plot':0,
            'epoch_cnt':0,
            'size_training_set':size_training_set,
            'base_filepath':base_filepath,
             }
    for arg,val in arg_dict.items():
        info[arg] = val
    if info['cuda']:
        info['device'] = 'cuda'
    else:
        info['device'] = 'cpu'
    return info

def create_conv_acn_pcnn_models(info, model_loadpath=''):
    '''
    load details of previous model if applicable, otherwise create new models
    '''
    train_cnt = 0
    epoch_cnt = 0

    # use argparse device no matter what info dict is loaded
    preserve_args = ['device', 'batch_size', 'save_every_epochs', 'base_filepath']
    preserve_dict = {}
    for key in preserve_args:
        preserve_dict[key] = info[key]

    if args.model_loadpath !='':
        tmlp =  model_loadpath+'.tmp'
        os.system('cp %s %s'%(args.model_loadpath, tmlp))
        _dict = torch.load(tmlp, map_location=lambda storage, loc:storage)
        info = _dict['info']
        train_cnt = info['train_cnts'][-1]
        epoch_cnt = info['epoch_cnt']

    # use argparse device no matter what device is loaded
    for key in preserve_args:
        info[key] = preserve_dict[key]

    encoder_model = ConvEncoder(info['code_length'], input_size=info['input_channels'],
                            encoder_output_size=info['encoder_output_size']).to(info['device'])
    prior_model = PriorNetwork(size_training_set=info['size_training_set'],
                               code_length=info['code_length'], k=info['num_k']).to(info['device'])
    pcnn_decoder = GatedPixelCNN(input_dim=info['target_channels'],
                                  dim=info['possible_values'],
                                  n_layers=info['num_pcnn_layers'],
                                  n_classes=info['num_classes'],
                                  float_condition_size=info['code_length'],
                                  last_layer_bias=info['last_layer_bias']).to(info['device'])

    model_dict = {'encoder_model':encoder_model, 'prior_model':prior_model, 'pcnn_decoder':pcnn_decoder}
    parameters = []
    for name,model in model_dict.items():
        parameters+=list(model.parameters())
    model_dict['opt'] = optim.Adam(parameters, lr=info['learning_rate'])

    if args.model_loadpath !='':
       for name,model in model_dict.items():
            model_dict[name].load_state_dict(_dict[name+'_state_dict'])
    return model_dict, info, train_cnt, epoch_cnt

def seed_everything(seed=394, max_threads=2):
    torch.manual_seed(394)
    torch.set_num_threads(max_threads)

def plot_example(img_filepath, example, plot_on=['target', 'yhat'], num_plot=10):
    '''
    img_filepath: location to write .png file
    example: dict with torch images of the same shape [bs,c,h,w] to write
    plot_on: list of keys of images in example dict to write
    num_plot: limit the number of examples from bs to this int
    '''
    for cnt, pon in enumerate(plot_on):
        bs,c,h,w = example[pon].shape
        num_plot = min([bs, num_plot])
        eimgs = example[pon].view(bs,c,h,w)[:num_plot]
        if not cnt:
            comparison = eimgs
        else:
            comparison = torch.cat([comparison, eimgs])
    save_image(comparison.cpu(), img_filepath, nrow=num_plot)
    print('writing comparison image: %s img_path'%img_filepath)

def rolling_average(a, n=5) :
    if n == 0:
        return a
    ret = np.cumsum(a, dtype=float)
    ret[n:] = ret[n:] - ret[:-n]
    return ret[n - 1:] / n

def plot_losses(train_cnts, train_losses, test_cnts, test_losses, name='loss_example.png', rolling_length=4):
    f,ax=plt.subplots(1,1,figsize=(3,3))
    ax.plot(rolling_average(train_cnts, rolling_length), rolling_average(train_losses, rolling_length), label='train loss', lw=1, c='orangered')
    ax.plot(rolling_average(test_cnts, rolling_length),  rolling_average(test_losses, rolling_length), label='test loss', lw=1, c='cornflowerblue')
    ax.scatter(rolling_average(test_cnts, rolling_length), rolling_average(test_losses, rolling_length), s=4, c='cornflowerblue')
    ax.scatter(rolling_average(train_cnts, rolling_length),rolling_average(train_losses, rolling_length), s=4, c='orangered')
    ax.legend()
    plt.savefig(name)
    plt.close()

def kl_loss_function(u_q, s_q, u_p, s_p):
    ''' reconstruction loss + coding cost
     coding cost is the KL divergence bt posterior and conditional prior
     Args:
         u_q: mean of model posterior
         s_q: log std of model posterior
         u_p: mean of conditional prior
         s_p: log std of conditional prior

     Returns: loss
     '''
    acn_KLD = torch.sum(s_p-s_q-0.5 + ((2*s_q).exp() + (u_q-u_p).pow(2)) / (2*(2*s_p).exp()))
    return acn_KLD

if __name__ == '__main__':
    from argparse import ArgumentParser
    parser = ArgumentParser(description='train acn')
    # operatation options
    parser.add_argument('-l', '--model_loadpath', default='', help='load model to resume training or sample')
    parser.add_argument('-c', '--cuda', action='store_true', default=False)
    parser.add_argument('--seed', default=394)
    parser.add_argument('--num_threads', default=2)
    parser.add_argument('-se', '--save_every_epochs', default=5, type=int)
    parser.add_argument('-bs', '--batch_size', default=128, type=int)
    parser.add_argument('-lr', '--learning_rate', default=1e-4)
    parser.add_argument('--input_channels', default=1, type=int, help='num of channels of input')
    parser.add_argument('--target_channels', default=1, type=int, help='num of channels of target')
    parser.add_argument('--num_examples_to_train', default=50000000, type=int)
    # acn model setup
    parser.add_argument('-cl', '--code_length', default=64, type=int)
    parser.add_argument('-k', '--num_k', default=5, type=int)
    parser.add_argument('--possible_values', default=1, help='num values that the pcnn output can take')
    parser.add_argument('--last_layer_bias', default=0.5, help='bias for output decoder')
    parser.add_argument('--num_classes', default=10, help='num classes for class condition in pixel cnn')
    parser.add_argument('--encoder_output_size', default=2048, help='output as a result of the flatten of the encoder - found experimentally')
    parser.add_argument('--num_pcnn_layers', default=12, help='num layers for pixel cnn')
    # dataset setup
    parser.add_argument('-d',  '--dataset_name', default='MNIST', help='which mnist to use', choices=['MNIST', 'FashionMNIST'])
    parser.add_argument('--exp_name', default='pcnn_acn', help='name of experiment')
    parser.add_argument('--model_savedir', default='../../model_savedir', help='save checkpoints here')
    parser.add_argument('--base_datadir', default='../../dataset/', help='save datasets here')
    # sampling info
    parser.add_argument('-s', '--sample', action='store_true', default=False)
    parser.add_argument('-tf', '--teacher_force', action='store_true', default=False)
    # tsne info
    parser.add_argument('-t', '--tsne', action='store_true', default=False)
    parser.add_argument('-p', '--perplexity', default=3)
    args = parser.parse_args()

    seed_everything(args.seed, args.num_threads)
    args.exp_name += '_'+args.dataset_name
    base_filepath = os.path.join(args.model_savedir, args.exp_name)
    if not os.path.exists(base_filepath):
        os.mkdir(base_filepath)

    data_dict, size_training_set, nchans, hsize, wsize = create_mnist_datasets(dataset_name=args.dataset_name, base_datadir=args.base_datadir, batch_size=args.batch_size)
    info = create_new_info_dict(vars(args), size_training_set, base_filepath)
    model_dict, info, train_cnt, epoch_cnt = create_conv_acn_pcnn_models(info, args.model_loadpath)

    if args.sample:
        sample(model_dict, data_dict, info)
    elif args.tsne:
        call_tsne_plot(model_dict, data_dict, info)
    else:
        train_acn(train_cnt, epoch_cnt, model_dict, data_dict, info)


# -*- coding: utf-8 -*-
"""
Created on Sun Nov 25 13:38:13 2018

@author: Despacito
"""

# Importing the libraries
import pandas as pd
import matplotlib.pyplot as plt

# Convert string to date format
from pandas import datetime
def parser(x):
    return datetime.strptime(x, '%Y-%m')

# Importing the dataset
sales = pd.read_csv('sales-cars.csv',index_col = 0, parse_dates = [0], date_parser = parser)

# Plotting the data
sales.plot()

# checking for stationarity uing Auto correlation function(ACF)
from statsmodels.graphics.tsaplots import plot_acf
plot_acf(sales)

# Integrated of order 1, to make data stationary. This is ARIMA model.
sales_diff = sales.diff(periods = 1)[1:]
plot_acf(sales_diff)
sales_diff.plot()

# Splitting the data into training and test set
X = sales.values
train_dataset = X[0:27]
test_dataset = X[26:]

predictions = []

# Creating the AutoRegressive Model (AR)
from statsmodels.tsa.ar_model import AR
from sklearn.metrics import mean_squared_error
model_ar = AR(train_dataset)
model_ar_fit = model_ar.fit()

# Predicting the result using AR model
predictions = model_ar_fit.predict(start = 27, end = 36)

# PLotting the test dataset and predictions of AR model
plt.plot(test_dataset)
plt.plot(predictions, color = 'red')

# Creating the Arima Model(p, d, q)
# p = periods taken for autoregressive model
# d = order of integration, how many time differencing is done
# q = number of periods in moving average model
from statsmodels.tsa.arima_model import ARIMA
model_arima = ARIMA(train_dataset, order = (4, 1, 0))
model_arima_fit = model_arima.fit()
print(model_arima_fit.aic) # 295.9979

# Prediction the result using ARIMA model
predictions = model_arima_fit.forecast(steps = 10)[0]

# PLotting the test dataset and predictions of ARIMA model
plt.plot(test_dataset)
plt.plot(predictions, color = 'red')

# Caluclate minimum AIC for different p, d, q
import itertools
p = d = q = range(0, 5)
pdq = list(itertools.product(p, d, q))
lesser_aic = 100000
pdq_for_lesser_aic = []

import warnings
warnings.filterwarnings('ignore')
for param in pdq:
    try: # because some combinations of p, d, q are not compatible, throwing exception
        model_arima = ARIMA(train_dataset, order = param)
        model_arima_fit = model_arima.fit()
        current_aic = model_arima_fit.aic
        if(current_aic < lesser_aic):
            lesser_aic = current_aic
            pdq_for_lesser_aic = param

    except:
        continue

# Now we can retrain the ARIMA model for that combination of p,d,q which has calculated less AIC above
from statsmodels.tsa.arima_model import ARIMA
model_arima = ARIMA(train_dataset, order = pdq_for_lesser_aic)
model_arima_fit = model_arima.fit()
print(model_arima_fit.aic) # 295.9979

# Prediction the result using ARIMA model
predictions = model_arima_fit.forecast(steps = 10)[0]

# PLotting the test dataset and predictions of ARIMA model
plt.plot(test_dataset)
plt.plot(predictions, color = 'red')

Part 2 - Regression/Section 10 - Time Series Forecasting/Programs/ARIMA.py


# n20056번 마법사 상어와 파이어볼 문제
# 문제링크 : https://www.acmicpc.net/problem/20056
from copy import deepcopy

N, M, K = map(int, input().split())
M_list = [list(map(int, input().split())) for _ in range(M)]

# M_list[0...4] > r,c,m,s,d #x좌표,y좌표,질량,속력,방향
Map = [[[] for _ in range(N)] for _ in range(N)]  # ***

# 0, 1, 2, 3, 4, ... 순으로
dx = [-1, -1, 0, 1, 1, 1, 0, -1]
dy = [0, 1, 1, 1, 0, -1, -1, -1]

for i in range(M):
    M_list[i][0] -= 1  # x좌표 -1 감소
    M_list[i][1] -= 1  # y좌표 -1 감소
    Map[M_list[i][0]][M_list[i][1]].append([M_list[i][2], M_list[i][3], M_list[i][4]])  # Map리스트에 파이어볼 리스트 담아두기

'''
Map1[0] -> [[0], [0], [0], [0]]
Map1[0][0] -> [0]
Map1[0][0][1] -> 2
'''

for _ in range(K):  # K는 이동횟수
    temp = [[[] for _ in range(N)] for _ in range(N)]  # ***

    # 1. 모든 파이어볼 이동
    for i in range(N):
        for j in range(N):
            if Map[i][j] != []:  # M은 파이어볼 개수
                for x in range(len(Map[i][j])):
                    m, s, d = Map[i][j][x]
                    nx = i + dx[d] * s
                    ny = j + dy[d] * s
                    nx = (nx + N) % N
                    ny = (ny + N) % N
                    temp[nx][ny].append([m, s, d])
    # 2. 2개 이상의 파이어들이 있는 칸을 찾아가서 4개의 파이어들로 다시 만드는 과정
    for x in range(N):
        for y in range(N):
            if len(temp[x][y]) > 1:
                list_cnt = len(temp[x][y])
                n_m, n_s, d = 0, 0, []
                for z in range(list_cnt):
                    n_m += temp[x][y][z][0]
                    n_s += temp[x][y][z][1]
                    d.append(temp[x][y][z][2] % 2)
                n_m = int(n_m // 5)
                n_s = int(n_s / list_cnt)
                temp[x][y] = []
                if n_m != 0:
                    if sum(d) == 0 or sum(d) == list_cnt:  # 모두 홀수이거나 또는 짝수인 경우
                        for i in range(4):
                            temp[x][y].append([n_m, n_s, i * 2])
                    else:
                        for i in range(4):
                            temp[x][y].append([n_m, n_s, i * 2 + 1])
    Map = deepcopy(temp)

answer = 0
for i in range(N):
    for j in range(N):
        if Map[i][j]:  # 내용물이 있을 경우를 찾는다.
            len_num = len(Map[i][j])
            for n in range(len_num):
                answer += Map[i][j][n][0]
print(answer)