"""This file implements Deep CFR, as described in Brown et al. - Deep Counterfactual Regret Minimization
(2019).
"""

import collections
import os
import time
import typing

import numpy as np
import tensorflow as tf
from tqdm import tqdm

import rlpoker.cfr_util
from rlpoker import cfr
from rlpoker.cfr_game import (
    get_available_actions, sample_chance_action, is_terminal, payoffs, which_player)
from rlpoker.util import sample_action
from rlpoker import buffer
from rlpoker import extensive_game
from rlpoker import best_response
from rlpoker import neural_game
from rlpoker import util

StrategyMemoryElement = collections.namedtuple('StrategyMemoryElement', [
    'info_set_id', 't', 'info_set_strategy'
])

AdvantageMemoryElement = collections.namedtuple('AdvantageMemoryElement', [
    'info_set_id', 't', 'info_set_advantages'
])

class RegretPredictor:

    """
    A RegretPredictor can be used inside cfr_traverse.
    """

    def predict_advantages(self, info_set_vector, action_indexer: neural_game.ActionIndexer) -> \
            extensive_game.ActionFloat:
        """
        Predicts advantages for each action available in the information set.

        Args:
            info_set_vector: ndarray.
            action_indexer: ActionIndexer. The mapping from actions to indices.

        Returns:
            ActionFloat. The predicted regret for each action in the information set.
        """
        raise NotImplementedError("Not implemented in the base class.")

    def compute_action_probs(self, info_set_vector, action_indexer: neural_game.ActionIndexer):
        """
        Compute action probabilities in this information set.

        Args:
            info_set_vector: ndarray.
            action_indexer: ActionIndexer.

        Returns:
            ActionFloat. The action probabilities in this information set.
        """
        action_advantages = self.predict_advantages(info_set_vector, action_indexer)
        return rlpoker.cfr_util.compute_regret_matching(action_advantages, highest_regret=True)

    def train(self, batch: typing.List[AdvantageMemoryElement],
              action_indexer: neural_game.ActionIndexer,
              info_set_vectoriser: neural_game.InfoSetVectoriser,
              current_time: int):
        """Train on one batch of AdvantageMemoryElements.

        Args:
            batch: list of AdvantageMemoryElement objects.
            action_indexer: ActionIndexer. Turns actions into indices.
            info_set_vectoriser: InfoSetVectoriser. Turns info set ids into vectors.
            current_time: int. The current iteration we are training on.

        Returns:
            loss: float.
        """
        pass

class DeepRegretNetwork(RegretPredictor):

    def __init__(self, state_shape: typing.Tuple[int], action_indexer: neural_game.ActionIndexer, player: int):
        """
        A DeepRegretNetwork uses a neural network to predict advantages for actions in information sets.

        Args:
            state_dim: int. The dimension of the state vector.
            action_indexer: ActionIndexer.
            player: int. The player number it represents. Used for scoping.
        """
        self.state_shape = state_shape
        self.action_indexer = action_indexer
        self.player = player

        self.sess = None

        self.scope = 'regret_network_{}'.format(self.player)
        self.tensors, self.init_op = self.build(self.state_shape, self.action_indexer.action_dim, self.scope)

        self.global_step = 0

    def initialise(self):
        """
        Initialise the weights of the network, using the tensorflow session self.sess.
        """
        self.sess.run(self.init_op)

    @staticmethod
    def build(state_shape, action_dim, scope, hidden_dim=64):
        with tf.variable_scope(scope):
            input_layer = tf.placeholder(tf.float32, shape=(None,) + state_shape, name='state')

            # For now, we flatten so that we can accept any state shape.
            hidden = tf.layers.flatten(input_layer, name='flatten')
            hidden = tf.layers.dense(hidden, hidden_dim, activation=tf.nn.relu)
            hidden = tf.layers.dense(hidden, hidden_dim, activation=tf.nn.relu)

            advantages = tf.layers.dense(hidden, action_dim)

            info_set_advantages = tf.placeholder(tf.float32, shape=(None, action_dim), name='info_set_advantages')
            times = tf.placeholder(tf.float32, shape=(None, 1), name='times')
            current_time = tf.placeholder(tf.float32, shape=(), name='current_time')

            regrets = tf.reduce_sum((info_set_advantages - advantages)**2, axis=1, name='regrets')

            loss = tf.reduce_mean(times * regrets) / current_time

            summary = tf.summary.scalar('loss', loss)

            train_op = tf.train.AdamOptimizer(learning_rate=1e-3).minimize(loss)

        tensors = {
            'input_layer': input_layer,
            'advantages': advantages,
            'train_op': train_op,
            'loss': loss,
            'times': times,
            'current_time': current_time,
            'info_set_advantages': info_set_advantages,
            'summary': summary
        }

        init_op = tf.variables_initializer(tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=scope))

        return tensors, init_op

    def set_sess(self, sess: tf.Session):
        self.sess = sess

    def predict_advantages(self, info_set_vector, action_indexer: neural_game.ActionIndexer) -> \
            extensive_game.ActionFloat:
        advantages = self.sess.run(self.tensors['advantages'], feed_dict={
            self.tensors['input_layer']: [info_set_vector]
        })

        return extensive_game.ActionFloat({
            action: advantages[0, self.action_indexer.get_index(action)] for action in self.action_indexer.actions
        })

    def train(self, batch: typing.List[AdvantageMemoryElement],
              action_indexer: neural_game.ActionIndexer,
              info_set_vectoriser: neural_game.InfoSetVectoriser,
              current_time: int):
        """Train on one batch of AdvantageMemoryElements.

        Args:
            batch: list of AdvantageMemoryElement objects.
            action_indexer: ActionIndexer. Turns actions into indices.
            info_set_vectoriser: InfoSetVectoriser. Turns info set ids into vectors.
            current_time: int. The current iteration we are training on.

        Returns:
            loss: float.
            summary: tensorflow summary.
        """
        # Each batch is an AdvantageMemoryElement.
        info_set_vectors = [info_set_vectoriser.get_vector(element.info_set_id) for element in batch]
        times = [element.t for element in batch]
        info_set_advantages = [info_set_advantages_to_vector(action_indexer, element.info_set_advantages)
                               for element in batch]

        _, computed_loss, summary = self.sess.run(
            [self.tensors['train_op'], self.tensors['loss'], self.tensors['summary']],
            feed_dict={
                self.tensors['input_layer']: np.array(info_set_vectors),
                self.tensors['times']: np.array(times).reshape(-1, 1),
                self.tensors['current_time']: current_time,
                self.tensors['info_set_advantages']: np.array(info_set_advantages).reshape(-1, action_indexer.action_dim)
        })

        self.global_step += 1

        return computed_loss, summary

def info_set_advantages_to_vector(action_indexer: neural_game.ActionIndexer,
                                  info_set_advantages: typing.Dict[typing.Any, float]):
    """

    Args:
        action_indexer:
        info_set_advantages: dict mapping actions to advantages.

    Returns:
        vector with the advantage for each action in the correct index.
    """
    advantages = np.zeros(action_indexer.action_dim)
    for action, advantage in info_set_advantages.items():
        advantages[action_indexer.get_index(action)] = advantage

    return advantages

def early_stopping(losses: typing.List[float], consecutive_increases: int=2):
    """Returns True if and only if losses[-consecutive_increases-1:] is monotonically increasing.

    Args:
        losses: list of floats. The losses.
        consecutive_increases: int. The number of consecutive increases to see before early stopping.

    Returns:
        early_stop: bool. True if and only if we should early stop.
    """
    # Can't early stop before we see enough losses.
    if len(losses) <= consecutive_increases:
        return False

    relevant_losses = losses[-consecutive_increases-1:]
    return sorted(relevant_losses) == relevant_losses

def early_stopping_water_mark(losses: typing.List[float], num_attempts: int=5):
    """Returns True if and only if the loss has failed to beat the low water mark in num_attempts
    attempts.

    Args:
        losses: list of floats. The losses.
        num_attempts: int. The number of attempts to beat the low water mark.

    Returns:
        early_stop: bool. True if and only if we should early stop.
    """
    # Can't early stop before we see enough losses.
    if len(losses) <= num_attempts:
        return False

    return min(losses[-num_attempts:]) > min(losses)

def train_network(network: DeepRegretNetwork, advantage_memory: buffer.Reservoir,
                  action_indexer: neural_game.ActionIndexer,
                  info_set_vectoriser: neural_game.InfoSetVectoriser,
                  current_time: int,
                  writer: tf.summary.FileWriter,
                  batch_size=1024, num_sgd_updates=4000):
    """Trains the given network from scratch

    Args:
        network: DeepRegretNetwork. The network to train.
        advantage_memory: Reservoir. Each entry should be an AdvantageMemoryElement.
        action_indexer: ActionIndexer. Turns actions into indices.
        info_set_vectoriser: InfoSetVectoriser. Turns information set ids into vectors.
        current_time: int. The current time.
        writer: tf.summary.FileWriter.
        batch_size: int. The size to use for each batch.
        num_sgd_updates: int. The number of sgd updates to do.

    Returns:
        mean_loss: float. The mean loss over the period.
    """
    # First reset the network.
    network.initialise()

    losses = []

    print("Training.")
    indices = list(range(len(advantage_memory)))
    for i in tqdm(range(num_sgd_updates)):
        # Shuffle the advantage memory.
        batch_indices = np.random.choice(indices, batch_size, replace=True)

        batch = advantage_memory.get_elements(batch_indices)

        loss, summary = network.train(batch, action_indexer, info_set_vectoriser, current_time=current_time)
        writer.add_summary(summary, network.global_step)
        losses.append(loss)

        # Early stopping.
        if early_stopping_water_mark(losses, num_attempts=20):
            print("Losses: {}".format(losses))
            print("Early stopping.")
            break

    print("Losses % through the data: {}".format(
        [losses[int(frac / 100 * len(losses))] for frac in [0.0, 25.0, 50.0, 75.0, 99.99]]
    ))

    return np.min(losses)

def compute_mean_strategy(strategy_memory: buffer.Reservoir):
    """Returns the mean strategy for each information set, weighted by time.

    Args:
        strategy_memory: Reservoir consisting of StrategyMemoryElement objects.

    Returns:
        Strategy.
    """
    strategies = collections.defaultdict(list)
    for info_set_id, t, info_set_strategy in strategy_memory.buffer:
        strategies[info_set_id].append((t, info_set_strategy))

    return extensive_game.compute_weighted_strategy(strategies)

def deep_cfr(n_game: neural_game.NeuralGame,
             num_iters: int=100, num_traversals: int=10000,
             advantage_maxlen: int=1000000, strategy_maxlen: int=1000000,
             batch_size: int=1024, num_sgd_updates: int=100):
    """
    Args:
        n_game: NeuralGame.
        num_iters: int. The number of iterations to run deep CFR for.
        num_traversals: int. The number of traversals per CFR iteration.
        advantage_maxlen: int. The maximum length of the advantage memories.
        strategy_maxlen: int. The maximum length of the strategy memory.
        batch_size: int. The batch size to use in training.
        num_sgd_updates: int. The number of sgd updates per training.

    Returns:
        strategy, exploitability.
    """
    game, action_indexer, info_set_vectoriser = n_game

    advantage_memory1 = buffer.Reservoir(maxlen=advantage_maxlen)
    advantage_memory2 = buffer.Reservoir(maxlen=advantage_maxlen)
    strategy_memory = buffer.Reservoir(maxlen=strategy_maxlen)

    # Create summary tensors
    valid_summariser = util.TBSummariser(['exploitability'])

    time_str = time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime())
    save_path = os.path.join('experiments', time_str)

    if not os.path.exists(save_path):
        print("Path doesn't exist, so creating: {}".format(save_path))
        os.makedirs(save_path)

    log_file = os.path.join(save_path, 'nfsp.log')
    print("Log file {}".format(log_file))

    print("To run tensorboard: tensorboard --logdir {}".format(os.path.join(os.getcwd(), save_path)))

    with tf.Session() as sess:
        network1 = DeepRegretNetwork(info_set_vectoriser.state_shape, action_indexer, 1)
        network1.set_sess(sess)
        network2 = DeepRegretNetwork(info_set_vectoriser.state_shape, action_indexer, 2)
        network2.set_sess(sess)

        network1.initialise()
        network2.initialise()

        tf_train_writer = tf.summary.FileWriter(os.path.join(save_path, 'train'), graph=sess.graph)

        # Iterate over players and do cfr traversals.
        for t in range(1, num_iters + 1):
            print("Iteration t = {}".format(t))
            for player in [1, 2]:
                print("Player: {}".format(player))
                print("Traversing")
                for i in tqdm(range(num_traversals)):
                    cfr_traverse(game, action_indexer, info_set_vectoriser,
                                 game.root, player, network1, network2,
                                 advantage_memory1, advantage_memory2,
                                 strategy_memory, t)

                # Train the traversing player's network on the cfr traversals.
                network = network1 if player == 1 else network2
                network.initialise()
                advantage_memory = advantage_memory1 if player == 1 else advantage_memory2
                mean_loss = train_network(
                    network, advantage_memory, action_indexer, info_set_vectoriser, t,
                    tf_train_writer, batch_size, num_sgd_updates)

                print("Mean loss: {}".format(mean_loss))
                tf_train_writer.flush()

            # print("################")
            #
            # print("----------------")
            # print("Advantage memory 1:")
            # print(advantage_memory1.buffer)
            # print("----------------")
            # print("Advantage memory 2:")
            # print(advantage_memory2.buffer)
            # print("----------------")
            #
            # print("################")
            #

            # print("----------------")
            # print("Predicted advantages:")
            # for info_set_id in set(game.info_set_ids.values()):
            #     print("{}: {}".format(
            #         info_set_id,
            #         network.predict_advantages(info_set_vectoriser.get_vector(info_set_id), action_indexer))
            #     )
            # print("----------------")
            #

            print("Advantage memory 1 length: {}".format(len(advantage_memory1)))
            print("Advantage memory 2 length: {}".format(len(advantage_memory2)))
            print("Strategy memory length: {}".format(len(strategy_memory)))

            mean_strategy = compute_mean_strategy(strategy_memory)
            # print("Strategy summary")
            # print(mean_strategy)
            if game.is_strategy_complete(mean_strategy):
                exploitability = best_response.compute_exploitability(game, mean_strategy)
            else:
                print("Strategy not complete, filling uniformly.")
                exploitability = best_response.compute_exploitability(
                    game,
                    mean_strategy,
                )
            print("Exploitability: {} mbb/h".format(exploitability * 1000))

            valid_summary = valid_summariser.summarise(sess, {'exploitability': exploitability})
            tf_train_writer.add_summary(valid_summary, global_step=t)

    # TODO(chrisn). Train the network on the strategy memory.
    return mean_strategy, exploitability

def cfr_traverse(game: extensive_game.ExtensiveGame, action_indexer: neural_game.ActionIndexer,
                 info_set_vectoriser: neural_game.InfoSetVectoriser,
                 node: extensive_game.ExtensiveGameNode, player: int,
                 network1: RegretPredictor, network2: RegretPredictor,
                 advantage_memory1: buffer.Reservoir, advantage_memory2: buffer.Reservoir,
                 strategy_memory: buffer.Reservoir, t: int):
    """

    Args:
        game: ExtensiveGame.
        action_indexer: ActionIndexer. This maps actions to indices, so that we can use neural networks.
        info_set_vectoriser: InfoSetVectoriser. This maps information sets to vectors, so we can use neural networks.
        node: ExtensiveGameNode. The current node.
        player: int. The traversing player. Either 1 or 2.
        network1: RegretPredictor. The network for player 1.
        network2: RegretPredictor. The network for player 2.
        advantage_memory1: Reservoir. The advantage memory for player 1.
        advantage_memory2: Reservoir. The advantage memory for player 2.
        strategy_memory: Reservoir. The strategy memory (for both players).
        t: int. The current iteration of deep cfr.

    Returns:

    """
    if is_terminal(node):
        return payoffs(node)[player]
    elif which_player(node) == 0:
        # Chance player
        a = sample_chance_action(node)
        return cfr_traverse(game, action_indexer, info_set_vectoriser, node.children[a], player,
                            network1, network2,
                            advantage_memory1, advantage_memory2, strategy_memory, t)
    elif which_player(node) == player:
        # It's the traversing player's turn.
        state_vector = info_set_vectoriser.get_vector(game.get_info_set_id(node))
        values = dict()
        for action in get_available_actions(node):
            child = node.children[action]
            values[action] = cfr_traverse(game, action_indexer, info_set_vectoriser, child, player,
                                          network1, network2,
                                          advantage_memory1, advantage_memory2, strategy_memory, t)
            assert values[action] is not None, print("Shouldn't be None! node was: {}".format(node))
        info_set_regrets = dict()

        # Compute the player's strategy
        network = network1 if player == 1 else network2
        if t == 1:
            # This is the equivalent of initialising the network so it starts with all zeroes.
            info_set_strategy = extensive_game.ActionFloat.initialise_uniform(action_indexer.actions)
        else:
            info_set_strategy = network.compute_action_probs(state_vector, action_indexer)

        sampled_counterfactual_value = sum([info_set_strategy[action] * values[action] for action in
                                            get_available_actions(
            node)])
        for action in get_available_actions(node):
            info_set_regrets[action] = values[action] - sampled_counterfactual_value

        info_set_id = game.info_set_ids[node]
        advantage_memory = advantage_memory1 if player == 1 else advantage_memory2
        advantage_memory.append(AdvantageMemoryElement(info_set_id, t, info_set_regrets))

        # In traverser infosets, the value passed back up is the weighted average of all action values,
        # where action a’s weight is info_set_strategy[a]
        return sampled_counterfactual_value
    else:
        # It's the other player's turn.
        state_vector = info_set_vectoriser.get_vector(game.get_info_set_id(node))

        # Compute the other player's strategy
        other_player = 1 if player == 2 else 2
        network = network1 if other_player == 1 else network2
        if t == 1:
            # This is the equivalent of initialising the network so it starts with all zeroes.
            info_set_strategy = extensive_game.ActionFloat.initialise_uniform(action_indexer.actions)
        else:
            info_set_strategy = network.compute_action_probs(state_vector, action_indexer)

        info_set_id = game.info_set_ids[node]
        strategy_memory.append(StrategyMemoryElement(info_set_id, t, info_set_strategy))

        action = sample_action(info_set_strategy, available_actions=get_available_actions(node))
        return cfr_traverse(game, action_indexer, info_set_vectoriser, node.children[action], player,
                            network1, network2, advantage_memory1, advantage_memory2, strategy_memory, t)


from faker import Faker
from faker.providers import BaseProvider

fake = Faker('pt_BR')

class ConsolidadoProvider(BaseProvider):
    def consolidado(self):
        return {
            "data": "2020-08-06 00:00", "dir": "C", "evap_piche": 4.45,
            "evap_piche_ar": 13.8, "piche": 4.45, "piche_ar": 13.8,
            "pressao": 704.2, "pressao_hpa": 802.7, "qtda": 0, "qtdb": 0,
            "qtdm": 0, "t10cm": 18.6, "t20cm": 20.7, "t30cm": 20.7,
            "t40cm": 20.5, "t5cm": 16.7, "temp_bar": 17.0, "tipoa": "",
            "tipob": "", "tipom": "", "tmax": 14.0, "tmin": 11.5,
            "tseco": 12.0, "tsfc": 11.9, "tumido": 11.4, "vento": 0.0, "vis": 6
        }

def consolidado_factory(lenght=1):
    data = list()
    fake.add_provider(ConsolidadoProvider)
    for _ in range(lenght):
        data.append(fake.consolidado())
    return data


import datetime
import calendar
import pytz
from tzlocal import get_localzone

UTC_TZ = pytz.timezone('utc')
LOCAL_TZ = get_localzone()

def utc_to_china_time(utc_dt):
    return convert_timezone(utc_dt, 'Asia/Shanghai')

def datetime_to_timestamp(dt):
    tz = dt.tzinfo
    if not tz:
        # dt MUST have timezone info
        raise ValueError
    return calendar.timegm(dt.utctimetuple())

def timestamp_to_datetime(ts, tz='utc'):
    dt = datetime.datetime.utcfromtimestamp(ts)
    utc_dt = UTC_TZ.localize(dt)
    target_tz = pytz.timezone(tz)
    return target_tz.normalize(utc_dt.astimezone(target_tz))

def localized_datetime(dt):
    """Convert Local Datetime to UCT Datetime."""
    _dt = dt
    if not dt.tzinfo:
        _dt = pytz.timezone(LOCAL_TZ.zone).localize(_dt)
    return convert_timezone(_dt, 'utc')

def convert_timezone(dt, target_tz):
    tz = dt.tzinfo
    if not tz:
        # dt MUST have timezone info
        raise ValueError
    tz = pytz.timezone(target_tz)
    return tz.normalize(dt.astimezone(tz))


# /usr/bin/env python
# -*- coding: utf-8 -*-
"""
@author: Vojtech Burian
@summary: Common library for functions executing actions over Ripple Emulator UI (Mobile OS web emulator)
"""
import time

from selenium.webdriver.common.by import By

class TestRipple():

    def __init__(self, driver):
        self.driver = driver
        self.left_section_locator = (By.CSS_SELECTOR, '.left.sortable.main.ui-sortable')
        self.right_section_locator = (By.CSS_SELECTOR, '.right.sortable.main.ui-sortable')
        self.left_arrow_locator = (By.CSS_SELECTOR, '#ui > section.left-panel-collapse.ui-state-default.'
                                                    'ui-corner-all.ui-state-hover > span')
        self.right_arrow_locator = (By.CSS_SELECTOR, '#ui > section.right-panel-collapse.ui-state-default.'
                                                     'ui-corner-all.ui-state-hover > span')
        self.gps_header_locator = (By.CSS_SELECTOR, '#gps-container > section.h2.info-header > section.collapse-handle')
        self.gps_container_locator = (By.CSS_SELECTOR, '#gps-container > section.info.ui-widget-content.ui-corner-all')
        self.geo_latitude_locator = (By.ID, 'geo-latitude')
        self.geo_longitude_locator = (By.ID, 'geo-longitude')

    def prepare_for_testing(self):
        """ prepares Ripple Simulator for App testing """
        self.expand_left_section(False)
        self.expand_right_section(False)
        self.switch_to_ripple_app()

    def expand_left_section(self, expand=True):
        """ expands or hides left section of Ripple UI controls """
        left_section = self.driver.find_element(*self.left_section_locator)
        left_arrow = self.driver.find_element(*self.left_arrow_locator)
        if expand:
            if left_section.get_attribute('style') not in ('left: 0px; opacity: 1;', ''):
                left_arrow.click()
                time.sleep(2)
        else:
            if left_section.get_attribute('style') in ('left: 0px; opacity: 1;', ''):
                left_arrow.click()
                time.sleep(2)

    def expand_right_section(self, expand=True):
        """ expands or hides right section of Ripple UI controls """
        right_section = self.driver.find_element(*self.right_section_locator)
        right_arrow = self.driver.find_element(*self.right_arrow_locator)
        if expand:
            if right_section.get_attribute('style') not in ('right: 0px; opacity: 1;', ''):
                right_arrow.click()
                time.sleep(2)
        else:
            if right_section.get_attribute('style') in ('right: 0px; opacity: 1;', ''):
                right_arrow.click()
                time.sleep(2)

    def switch_to_ripple_app(self):
        """ switches to PhoneGap HMTL app iframe (so selenium can target elements inside it) """
        app_frame = self.driver.find_elements_by_tag_name('iframe')[0]
        self.driver.switch_to_frame(app_frame)

    def switch_from_ripple_app(self):
        """ switches into the default Ripple Emulator UI DOM (so selenium can target Ripple control elements) """
        self.driver.switch_to_default_content()

    def set_geo_location(self, lat, long):
        """ sets location to given coordinates """
        self.switch_from_ripple_app()
        self.expand_right_section(True)
        if not self.driver.find_element(*self.gps_container_locator).is_displayed():
            self.driver.find_element(*self.gps_header_locator).click()
            time.sleep(1)
        latitude = self.driver.find_element(*self.geo_latitude_locator)
        longitude = self.driver.find_element(*self.geo_longitude_locator)
        latitude.clear()
        longitude.clear()
        latitude.send_keys(str(lat))
        longitude.send_keys(str(long))
        self.expand_right_section(False)
        self.driver.refresh()
        self.switch_to_ripple_app()


import discord
from discord.ext import commands
import bs4 as bs
import urllib.request
from bs4 import BeautifulSoup
from urllib.request import Request, urlopen
import json
import io
import safygiphy
from ext import embedtobox

class Nsfw:
    """ Nsfw commands """
    def __init__(self, bot):
        self.bot = bot

    async def __local_check(self, ctx):
        if not ctx.channel.is_nsfw():
            return False
        git = self.bot.get_cog('Git')
        if not await git.starred('kyb3r/selfbot.py'):
            return False
        return True

    @commands.group(invoke_without_command=True)
    async def nsfw(self, ctx):
        """ Get random lewds from the web """
        pass

    @nsfw.command()
    async def xbooru(self, ctx):
        """ Random image from Xbooru """
        try:
            try:
                await ctx.message.delete()
            except discord.Forbidden:
                pass
            await ctx.channel.trigger_typing()
            query = urllib.request.urlopen("http://xbooru.com/index.php?page=post&s=random").read()
            soup = bs.BeautifulSoup(query, 'html.parser')
            image = soup.find(id="image").get("src")
            last = str(image.split('?')[-2]).replace('//', '/').replace(':/', '://')
            em = discord.Embed(colour=discord.Colour(0xed791d))
            em.description = f'[Full Size Link*]({last})'
            em.set_image(url=last)
            em.set_footer(text='* click link at your own risk!')
            try:
                await ctx.send(embed=em)
            except discord.HTTPException:
                await ctx.send('Unable to send embeds here!')
                try:
                    async with ctx.session.get(image) as resp:
                        image = await resp.read()
                    with io.BytesIO(image) as file:
                        await ctx.send(file=discord.File(file, 'xbooru.png'))
                except discord.HTTPException:
                    await ctx.send(image)

        except Exception as e:
            await ctx.send(f'```{e}```')

    @commands.command(aliases=['gelbooru'])
    async def gel(self, ctx):
        """ Random image from Gelbooru """
        try:
            try:
                await ctx.message.delete()
            except discord.Forbidden:
                pass

            await ctx.channel.trigger_typing()
            query = urllib.request.urlopen("http://www.gelbooru.com/index.php?page=post&s=random").read()
            soup = bs.BeautifulSoup(query, 'html.parser')
            sans = soup.find_all('div', {'class': 'highres-show'})
            partial = soup.find(id="image").get("src")
            image = partial.replace('//', '/').replace(':/', '://')

            em = discord.Embed(colour=discord.Colour(0xed791d))
            em.description = f'[Full Size Link*]({image})'
            em.set_image(url=image)
            em.set_footer(text='* click link at your own risk!')
            try:
                await ctx.send(embed=em)
            except discord.HTTPException:
                # em_list = await embedtobox.etb(em)
                # for page in em_list:
                #    await ctx.send(page)
                await ctx.send('Unable to send embeds here!')
                try:
                    async with ctx.session.get(image) as resp:
                        image = await resp.read()
                    with io.BytesIO(image) as file:
                        await ctx.send(file=discord.File(file, 'gelbooru.png'))
                except discord.HTTPException:
                    await ctx.send(image)

        except Exception as e:
            await ctx.send(f'```{e}```')

    @nsfw.command()
    async def gif(self, ctx, *, tag):
        """ Get a random lewd gif
        Usage: gif <tag>
        Available tags: rule34, nsfw, hentai, tits... """
        try:
            await ctx.message.delete()
        except discord.Forbidden:
            pass
        g = safygiphy.Giphy()
        gif = g.random(tag=tag)
        color = await ctx.get_dominant_color(ctx.author.avatar_url)
        em = discord.Embed(color=color)
        em.set_image(url=str(gif.get('data', {}).get('image_original_url')))
        try:
            await ctx.send(embed=em)
        except discord.HTTPException:
            em_list = await embedtobox.etb(em)
            for page in em_list:
                await ctx.send(page)

def setup(bot):
    bot.add_cog(Nsfw(bot))


# -*- coding: utf-8 -*-
"""
Created on Thu Jul 21 10:02:43 2016
@author: zche

Given model_?.vector, reorder the embeddings for features by mapping_8627.npy and save the array to word2vec_?.npy
"""

import numpy as np

mapping = np.load('mapping_8627.npy').item()

for n_dim in [50, 200, 500, 800]:
    input_file = 'model_{}.vector'.format(n_dim)
    output_file = 'word2vec_{}.npy'.format(n_dim)
    with open(input_file, 'r') as f:
        lines = f.readlines()
    output = np.zeros([len(lines), n_dim], dtype=float)
    for line in lines[1:]:
        word_id = int(line.split(' ')[0])
        word_embedding = line.split(' ')[1:]
        line_id = mapping[word_id]
        output[line_id] = word_embedding
    np.save(output_file, output)


#!/usr/bin/env python
# coding: utf-8

import sys
sys.path.insert(0, '../')

# No GPU because working locally
import os
#os.environ["CUDA_VISIBLE_DEVICES"]=""

import time
import tensorflow as tf
from gantools import utils
from gantools.gansystem import GANsystem
from gantools import blocks
from audioinpainting.load_generator import Dataset_maestro
from audioinpainting.model_extend import InpaintingGAN

# # Parameters

downscale = 2

# # Data handling
# Load the data

start = time.time()
dataset = Dataset_maestro(scaling=downscale, spix=1024*52, augmentation=True, maxsize=2, type='maestro', path='../data', fs_rate=48000, files=8, preprocessing=False)

print('Number of samples: {}'.format(dataset.N))

#%%
# # Define parameters for the WGAN
time_str = 'extend_maestro'
global_path = '/saved_results'

name = 'WGAN' + '_' + time_str

#%%
# ## Parameters

bn = False
signal_length = 1024*52
signal_split = [1024*18, 1024*6, 1024*4, 1024*6, 1024*18]
md = 64

params_discriminator = dict()
params_discriminator['stride'] = [4,4,4,4,4]
params_discriminator['nfilter'] = [md, 2*md, 4*md, 8*md, 16*md]
params_discriminator['shape'] = [[25], [25], [25], [25], [25]]
params_discriminator['batch_norm'] = [bn, bn, bn, bn, bn]
params_discriminator['full'] = [md*4]
params_discriminator['minibatch_reg'] = False
params_discriminator['summary'] = True
params_discriminator['data_size'] = 1
params_discriminator['apply_phaseshuffle'] = True
params_discriminator['spectral_norm'] = True
params_discriminator['activation'] = blocks.lrelu

params_generator = dict()
params_generator['stride'] = [4, 4, 4, 4, 4]
params_generator['latent_dim'] = 100
params_generator['nfilter'] = [8*md, 4*md, 2*md, md, 1]
params_generator['shape'] = [[25], [25], [25], [25], [25]]
params_generator['batch_norm'] = [bn, bn, bn, bn]
params_generator['full'] = [64*md]
params_generator['summary'] = True
params_generator['non_lin'] = tf.nn.tanh
params_generator['activation'] = tf.nn.relu
params_generator['data_size'] = 1
params_generator['spectral_norm'] = True
params_generator['in_conv_shape'] =[4]

params_generator['borders'] = dict()
params_generator['borders']['nfilter'] = [md, 2*md, 4*md, 8*md, 2*md]
params_generator['borders']['batch_norm'] = [bn, bn, bn, bn, bn]
params_generator['borders']['shape'] = [[25], [25], [25], [25], [25]]
params_generator['borders']['stride'] = [4, 4, 4, 4, 4]
params_generator['borders']['data_size'] = 1
params_generator['borders']['width_full'] = 128
params_generator['borders']['activation'] = tf.nn.relu

params_optimization = dict()
params_optimization['batch_size'] = 64
params_optimization['epoch'] = 10000
params_optimization['n_critic'] = 5
params_optimization['generator'] = dict()
params_optimization['generator']['optimizer'] = 'adam'
params_optimization['generator']['kwargs'] = {'beta1':0.5, 'beta2':0.9}
params_optimization['generator']['learning_rate'] = 1e-4
params_optimization['discriminator'] = dict()
params_optimization['discriminator']['optimizer'] = 'adam'
params_optimization['discriminator']['kwargs'] = {'beta1':0.5, 'beta2':0.9}
params_optimization['discriminator']['learning_rate'] = 1e-4

# all parameters
params = dict()
params['net'] = dict() # All the parameters for the model
params['net']['generator'] = params_generator
params['net']['discriminator'] = params_discriminator
params['net']['prior_distribution'] = 'gaussian'
params['net']['shape'] = [signal_length, 1] # Shape of the image
params['net']['inpainting'] = dict()
params['net']['inpainting']['split'] = signal_split
params['net']['gamma_gp'] = 10 # Gradient penalty
params['net']['fs'] = 16000//downscale
params['net']['loss_type'] ='wasserstein'

params['optimization'] = params_optimization
params['summary_every'] = 100 # Tensorboard summaries every ** iterations
params['print_every'] = 50 # Console summaries every ** iterations
params['save_every'] = 1000 # Save the model every ** iterations
params['summary_dir'] = os.path.join(global_path, name +'_summary/')
params['save_dir'] = os.path.join(global_path, name + '_checkpoints/')
params['Nstats'] = 0

resume, params = utils.test_resume(False, params)

#%%
# # Build the model
wgan = GANsystem(InpaintingGAN, params)

# # Train the model
wgan.train(dataset, resume=resume)

end = time.time()
print('Elapse time: {} minutes'.format((end - start)/60))


# coding: mbcs

import wx
import MainFrame
from todpython import *

#-------------------------------------------------------------------------------
def initialize(parent, obj):
    frame = RpBaseEditor(MainFrame.MainFrame.instance())
    frame.initialize(obj)
    MainFrame.MainFrame.instance().auimgr.AddPane(frame, wx.aui.AuiPaneInfo().MinSize(wx.Size(200, 100)).
        Caption('RenderPath - Base Editor').Dockable(True).Right().CloseButton(True).MinimizeButton(True).DestroyOnClose(True).Float())
    MainFrame.MainFrame.instance().auimgr.Update()

#-------------------------------------------------------------------------------
class RpBaseEditor(wx.Panel):
    def __init__(self, parent):
        wx.Panel.__init__(self, parent, style=wx.WANTS_CHARS)

    def initialize(self, obj):
        self.obj = obj

        self.mainSizer = wx.BoxSizer(wx.VERTICAL)

        for i in range(0, self.obj.getShaderParamNum()):
            name, semantic, type, value = self.obj.getShaderParamDesc(i)

            bsizer = wx.BoxSizer(wx.HORIZONTAL)
            static = wx.StaticText(id=wx.NewId(), label=name, parent=self, style=wx.ALIGN_RIGHT)
            static.SetMinSize(wx.Size(150, 14))
            bsizer.Add(static)
            textctrl = wx.TextCtrl(id=wx.NewId(), name=name + ":" + type, value=value, parent=self)
            self.Bind(wx.EVT_TEXT_ENTER, self.OnTextEnter, id=textctrl.GetId())
            bsizer.Add(textctrl, 1, wx.ALL | wx.EXPAND)

            self.mainSizer.Add(bsizer, 0, wx.ALL | wx.EXPAND)

        self.SetSizer(self.mainSizer)
        self.Fit()

    def OnTextEnter(self, event):
        t = event.GetEventObject()
        n = t.GetName().split(':')
        param_name = str(n[0])
        param_type = str(n[1])

        if param_type == 'TYPE_FLOAT':
            self.obj.setFloat(param_name, float(t.GetValue()))
        elif param_type == 'TYPE_TEXTURE':
            self.obj.setTexture(param_name, str(t.GetValue()))


import os
import json

with open('E:\桌面\笔记\Django\DrPro\static\json\index.json', 'r', encoding="utf-8") as f:
    obj = json.load(f)
    list1 = []
    # print(obj, type(obj))
    for i in obj:
        # print('insert into lunbo("img","id",) values("%s","%s");' % (i.get("img"), i.get("id")))
        # print(i[-1])
        print(obj.index(i))
        for j in i:
            list1.append(j)
            # print(i.index(j))
            # print(j)
            # print('insert into dr_goodlist(index_id,title,src,price,discript,sale,com,material1,material2,price1,price2,is_delete) values(%s,"%s","%s","%s","%s","%s","%s","%s","%s","%s","%s",0);' % (j.get("index"),j.get("title"),j.get("src"),j.get("price"),j.get("discript"),j.get("sale"),j.get("com"),j.get("material1"),j.get("material2"),j.get("price1"),j.get("price2")))
    # for k in (list1[:2] + list1[14:18]):  # 导航栏
    # for k in list1[18:22]:  # 主页大图
    # for k in list1[22:26]:  # 店铺图
    # for k in list1[2:12]:  # 产品图
    for k in list1:
        print(k, list1.index(k))
        # print('insert into dr_bar(title,src) values("%s","/static/%s");' % (k.get("title"), k.get("src")))  # 导航栏
    #     print('insert into dr_storeimg(title,src) values("%s","/static/%s");' % (k.get("title"), k.get("src")))  # 店铺图
        # print('insert into dr_productimg(title,src,href) values("%s","/static/%s","%s");' % (k.get("title"), k.get("src"), k.get("href")))  # 产品图
        # for m in k:
        #     print('insert into dr_indeximg(src) values("/static/%s");' % m.get("src"))  # 主页大图


#!/usr/bin/env python
#
# Copyright 2007 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import webapp2
from google.appengine.ext.webapp.util import run_wsgi_app
from google.appengine.api import users
import wsgiref.handlers
from google.appengine.ext.webapp import template

import atom.url

import gdata.service
import gdata.alt.appengine
import gdata.photos.service
import gdata.media
import gdata.geo
import gdata.youtube
import gdata.youtube.service

from getdata import *
from pygooglechart import Chart
from pygooglechart import SimpleLineChart
from pygooglechart import Axis

class IndexHandler(webapp2.RequestHandler):
    def get(self):
        username = 'aayushahuja'
        gd_client = gdata.photos.service.PhotosService()

        albums = gd_client.GetUserFeed(user=username)
        for album in albums.entry:
            if album.title.text == 'test':
                photos = gd_client.GetFeed(
                                   '/data/feed/api/user/%s/albumid/%s?kind=photo' % (
                                                                     username, album.gphoto_id.text))
        title  = 'about'
        subtitle = getdata(title,'NGOData').custom['subtitle'].text
        content = getdata(title,'NGOData').custom['content'].text
        self.response.content_type='text/html'
        self.response.out.write(template.render('template/index.html',{
                                                'photolist':photos.entry,
                                                'page_title':title.title(),
                                                'page_subtitle':subtitle,
                                                'page_content':content,
                                                                                }))
class CharitiesHandler(webapp2.RequestHandler):
    def get(self):

        title  = 'charities'
        subtitle = getdata(title,'NGOData').custom['subtitle'].text
        content = getdata(title,'NGOData').custom['content'].text
        self.response.content_type='text/html'
        self.response.out.write(template.render('template/standard.html',{

                                                'page_title':title.title(),
                                                'page_subtitle':subtitle,
                                                'page_content':content,
                                                                                }))

class ProgramsHandler(webapp2.RequestHandler):
    def get(self):
        title  = 'programs'
        subtitle = getdata(title,'NGOData').custom['subtitle'].text
        content = getdata(title,'NGOData').custom['content'].text
        self.response.content_type='text/html'
        self.response.out.write(template.render('template/standard.html',{

                                                'page_title':title.title(),
                                                'page_subtitle':subtitle,
                                                'page_content':content,
                                                                                }))
class ActivitiesHandler(webapp2.RequestHandler):
    def get(self):

        title  = 'activities'
        subtitle = getdata(title,'NGOData').custom['subtitle'].text
        content = getdata(title,'NGOData').custom['content'].text
        self.response.content_type='text/html'
        self.response.out.write(template.render('template/standard.html',{

                                                'page_title':title.title(),
                                                'page_subtitle':subtitle,
                                                'page_content':content,
                                                                                }))

class YoutubeHandler(webapp2.RequestHandler):
    def PrintVideoFeed(feed):
        for entry in feed.entry:
            PrintEntryDetails(entry)
    def get(self):
        username = 'tseries' # to be changed to NGO's username
        yt_service = gdata.youtube.service.YouTubeService()
        uri = 'http://gdata.youtube.com/feeds/api/users/%s/uploads' % username
        feed = yt_service.GetYouTubeVideoFeed(uri)
        list = []
        for entry in feed.entry:
            list.append(entry.GetSwfUrl())
        list = list[0:4]
        self.response.out.write(template.render('template/videos.html',{
                                                'list':list,
                                                                                }))

class MainHandler(webapp2.RequestHandler):
    def get(self):
        self.redirect('/index')

class ContactsHandler(webapp2.RequestHandler):
    def get(self):
        title  = 'contacts'
        subtitle = getdata(title,'NGOData').custom['subtitle'].text
        content = getdata(title,'NGOData').custom['content'].text
        self.response.content_type='text/html'
        self.response.out.write(template.render('template/contacts.html',{

                                                                 'page_title':title.title(),
                                                                 'page_subtitle':subtitle,
                                                                 'page_content':content,                  }))

class DonationHandler(webapp2.RequestHandler):
    def get(self):
        list = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep']
        list2 = []
        for title in list:
            list2.append( int(getdata(title,'funds').custom['amount'].text))
        max_y = 10000
        chart = SimpleLineChart(200, 125, y_range=[0, max_y])

        chart.add_data([2000,3000,5000,1200,5000,4000,1000,3000,5900])

        # Set the line colour to blue
        chart.set_colours(['0000FF'])

        # Set the vertical stripes
        chart.fill_linear_stripes(Chart.CHART, 0, 'CCCCCC', 0.2, 'FFFFFF', 0.2)

        # Set the horizontal dotted lines
        chart.set_grid(0, 25, 5, 5)

        # The Y axis labels contains 0 to 100 skipping every 25, but remove the
        # first number because it's obvious and gets in the way of the first X
        # label.
        left_axis = range(0, max_y + 1, 25)
        left_axis[0] = ''
        chart.set_axis_labels(Axis.LEFT, left_axis)

        # X axis labels
        chart.set_axis_labels(Axis.BOTTOM, list)

        url2 = chart.get_url()
        self.response.out.write(template.render('template/donate.html',{

                                                                 'url2' :url2,                }))

app = webapp2.WSGIApplication([
    ('/', MainHandler),
    ('/index', IndexHandler),
    ('/activities', ActivitiesHandler),
    ('/charities', CharitiesHandler),
    ('/programs', ProgramsHandler),
    ('/contacts', ContactsHandler),
    ('/donate', DonationHandler),
    ('/videos', YoutubeHandler),
], debug=True)


import re
import logging
import traceback
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import get_object_or_404, render
from django.contrib.auth.decorators import login_required
from django.contrib.auth import logout as django_logout
from django.template import RequestContext
from django.conf import settings
from django.views.decorators.cache import cache_control
from userservice.user import UserService
from myuw.dao.term import get_current_quarter
from myuw.dao.pws import is_student
from myuw.dao.affiliation import get_all_affiliations, is_oldmyuw_user
from myuw.dao.emaillink import get_service_url_for_address
from myuw.dao.exceptions import EmailServiceUrlException
from myuw.logger.timer import Timer
from myuw.logger.logback import log_exception
from myuw.logger.logresp import log_invalid_netid_response
from myuw.logger.logresp import log_success_response_with_affiliation
from myuw.logger.session_log import log_session
from myuw.views.rest_dispatch import invalid_session
from myuw.dao.uwemail import get_email_forwarding_for_current_user
from myuw.dao.card_display_dates import get_card_visibilty_date_values

logger = logging.getLogger(__name__)
LOGOUT_URL = "/user_logout"

@login_required
@cache_control(max_age=0, no_cache=True, no_store=True, must_revalidate=True)
def index(request,
          year=None,
          quarter=None,
          summer_term=None):

    timer = Timer()
    netid = UserService().get_user()
    if not netid:
        log_invalid_netid_response(logger, timer)
        return invalid_session()

    if _is_mobile(request):
        # On mobile devices, all students get the current myuw.  Non-students
        # are sent to the legacy site.
        try:
            if not is_student():
                logger.info("%s not a student, redirect to legacy!" % netid)
                return redirect_to_legacy_site()
        except Exception:
            log_exception(logger,
                          '%s is_student' % netid,
                          traceback.format_exc())
            logger.info("%s, redirected to legacy!" % netid)
            return redirect_to_legacy_site()

    else:
        if is_oldmyuw_user():
            return redirect_to_legacy_site()

    context = {
        "year": year,
        "quarter": quarter,
        "summer_term": summer_term,
        "home_url": "/",
        "err": None,
        "user": {
            "netid": None,
            "affiliations": get_all_affiliations(request)
        },
        "card_display_dates": get_card_visibilty_date_values(request),
    }

    context["user"]["session_key"] = request.session.session_key
    log_session(netid, request.session.session_key, request)
    try:
        my_uwemail_forwarding = get_email_forwarding_for_current_user()
        if my_uwemail_forwarding.is_active():
            c_user = context["user"]
            try:
                (c_user['email_forward_url'],
                 c_user['email_forward_title'],
                 c_user['email_forward_icon']) = get_service_url_for_address(
                     my_uwemail_forwarding.fwd)
            except EmailServiceUrlException:
                c_user['login_url'] = None
                c_user['title'] = None
                c_user['icon'] = None
                logger.info('No Mail Url: %s' % (
                    my_uwemail_forwarding.fwd))

    except Exception:
        log_exception(logger,
                      'get_email_forwarding_for_current_user',
                      traceback.format_exc())
        pass

    context["user"]["netid"] = netid
    if year is None or quarter is None:
        cur_term = get_current_quarter(request)
        if cur_term is None:
            context["err"] = "No current quarter data!"
        else:
            context["year"] = cur_term.year
            context["quarter"] = cur_term.quarter
    else:
        pass
    log_success_response_with_affiliation(logger, timer, request)
    return render(request, "index.html", context)

def _is_mobile(request):
    user_agent = request.META.get("HTTP_USER_AGENT")

    if not user_agent:
        return False

    # This is the check we were doing in our apache config...
    if re.match('.*iPhone.*', user_agent):
        return True

    if re.match('.*Android.*Mobile.*', user_agent):
        return True
    return False

def redirect_to_legacy_site():
    legacy_url = getattr(settings,
                         "MYUW_USER_SERVLET_URL",
                         "https://myuw.washington.edu/servlet/user")
    return HttpResponseRedirect(legacy_url)

def logout(request):
    # Expires current myuw session
    django_logout(request)

    # Redirects to weblogin logout page
    return HttpResponseRedirect(LOGOUT_URL)


import pandas as pd
import matplotlib.pyplot as plt
bridge_readings={'Distance_mm':[50012,50015,50005,502412,50007,50014]}
df=pd.DataFrame(bridge_readings)
#df.plot()
#plt.show()
stats=df.describe()
print(stats)
print(stats.Distance_mm['std'])
df=df[(df['std']<50)]


import struct
from enum import Enum

from twisted.internet import defer
from twisted.internet.protocol import Factory, Protocol

from qtm.packet import QRTPacketType, QRTPacket, QRTEvent

RTheader = struct.Struct("<II")
RTEvent = struct.Struct("<c")

RTCommand = "<II%dsc"

class QRTLoggerInfo(Enum):
    Sent = 0
    Received = 1
    Error = 2
    Event = 3

class QRTCommandException(Exception):
    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)

# noinspection PyPep8Naming
class QRTProtocol(Protocol):
    def __init__(self, version, on_disconnect=None, on_event=None, logger=None):
        self.version_checked = False
        self.received_data = b''
        self.version = version

        self.on_disconnect = on_disconnect
        self.on_event = on_event
        self.logger = logger

        self.on_packet = None

        self.connected_d = defer.Deferred()

        self.request_queue = []

    @defer.inlineCallbacks
    def get_version(self):
        version_cmd = "Version %s" % self.version
        try:
            self.version_checked = True
            result = yield self.send_command(version_cmd)
        except QRTCommandException as e:
            self.connected_d.errback(e)
        else:
            self.connected_d.callback(result)

    def send_command(self, command, callback=True, command_type=QRTPacketType.PacketCommand):
        if self.logger:
            self.logger(command, QRTLoggerInfo.Sent)

        cmd_length = len(command)
        self.transport.write(struct.pack(RTCommand % cmd_length, RTheader.size + cmd_length + 1,
                                         command_type.value, command.encode(), b'\0'))
        d = None
        if callback:
            d = defer.Deferred()
            self.request_queue.append(d)

        return d

    def connectionMade(self):
        self.transport.setTcpNoDelay(True)

    def connectionLost(self, reason):
        if self.on_disconnect:
            self.on_disconnect(reason.getErrorMessage())

    def dataReceived(self, data):
        self.received_data += data
        h_size = RTheader.size

        data = self.received_data
        size, type_ = RTheader.unpack_from(data, 0)

        while len(data) >= size:
            self.parse_received(data[h_size:size], type_)
            data = data[size:]

            if len(data) < h_size:
                break

            size, type_ = RTheader.unpack_from(data, 0)

        self.received_data = data

    def set_on_packet(self, on_packet):
        self.on_packet = on_packet

    def parse_received(self, data, type_):
        type_ = QRTPacketType(type_)
        # print data, type_

        # never any callbacks
        if type_ == QRTPacketType.PacketEvent:
            event, = RTEvent.unpack(data)
            event = QRTEvent(ord(event))

            if self.logger:
                self.logger(event.name, QRTLoggerInfo.Event)

            if self.on_event:
                self.on_event(event)
            return

        # Get a deferred to return result
        d = self.request_queue.pop(0) if len(self.request_queue) > 0 else None

        if type_ == QRTPacketType.PacketError:
            response = data[:-1]
            if self.logger:
                self.logger(response, QRTLoggerInfo.Error)

            if d:
                d.errback(QRTCommandException(response))

        elif type_ == QRTPacketType.PacketXML:
            response = data[:-1]

            if d:
                d.callback(response)

        elif type_ == QRTPacketType.PacketCommand:
            response = data[:-1]

            if not self.version_checked:
                self.get_version()

            if self.logger:
                self.logger(response, QRTLoggerInfo.Received)

            if d:
                d.callback(response)

        elif type_ == QRTPacketType.PacketData:
            if d:
               d.callback('Ok')

            packet = QRTPacket(data)
            if self.on_packet:
                self.on_packet(packet)

class QRTFactory(Factory):
    protocol = QRTProtocol

    def __init__(self, version, on_disconnect=None, on_event=None, logger=None):
        self.version = version
        self.logger = logger

        self.on_disconnect = on_disconnect
        self.on_event = on_event

    def buildProtocol(self, addr):
        p = self.protocol(self.version, self.on_disconnect, self.on_event, self.logger)
        p.factory = self
        return p


import random
import re
import requests
import sys
import time

import bottlenose
from bs4 import BeautifulSoup

from django.core.management.base import BaseCommand, CommandError

from camera.models import CameraBrand, Camera
from camera_lens.models import CameraLens

AWS_ACCESS_KEY_ID=''
AWS_SECRET_ACCESS_KEY=''
AWS_ASSOCIATE_TAG=''

amazon = bottlenose.Amazon(AWS_ACCESS_KEY_ID,
    AWS_SECRET_ACCESS_KEY, AWS_ASSOCIATE_TAG)

# I don't know the URL to look these up, but these are what amazon has in the finder as of 1/08/2014
CAMERA_BRANDS = ["Canon","Fujifilm","Leica","Nikon","Olympus","Panasonic","Pentax","Samsung","Sony"]

CANON_BRAND_SERIES_LIST = ["EOS", "EOS Rebel"]
FUJI_BRAND_SERIES_LIST = ["X series"]
LEICA_BRAND_SERIES_LIST = ["M series", "T series", ]
NIKON_BRAND_SERIES_LIST = ["CX (Nikon 1)", "DX", "FX"]
OLYMPUS_BRAND_SERIES_LIST = ["E series", "O-MD", "PEN"]
PANASONIC_BRAND_SERIES_LIST = ["Lumix G"]
PENTAX_BRAND_SERIES_LIST = ["K series", "Q series"]
SAMSUNG_BRAND_SERIES_LIST = ["NX"]
SONY_BRAND_SERIES_LIST = ["A-mount", "E-mount", "FE-mount"]

CANON_EOS_CAMERAS = ["10D","1D","1D C","1D Mark II","1D Mark II N","1D Mark III","1D Mark IV","1D X","1Ds","1Ds Mark II","1Ds Mark III","20D","20Da","300D","30D","40D","50D","5D","5D Mark II","5D Mark III","60D","60Da","6D","70D","7D","7D Mark II","D30","D60","M"]
CANON_EOS_REBEL_CAMERAS = ["SL1","T1i","T2i","T3","T3i","T4i","T5","T5i","XS","XSi","XT","XTi"]

FUJI_X_SERIES_CAMERAS = ["X-A1","X-E1","X-E2","X-M1","X-Pro1","X-T1"]

LEICA_M_SERIES_CAMERAS = ["M Typ 240","M-E Typ 220","M-Monochrom","M8","M8.2","M9","M9-P"]
LEICA_T_SERIES_CAMERAS = ["T (Typ 701)"]

NIKON_CX_CAMERAS = ["AW1","J1","J2","J3","J4","S1","S2","V1","V2","V3"]
NIKON_DX_CAMERAS = ["D100","D1H","D1X","D200","D2H","D2Hs","D2X","D2Xs","D300","D3000","D300S","D3100","D3200","D3300","D40","D40X","D50","D5000","D5100","D5200","D5300","D60","D70","D7000","D70s","D7100","D80","D90"]
NIKON_FX_CAMERAS = ["D3", "D3S", "D3X", "D4", "D4s", "D600", "D610", "D700", "D750", "D800", "D800E", "D810", "Df"]

OLYMPUS_E_SERIES_CAMERAS = ["E-1","E-3","E-30","E-300","E-330","E-410","E-420","E-450","E-5","E-500","E-510","E-520","E-600","E-620"]
OLYMPUS_OMD_CAMERAS = ["E-M1","E-M10","E-M5"]
OLYMPUS_PEN_CAMERAS = ["E-P1","E-P2","E-P3","E-P5","E-PL1","E-PL1s","E-PL2","E-PL3","E-PL5","E-PL6","E-PL7","E-PM1","E-PM2"]

PANASONIC_LUMIX_G_CAMERAS = ["G1","G10","G2","G3","G5","G6","GF1","GF2","GF3","GF5","GF6","GH1","GH2","GH3","GH4","GM1","GM5","GX1","GX7"]

PENTAX_K_SERIES_CAMERAS = ["K-01","K-3","K-30","K-5","K-5 II","K-5 IIs","K-50","K-500","K-7","K-m","K-r","K-S1","K-x","K100D","K100D Super","K10D","K110D","K200D","K20D"]
PENTAX_Q_SERIES_CAMERAS = ["Q","Q-S1","Q10","Q7"]

SAMSUNG_NX_CAMERAS = ["NX mini","NX1","NX10","NX100","NX1000","NX1100","NX20","NX200","NX2000","NX210","NX30","NX300","NX3000","NX300M"]

SONY_A_MOUNT_CAMERAS = ["Alpha DSLR-A100","Alpha DSLR-A200","Alpha DSLR-A230","Alpha DSLR-A290","Alpha DSLR-A300","Alpha DSLR-A330","Alpha DSLR-A350","Alpha DSLR-A390","Alpha DSLR-A500","Alpha DSLR-A550","Alpha DSLR-A560","Alpha DSLR-A580","Alpha DSLR-A700","Alpha DSLR-A850","Alpha DSLR-A900","Alpha SLT-A33","Alpha SLT-A35","Alpha SLT-A37","Alpha SLT-A55","Alpha SLT-A57","Alpha SLT-A58","Alpha SLT-A65","Alpha SLT-A77","Alpha SLT-A77 II","Alpha SLT-A99"]
SONY_E_MOUNT_CAMERAS = ["3","3N","5","5N","5R","5T","6","7","Alpha 3000","Alpha 5000","Alpha 5100","Alpha 6000","C3","F3"]
SONY_FE_MOUNT_CAMERAS = ["Alpha 7","Alpha 7 II","Alpha 7R","Alpha 7S"]

# set the headers to something other than python user agent.
r_headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:34.0) Gecko/20100101 Firefox/34.0' }

def lookup_lens_asins_for_cam(brand, brand_series, model_name):
    url = "http://www.amazon.com/gp/finders/ajax/finderajax.html?action=getSupplies&finderId=lens&nodeId=" + brand + "%5E" + brand_series.replace(" ", "+") + "%5E" + model_name.replace(" ", "+") + "&supplyPage=1&pageSize=999&refinements=&supplyBrand=&redir=pf_rd_p%3D1561025342%26pf_rd_s%3Dcenter-5%26pf_rd_t%3D101%26pf_rd_i%3D6207565011%26pf_rd_m%3DATVPDKIKX0DER%26pf_rd_r%3D0B8F6QAY4MYB29EFZS0H&PowerBar=0"
    try:
        r = requests.get(url, headers=r_headers)
        soup = BeautifulSoup(r.text)
        asins = re.findall(r'"ASIN":"(\w+)"', str(soup.prettify()))
        return asins
    except:
        print("Couldnt return asins for " + brand + " " + model_name)

def get_all_brand_series(brand):
    if brand == 'Canon':
        # can re-work this later incorporating url to get all the series but since its not updated often and the urls are tricky hard-coding per brand for now
        return CANON_BRAND_SERIES_LIST
    elif brand == 'Fujifilm':
        return FUJI_BRAND_SERIES_LIST
    elif brand == 'Leica':
        return LEICA_BRAND_SERIES_LIST
    elif brand == 'Nikon':
        return NIKON_BRAND_SERIES_LIST
    elif brand == 'Olympus':
        return OLYMPUS_BRAND_SERIES_LIST
    elif brand == 'Panasonic':
        return PANASONIC_BRAND_SERIES_LIST
    elif brand == 'Pentax':
        return PENTAX_BRAND_SERIES_LIST
    elif brand == 'Samsung':
        return SAMSUNG_BRAND_SERIES_LIST
    elif brand == "Sony":
        return SONY_BRAND_SERIES_LIST
    else:
        print("Sorry no matches for this, this should never happen, going to exit now with error.")
        sys.exit(1)

def get_series_models(brand, brand_series):
    if brand == 'Canon':
        if brand_series == 'EOS':
            return CANON_EOS_CAMERAS
        elif brand_series == 'EOS Rebel':
            return CANON_EOS_REBEL_CAMERAS
        else:
            print("No match, this shouldnt ever happen. Exiting with error...")
            sys.exit(1)

    elif brand == 'Fujifilm':
        if brand_series == 'X series':
            return FUJI_X_SERIES_CAMERAS
        else:
            print("No match, this shouldnt ever happen. Exiting with error...")
            sys.exit(1)

    elif brand == 'Leica':
        if brand_series == 'M series':
            return LEICA_M_SERIES_CAMERAS
        elif brand_series == 'T series':
            return LEICA_T_SERIES_CAMERAS
        else:
            print("No match, this shouldnt ever happen. Exiting with error...")
            sys.exit(1)

    elif brand == 'Nikon':
        if brand_series == 'CX (Nikon 1)':
            return NIKON_CX_CAMERAS
        elif brand_series == 'DX':
            return NIKON_DX_CAMERAS
        elif brand_series == 'FX':
            return NIKON_FX_CAMERAS
        else:
            print("No match, this shouldnt ever happen. Exiting with error...")
            sys.exit(1)

    elif brand == 'Olympus':
        if brand_series == 'E series':
            return OLYMPUS_E_SERIES_CAMERAS
        elif brand_series == 'O-MD':
            return OLYMPUS_OMD_CAMERAS
        elif brand_series == 'PEN':
            return OLYMPUS_PEN_CAMERAS
        else:
            print("No match, this shouldnt ever happen. Exiting with error...")
            sys.exit(1)

    elif brand == 'Panasonic':
        if brand_series == 'Lumix G':
            return PANASONIC_LUMIX_G_CAMERAS
        else:
            print("No match, this shouldnt ever happen. Exiting with error...")
            sys.exit(1)

    elif brand == 'Pentax':
        if brand_series == 'K series':
            return PENTAX_K_SERIES_CAMERAS
        elif brand_series == 'Q series':
            return PENTAX_Q_SERIES_CAMERAS
        else:
            print("No match, this shouldnt ever happen. Exiting with error...")
            sys.exit(1)

    elif brand == 'Samsung':
        if brand_series == 'NX':
            return SAMSUNG_NX_CAMERAS
        else:
            print("No match, this shouldnt ever happen. Exiting with error...")
            sys.exit(1)

    elif brand == 'Sony':
        if brand_series == 'A-mount':
            return SONY_A_MOUNT_CAMERAS
        elif brand_series == 'E-mount':
            return SONY_E_MOUNT_CAMERAS
        elif brand_series == 'FE-mount':
            return SONY_FE_MOUNT_CAMERAS
        else:
            print("No match, this shouldnt ever happen. Exiting with error...")
            sys.exit(1)

    else:
        print("Sorry no matches for this, this should never happen, going to exit now with error.")
        sys.exit(1)

def amazon_query_asin(asin):
    try:
        response = amazon.ItemLookup(ItemId=asin)
    except:
        print("error performing amazon lookup of " + asin)
        sys.exit(1)
    try:
        soup = BeautifulSoup(response)
    except:
        print("error creating soup from response.")
    return soup

"""
    response = amazon.ItemLookup(ItemId=asin)
    soup = BeautifulSoup(response)
    return unicode(soup)
"""

"""
def re_lens_title(title_long):
    try:
        title = re.search(r"(^.*?) Lens for", title_long).group(1)
        print("regex match - title set to " + title)
    except:
        title = str(title_long)
        print("no regex match - title set to " + title)
    return title
"""

class Command(BaseCommand):
    def handle(self, **options):
        for brand in CAMERA_BRANDS:
            for brand_series in get_all_brand_series(brand=brand):
                for model_name in get_series_models(brand=brand, brand_series=brand_series):
                    cam_title = brand + " " + model_name
                    print(cam_title)
                    try:
                        c = Camera.objects.get(title=cam_title)
                        print("Matched on " + cam_title)
                    except:
                        print("No match for " + cam_title + " creating...")
                        c = Camera.objects.create(title=cam_title, brand=CameraBrand.objects.get(title=brand))

                    asins = lookup_lens_asins_for_cam(brand=brand, brand_series=brand_series, model_name=model_name)
                    for asin in asins:
                        print("Checking if ASIN already exists... " + asin)
                        try:
                            l = CameraLens.objects.get(amazon_asin=asin)
                        except:
                            print("Lens with ASIN " + asin + " doesnt already exist, querying amazon and creating.")

                            soup = amazon_query_asin(asin=asin)
                            try:
                                title_long = soup.find("title").text
                                title = unicode(title_long)
                                #title = re_lens_title(title_long)
                            except:
                                print("errored the fucking souping title_long text...")
                                print(soup)

                            try:
                                amazon_detail_page_url = unicode(soup.find("detailpageurl").text)
                            except:
                                print("error getting amazon detail page, setting it to blank")
                                amazon_detail_page_url = ""

                            try:
                                l = CameraLens.objects.get_or_create(title=title, amazon_asin=asin,amazon_detail_page_url=amazon_detail_page_url)
                            except:
                                print("Error: Failed to get or create CameraLens object...")
                                #sys.exit(1)
                        try:
                            c.lenses.add(l)
                        except:
                            print("Something fucked up adding lens to camera")
                            #sys.exit(1)

                        print("looks like we made it through, sleeping for a bit then we go again...")

                        sleep_time = random.randint(5,6)
                        time.sleep(sleep_time)


import urllib.request
from bs4 import BeautifulSoup
import re

id_list = []
video_list = []
count = 0
for i in ['1', '2', '3', '4']:
    html = urllib.request.urlopen('http://www.marschen.com/forum.php?mod=forumdisplay&fid=2&page=' + i).read().decode('gbk')
    soup = BeautifulSoup(html, 'html.parser')
    tids = soup.select('tbody')
    for tid in tids:
        id = tid.get('id')
        if '_' in id:
            id_list.append(id.split('_')[1])
print(id_list)
print(len(id_list))

for id in id_list:
    try:
        html = urllib.request.urlopen('http://www.marschen.com/forum.php?mod=viewthread&tid=' + id).read().decode('gbk')
        soup = BeautifulSoup(html, 'html.parser')
        if soup.select('td.t_f font'):
            video_title = soup.select('span#thread_subject')[0].get_text()
            video_name = soup.select('td.t_f font')[0].get_text()
            video_js = soup.select('td.t_f script')[-1].get_text()
            if re.findall(r'http.*swf', video_js):
                count += 1
                video_url = re.findall(r'http.*swf', video_js)[0]
                # video_data = {
                #     'title': video_title,
                #     'name': video_name,
                #     'url': video_url
                # }
                print(str(count), video_title, video_name, video_url)
                # video_list.append(video_data)
    except:
        pass
# print(video_list)
# print(len(video_list))


from icevision.imports import *
from icevision.backbones import resnet_fpn
from icevision.models.rcnn.utils import *

from torchvision.models.detection.faster_rcnn import (
    fasterrcnn_resnet50_fpn,
    FasterRCNN,
    FastRCNNPredictor,
)

def model(
    num_classes: int,
    backbone: Optional[nn.Module] = None,
    remove_internal_transforms: bool = True,
    **faster_rcnn_kwargs
) -> nn.Module:
    """FasterRCNN model implemented by torchvision.

    # Arguments
        num_classes: Number of classes.
        backbone: Backbone model to use. Defaults to a resnet50_fpn model.
        remove_internal_transforms: The torchvision model internally applies transforms
        like resizing and normalization, but we already do this at the `Dataset` level,
        so it's safe to remove those internal transforms.
        **faster_rcnn_kwargs: Keyword arguments that internally are going to be passed to
        `torchvision.models.detection.faster_rcnn.FastRCNN`.

    # Returns
        A Pytorch `nn.Module`.
    """
    if backbone is None:
        model = fasterrcnn_resnet50_fpn(pretrained=True, **faster_rcnn_kwargs)

        in_features = model.roi_heads.box_predictor.cls_score.in_features
        model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)

        backbone_param_groups = resnet_fpn.param_groups(model.backbone)
    else:
        model = FasterRCNN(backbone, num_classes=num_classes, **faster_rcnn_kwargs)
        backbone_param_groups = backbone.param_groups()

    patch_param_groups(model=model, backbone_param_groups=backbone_param_groups)

    if remove_internal_transforms:
        remove_internal_model_transforms(model)

    return model


def bijiao(fname1,fname2):
    f1 = open(fname1,'r')
    f2 = open(fname2,'r')
    count = 0 #统计行数
    differ = [] #统计不一样的行数
    for line1 in f1:
        line2 = f2.readline() #记住这里的编程变换方法
        count += 1
        if line1 != line2:
            differ.append(count)
    f1.close()
    f2.close()
    return differ

fname1 = input('请输入第一个文件名：')
fname2 = input('请输入第二个文件名：')
differ = bijiao(fname1,fname2)
if len(differ) == 0:
    print('两个文件完全一样')
else:
    print('两个文件共有【%d】处不同'%len(differ))
    for each in differ:
        print('第%d行不一样'%each)


#import statement needed to change pathing
import sys
sys.path.append("./../")

from pymatgen import MPRester
import pymatgen as pg
import numpy as np
import cPickle as pickle
import csv
import os
import re
from collections import defaultdict
import copy

import mpdata as mp
import gatherLPNorm as gm

PARTIAL_PATH = "Norm_mapping"
FILEPATH = "Computed CSV/NonLithiated_Atoms.csv"
OUTPUT_FILE = "Num_Compound.csv"

class getLiRatio:
	getAtomNum = re.compile(r'[^\d]')
	getAtomType = re.compile(r'[\d]')

	def __init__(self, li_atomList, nonLi_atomList, LiMapping):
		#define some new constants here
		self.li_atomList = li_atomList
		self.nonLi_atomList = nonLi_atomList
		self.LiMapping = LiMapping
		self.DIR = os.path.dirname(__file__)

		self.gatherFeatures()

	def gatherFeatures(self):
		#variable initialization
		self.LiRatioDict = {}
		self.nonLiRatioDict = {}

		self.gatherRatios(self.LiRatioDict, self.li_atomList)
		self.gatherRatios(self.nonLiRatioDict, self.nonLi_atomList)

		self.getCompoundNum()

		self.exportRatio()

	def exportRatio(self):
		def writeRatio(compound, writer):
			outputDict = {"Li_atom": compound, "nonLi_atom": self.LiMapping[compound]}
			# outputDict["liAtomTotal"] = self.LiRatioDict[compound]
			outputDict["compoundTotal"] = self.compoundNum[compound]
			writer.writerow(outputDict)

		outputFile = os.path.join(self.DIR, OUTPUT_FILE)
		csvfile = open(outputFile, 'w')
		fieldnames = ["Li_atom", "nonLi_atom", "compoundTotal"]#"liAtomTotal", "nonLiAtomTotal"]
		writer = csv.DictWriter(csvfile, fieldnames = fieldnames)
		writer.writeheader()
		for compound in self.li_atomList:
			writeRatio(compound, writer)

	def getCompoundNum(self):
		def getFormulaList(compound):
			return pg.Composition(compound).formula.split()

		def getNum(formulaList, Li_compound, nonLi_compound):
			for atom in formulaList:
				if getLiRatio.getAtomType.sub('', str(atom)) == "Li":
					LiNum = getLiRatio.getAtomNum.sub('', str(atom))
					break
			return (self.LiRatioDict[Li_compound] - int(LiNum))/self.nonLiRatioDict[nonLi_compound]

		self.compoundNum = {}
		for i in range(len(self.li_atomList)):
			Li_compound = self.li_atomList[i]
			nonLi_compound = self.nonLi_atomList[i]

			formulaList = getFormulaList(Li_compound)

			self.compoundNum[Li_compound] = getNum(formulaList, Li_compound, nonLi_compound)

	def gatherRatios(self, ratioDict, currAtomList):
		def getRatio(formulaList):
			total = 0
			LiNum = 0
			for atom in formulaList:
				total += int(getLiRatio.getAtomNum.sub('', str(atom)))
				if getLiRatio.getAtomType.sub('', str(atom)) == "Li":
					LiNum = getLiRatio.getAtomNum.sub('', str(atom))
			return total

		for compound in currAtomList:
			comp = pg.Composition(compound)
			formulaList = comp.formula.split()

			ratioDict[compound] = getRatio(formulaList)

def preInit():
	# FILEPATH = "../Computed CSV/Energy Computed CSV/reduced_coalesced_computed_File.csv"
	normList = [2, 3, 5, 7]
	gather = gm.extractNorm(FILEPATH, normList, PARTIAL_PATH + "_LI.csv", PARTIAL_PATH + "_nonLi.csv")
	gather.getAtomList()
	return gather.Li_atomList, gather.nonLi_atomList, gather.LiMapping

def main():
	li_atomList, nonLi_atomList, LiMapping = preInit()
	getLiRatio(li_atomList, nonLi_atomList, LiMapping)

if __name__ == "__main__":
	main()


import random
import numpy as np
import torch

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from scipy.ndimage.filters import uniform_filter1d

from .ManagerFeatures import state_to_features
from .ManagerRewards import *

ACTIONS_IDX = {'LEFT':0, 'RIGHT':1, 'UP':2, 'DOWN':3, 'WAIT':4, 'BOMB':5}

def generate_eps_greedy_policy(network):
    return np.linspace(network.epsilon_begin, network.epsilon_end, network.training_episodes)

def add_experience(self, old_game_state, self_action, new_game_state, events):
    old_features = state_to_features(old_game_state)
    if old_features is not None:
        if new_game_state is None:
            new_features = old_features
        else:
            new_features = state_to_features(new_game_state)
        reward = reward_from_events(self, events)
        reward += rewards_from_own_events(self, old_game_state, self_action, new_game_state, events)

        action_idx = ACTIONS_IDX[self_action]
        action = torch.zeros(6)
        action[action_idx] = 1

        self.experience_buffer.append((old_features, action, reward, new_features))
        number_of_elements_in_buffer = len(self.experience_buffer)
        if number_of_elements_in_buffer > self.network.buffer_size:
            self.experience_buffer.popleft()

def update_network(self):
    '''
    network: the network that gets updated
    experience_buffer: the collected experiences, list of game_episodes
    '''
    network = self.network
    experience_buffer = self.experience_buffer

    #randomly choose batch out of the experience buffer
    number_of_elements_in_buffer = len(experience_buffer)
    batch_size = min(number_of_elements_in_buffer, network.batch_size)

    random_i = [ random.randrange(number_of_elements_in_buffer) for _ in range(batch_size)]

    #compute for each experience in the batch
    # - the Ys using n-step TD Q-learning
    # - the current guess for the Q function
    sub_batch = []
    Y = []
    for i in random_i:
        random_experience = experience_buffer[i]
        sub_batch.append(random_experience)

    for b in sub_batch:
        old_state = b[0]
        action = b[1]
        reward = b[2]
        new_state = b[3]

        y = reward
        if new_state is not None:
            y += network.gamma * torch.max(network(new_state))

        Y.append(y)

    Y = torch.tensor(Y)

    #Qs
    states = torch.cat(tuple(b[0] for b in sub_batch))  #put all states of the sub_batch in one batch
    q_values = network(states)
    actions = torch.cat([b[1].unsqueeze(0) for b in sub_batch])
    Q = torch.sum(q_values*actions, dim=1)

    loss = network.loss_function(Q, Y)
    network.optimizer.zero_grad()
    loss.backward()
    network.optimizer.step()

def save_parameters(self, string):
    torch.save(self.network.state_dict(), f"network_parameters/{string}.pt")

    #plot scores
    y = self.game_score_arr
    y = uniform_filter1d(y, 10, mode="nearest", output="float")
    x = range(len(y))
    fig, ax = plt.subplots()
    ax.set_title('score')
    ax.set_xlabel('episode')
    ax.set_ylabel('total points')
    ax.plot(x,y, marker='o', markersize=3, linewidth=1)
    plt.savefig('network_parameters/training_progress.png')

def get_score(events):
    true_game_rewards = {
        e.COIN_COLLECTED: 1,
        e.KILLED_OPPONENT: 5,
    }
    score = 0
    for event in events:
        if event in true_game_rewards:
            score += true_game_rewards[event]
    return score

def track_game_score(self):
    self.game_score_arr.append(self.game_score)
    self.game_score = 0


import argparse

def read_file():
    # initialize
    parser = argparse.ArgumentParser(description='This script is ...')
    parser.add_argument("string", type=str)
    args = parser.parse_args()
    filename = args.string
    print(filename + " is reading...")
    path = "dataset/" + filename
    return path

def read_files():
    # initialize
    parser = argparse.ArgumentParser(description='This script is ...')
    parser.add_argument("string", type=str)
    args = parser.parse_args()
    filenames = args.string
    filename_arr = filenames.split(",")
    print(filename_arr)
    print("these files are reading...")
    path_arr = []
    for name in filename_arr:
        path_arr.append("dataset/" + name)
    return path_arr


from app import db
from auth import authorize
from auth.AuthConstants import DATA_EDITOR_ROLE, EC_LEADERSHIP_ROLE
from util import RequestBody
from schemas import TallySheetVersionCE201Schema, TallySheetVersionSchema
from orm.entities.Submission import TallySheet
from orm.entities.SubmissionVersion.TallySheetVersion import TallySheetVersionCE201
from exception import NotFoundException

@authorize(required_roles=[DATA_EDITOR_ROLE, EC_LEADERSHIP_ROLE])
def get_by_id(tallySheetId, tallySheetVersionId):
    result = TallySheetVersionCE201.get_by_id(
        tallySheetId=tallySheetId,
        tallySheetVersionId=tallySheetVersionId
    )

    return TallySheetVersionCE201Schema().dump(result).data

@authorize(required_roles=[DATA_EDITOR_ROLE, EC_LEADERSHIP_ROLE])
def get_all(tallySheetId):
    tallySheet = TallySheet.get_by_id(tallySheetId=tallySheetId)
    if tallySheet is None:
        raise NotFoundException("Tally sheet not found. (tallySheetId=%d)" % tallySheetId)

    result = TallySheetVersionCE201.get_all(
        tallySheetId=tallySheetId
    )

    return TallySheetVersionCE201Schema(many=True).dump(result).data

@authorize(required_roles=[DATA_EDITOR_ROLE])
def create(tallySheetId, body):
    request_body = RequestBody(body)
    tallySheetVersion = TallySheetVersionCE201.create(
        tallySheetId=tallySheetId
    )

    tally_sheet_content = request_body.get("content")
    if tally_sheet_content is not None:
        for party_count_body in tally_sheet_content:
            party_count_body = RequestBody(party_count_body)
            tallySheetVersionRow = tallySheetVersion.add_row(
                areaId=party_count_body.get("areaId"),
                ballotsIssued=party_count_body.get("ballotsIssued"),
                ballotsReceived=party_count_body.get("ballotsReceived"),
                ballotsSpoilt=party_count_body.get("ballotsSpoilt"),
                ballotsUnused=party_count_body.get("ballotsUnused"),
                ordinaryBallotCountFromBoxCount=party_count_body.get("ordinaryBallotCountFromBoxCount"),
                tenderedBallotCountFromBoxCount=party_count_body.get("tenderedBallotCountFromBoxCount"),
                ordinaryBallotCountFromBallotPaperAccount=party_count_body.get(
                    "ordinaryBallotCountFromBallotPaperAccount"),
                tenderedBallotCountFromBallotPaperAccount=party_count_body.get(
                    "tenderedBallotCountFromBallotPaperAccount")
            )

            for issued_ballot_box_id in party_count_body.get("ballotBoxesIssued"):
                tallySheetVersionRow.add_issued_ballot_box(issued_ballot_box_id)

            for received_ballot_box_id in party_count_body.get("ballotBoxesReceived"):
                tallySheetVersionRow.add_received_ballot_box(received_ballot_box_id)

    db.session.commit()

    return TallySheetVersionSchema().dump(tallySheetVersion).data


# --------------
# Code starts here

# Create the lists
class_1 = ['Geoffrey Hinton','Andrew Ng','Sebastian Raschka','Yoshua Bengio' ]
class_2 = ['Hilary Mason','Carla Gentry','Corinna Cortes']

# Concatenate both the strings
new_class = (class_1 + class_2)
print (new_class)

# Append the list
new_class.append('Peter Warden')

# Print updated list
print(new_class)

# Remove the element from the list
new_class.remove('Carla Gentry')

# Print the list
print(new_class)

# Code ends here

# --------------
# Code starts here0
courses={"Math":65 ,"English":70,"History":80,"French":70,"Science":60}
#printing values
for x in courses.values():
    print(x)

total=0
#finding total
for x in courses.values():
    total=total+x

print(total)

#finding percentage

percentage=(total)/500*100
print(percentage)

# Code ends here

# --------------
# Code starts here
mathematics={"Geoffrey Hinton":78,"Andrew Ng":95,"Sebastian Raschka":65,
"Yoshua Benjio":50,"Hilary Mason":70,"Corinna Cortes":66,"Peter Warden":75}

for x,y in mathematics.items():
    print(x,y)

topper=max(mathematics,key=mathematics.get)
print(topper)

# Code ends here

# --------------
# Given string
topper = 'andrew ng'

# Code starts here

#split the name
first_name,last_name=topper.split(" ")

#conactenate the name
full_name=last_name+" "+first_name
print(full_name)

#convert to upper
certificate_name=full_name.upper()
print(certificate_name)

# Code ends here


def foo(weights):
    """
    This function returns 0 if it is impossible to split items into two piles,
    such that weight of one pile is not larger than 2 weights of the other, and
    list of items' weight from the first pile otherwise.
    [100, 20, 20] -> 0
    [10, 10, 11] -> [10, 10]
    """
    W = sum(weights)
    low = W/3
    high = 2*low
    res = []
    weight = 0
    for item in weights:
        if weight + item <= high:
            weight += item
            res.append(item)
            if weight >= low:
                return res
    else:
        return 0

if __name__ == '__main__':
    input_data = list(map(int, input('Enter weights of all items separated with space: ').split()))
    print(foo(input_data))


#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import os
import operator
import tempfile
from unittest import TestCase

import pytest
import torch
from pytorch_lightning import LightningModule
from torch import nn
from test.pytorch.utils._train_torch_lightning import create_data_loader, data_transform
import torchmetrics

from bigdl.nano.pytorch import Trainer
from bigdl.nano.pytorch import InferenceOptimizer
from bigdl.nano.pytorch.vision.models import vision
from bigdl.nano.utils.log4Error import invalidOperationError
from bigdl.nano.utils.util import compare_version

batch_size = 256
num_workers = 0
data_dir = os.path.join(os.path.dirname(__file__), "data")

class ResNet18(nn.Module):
    def __init__(self, num_classes, pretrained=True, include_top=False, freeze=True):
        super().__init__()
        backbone = vision.resnet18(pretrained=pretrained, include_top=include_top, freeze=freeze)
        output_size = backbone.get_output_size()
        head = nn.Linear(output_size, num_classes)
        self.model = nn.Sequential(backbone, head)

    def forward(self, x):
        return self.model(x)

class LitResNet18(LightningModule):
    def __init__(self, num_classes, pretrained=True, include_top=False, freeze=True):
        super().__init__()
        backbone = vision.resnet18(pretrained=pretrained, include_top=include_top, freeze=freeze)
        output_size = backbone.get_output_size()
        head = nn.Linear(output_size, num_classes)
        self.classify = nn.Sequential(backbone, head)

    def forward(self, *args):
        return self.classify(args[0])

class ModelCannotCopy(ResNet18):
    def __deepcopy__(self, obj):
        invalidOperationError(False, "This model cannot be deepcopy")

class TestTrainer(TestCase):
    model = ResNet18(10, pretrained=False, include_top=False, freeze=True)
    loss = nn.CrossEntropyLoss()
    optimizer = torch.optim.Adam(model.parameters(), lr=0.01)
    train_loader = create_data_loader(data_dir, batch_size, num_workers, data_transform)
    user_defined_pl_model = LitResNet18(10)

    def test_quantize_inc_ptq_compiled(self):
        # Test if a Lightning Module compiled by nano works
        train_loader_iter = iter(self.train_loader)
        trainer = Trainer(max_epochs=1)
        pl_model = Trainer.compile(self.model, self.loss, self.optimizer)
        x = next(train_loader_iter)[0]

        # Case 1: Default
        qmodel = InferenceOptimizer.quantize(pl_model,
                                             calib_data=self.train_loader)
        assert qmodel
        out = qmodel(x)
        assert out.shape == torch.Size([256, 10])

        # Case 2: Override by arguments
        qmodel = InferenceOptimizer.quantize(pl_model,
                                             calib_data=self.train_loader,
                                             metric=torchmetrics.F1Score('multiclass', num_classes=10),
                                             approach='static',
                                             tuning_strategy='basic',
                                             accuracy_criterion={'relative': 0.99,
                                                                 'higher_is_better': True})

        assert qmodel
        out = qmodel(x)
        assert out.shape == torch.Size([256, 10])

        # Case 3: Dynamic quantization
        qmodel = InferenceOptimizer.quantize(pl_model, approach='dynamic')
        assert qmodel
        out = qmodel(x)
        assert out.shape == torch.Size([256, 10])

        # Case 4: Invalid approach
        invalid_approach = 'qat'
        with pytest.raises(RuntimeError, match="Approach should be 'static' or 'dynamic', "
                                               "{} is invalid.".format(invalid_approach)):
            InferenceOptimizer.quantize(pl_model, approach=invalid_approach)

        # Case 5: Test if registered metric can be fetched successfully
        qmodel = InferenceOptimizer.quantize(pl_model,
                                             calib_data=self.train_loader,
                                             metric=torchmetrics.F1Score('multiclass', num_classes=10),
                                             accuracy_criterion={'relative': 0.99,
                                                                'higher_is_better': True})
        assert qmodel
        out = qmodel(x)
        assert out.shape == torch.Size([256, 10])

        trainer.validate(qmodel, self.train_loader)
        trainer.test(qmodel, self.train_loader)
        trainer.predict(qmodel, self.train_loader)

        # save and load
        with tempfile.TemporaryDirectory() as tmp_dir_name:
            InferenceOptimizer.save(qmodel, tmp_dir_name)
            loaded_qmodel = InferenceOptimizer.load(tmp_dir_name, pl_model)
            assert loaded_qmodel
            out = loaded_qmodel(x)
            assert out.shape == torch.Size([256, 10])

    def test_quantize_inc_ptq_customized(self):
        # Test if a Lightning Module not compiled by nano works
        train_loader_iter = iter(self.train_loader)
        x = next(train_loader_iter)[0]
        trainer = Trainer(max_epochs=1)

        qmodel = InferenceOptimizer.quantize(self.user_defined_pl_model,
                                             calib_data=self.train_loader)
        assert qmodel
        out = qmodel(x)
        assert out.shape == torch.Size([256, 10])

        # save and load
        with tempfile.TemporaryDirectory() as tmp_dir_name:
            InferenceOptimizer.save(qmodel, tmp_dir_name)
            loaded_qmodel = InferenceOptimizer.load(tmp_dir_name, self.user_defined_pl_model)
            assert loaded_qmodel
            out = loaded_qmodel(x)
            assert out.shape == torch.Size([256, 10])

    def test_quantize_inc_ptq_with_tensor(self):
        train_loader_iter = iter(self.train_loader)
        trainer = Trainer(max_epochs=1)
        pl_model = Trainer.compile(self.model, self.loss, self.optimizer)
        x = next(train_loader_iter)[0]

        # Case 1: quantize with single tensor
        qmodel = InferenceOptimizer.quantize(pl_model,
                                             calib_data=x)
        assert qmodel
        out = qmodel(x)
        assert out.shape == torch.Size([256, 10])

        # Case 2: quantize with tensor tuple
        qmodel = InferenceOptimizer.quantize(pl_model,
                                             # fake a label
                                             calib_data=(x, torch.ones(1)))
        assert qmodel
        out = qmodel(x)
        assert out.shape == torch.Size([256, 10])

    def test_quantize_inc_ptq_compiled_context_manager(self):
        # Test if a Lightning Module compiled by nano works
        train_loader_iter = iter(self.train_loader)
        trainer = Trainer(max_epochs=1)
        pl_model = Trainer.compile(self.model, self.loss, self.optimizer)
        x = next(train_loader_iter)[0]

        qmodel = InferenceOptimizer.quantize(pl_model,
                                             calib_data=self.train_loader,
                                             thread_num=2)
        assert qmodel

        with InferenceOptimizer.get_context(qmodel):
            assert torch.get_num_threads() == 2
            out = qmodel(x)
        assert out.shape == torch.Size([256, 10])

        with tempfile.TemporaryDirectory() as tmp_dir_name:
            InferenceOptimizer.save(qmodel, tmp_dir_name)
            model = InferenceOptimizer.load(tmp_dir_name, pl_model)

        with InferenceOptimizer.get_context(model):
            assert torch.get_num_threads() == 2
            out = model(x)
        assert out.shape == torch.Size([256, 10])

    def test_quantize_inc_ptq_compiled_additional_attributes(self):
        # Test if a Lightning Module compiled by nano works
        train_loader_iter = iter(self.train_loader)
        pl_model = Trainer.compile(self.model, self.loss, self.optimizer)
        # patch a attribute
        pl_model.channels = 3
        def hello():
            print("hello world!")
        # patch a function
        pl_model.hello = hello
        x = next(train_loader_iter)[0]

        qmodel = InferenceOptimizer.quantize(pl_model,
                                             calib_data=self.train_loader,
                                             thread_num=2)
        assert qmodel
        assert qmodel.channels == 3
        qmodel.hello()

        with InferenceOptimizer.get_context(qmodel):
            assert torch.get_num_threads() == 2
            out = qmodel(x)
        assert out.shape == torch.Size([256, 10])

    # This UT will fail with INC < 2.0
    @pytest.mark.skipif(compare_version("neural_compressor", operator.lt, "2.0"), reason="")
    def test_ipex_int8_quantize_with_model_cannot_deepcopy(self):
        model = ModelCannotCopy(num_classes=10)
        InferenceOptimizer.quantize(model,
                                    calib_data=self.train_loader,
                                    method="ipex",
                                    # inplace=False is setting to disable back up ipex quantization
                                    inplace=False)

    # INC 1.14 and 2.0 doesn't supprot quantizing pytorch-lightning module,
    # but we have some workaround for pl models returned by our `Trainer.compile`
    def test_quantize_with_pl_model(self):
        trainer = Trainer(max_epochs=1)
        pl_model = Trainer.compile(self.model, self.loss, self.optimizer)
        trainer.fit(pl_model, self.train_loader)
        InferenceOptimizer.quantize(pl_model,
                                    calib_data=self.train_loader)


#!/usr/bin/env python3
# -*- encoding: utf-8 -*-

#  Licensed to the Apache Software Foundation (ASF) under one
#  or more contributor license agreements.  See the NOTICE file
#  distributed with this work for additional information
#  regarding copyright ownership.  The ASF licenses this file
#  to you under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance
#  with the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing,
#  software distributed under the License is distributed on an
#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#  KIND, either express or implied.  See the License for the
#  specific language governing permissions and limitations
#  under the License.

"""test_kill_stmgr.py"""
import logging

from contextlib import contextmanager
from subprocess import run, Popen
from time import sleep

from . import test_template

@contextmanager
def bound_subprocess(popen: Popen) -> None:
  with popen:
    logging.debug(f"starting {popen!r}")
    try:
      yield
    finally:
      logging.debug(f"killing {popen!r}")
      popen.kill()
      popen.communicate()

def run_explorer(*args):
  cmd = ["heron-explorer", *args]
  logging.debug(f"running command {cmd!r}")
  run(cmd, check=True, timeout=5)
  logging.debug(f"finished command {cmd!r}")

class TestExplorer(test_template.TestTemplate):

  def execute_test_case(self):
    from getpass import getuser
    cre = f"{self.params['cluster']}/{getuser()}/default"
    topology = self.params["topologyName"]
    # heron-explorer depens on heron-tracker, so start an instance as it is not started
    # by heron-cli when running against the local "cluster"
    with bound_subprocess(Popen(["heron-tracker"])):
      sleep(2)
      run_explorer("clusters")

      cre_parts = cre.split("/")
      for i in range(len(cre_parts)):
        run_explorer("topologies", "/".join(cre_parts[:i+1]))

      run_explorer("logical-plan", cre, topology)
      run_explorer("logical-plan", cre, topology, "--component-type=bolts")
      run_explorer("logical-plan", cre, topology, "--component-type=spouts")

      run_explorer("physical-plan", "containers", cre, topology)
      run_explorer("physical-plan", "containers", cre, topology, "--id=1")
      run_explorer("physical-plan", "metrics", cre, topology)