from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import numpy as np
import tensorflow as tf
from src.common_ops import create_weight, batch_norm, batch_norm_with_mask, global_avg_pool, conv_op, pool_op
from src.utils import count_model_params, get_train_ops, get_C, get_strides
from src.cifar10.models import Model

class GeneralChild(Model):
    def __init__(self,
                 images,
                 labels,
                 cutout_size=None,
                 whole_channels=False,
                 fixed_arc=None,
                 out_filters_scale=1,
                 num_layers=2,
                 num_branches=6,
                 out_filters=24,
                 keep_prob=1.0,
                 batch_size=32,
                 clip_mode=None,
                 grad_bound=None,
                 l2_reg=1e-4,
                 lr_init=0.1,
                 lr_dec_start=0,
                 lr_dec_every=10000,
                 lr_dec_rate=0.1,
                 lr_cosine=False,
                 lr_max=None,
                 lr_min=None,
                 lr_T_0=None,
                 lr_T_mul=None,
                 optim_algo=None,
                 sync_replicas=False,
                 num_aggregate=None,
                 num_replicas=None,
                 data_format="NHWC",
                 name="child",
                 mode="subgraph",
                 *args,
                 **kwargs
                 ):

        super(self.__class__, self).__init__(
            images,
            labels,
            cutout_size=cutout_size,
            batch_size=batch_size,
            clip_mode=clip_mode,
            grad_bound=grad_bound,
            l2_reg=l2_reg,
            lr_init=lr_init,
            lr_dec_start=lr_dec_start,
            lr_dec_every=lr_dec_every,
            lr_dec_rate=lr_dec_rate,
            keep_prob=keep_prob,
            optim_algo=optim_algo,
            sync_replicas=sync_replicas,
            num_aggregate=num_aggregate,
            num_replicas=num_replicas,
            data_format=data_format,
            name=name)

        self.whole_channels = whole_channels
        self.lr_cosine = lr_cosine
        self.lr_max = lr_max
        self.lr_min = lr_min
        self.lr_T_0 = lr_T_0
        self.lr_T_mul = lr_T_mul
        self.out_filters = out_filters * out_filters_scale
        self.num_layers = num_layers
        self.mode = mode

        self.num_branches = num_branches
        self.fixed_arc = fixed_arc
        self.out_filters_scale = out_filters_scale

        pool_distance = self.num_layers // 3
        self.pool_layers = [pool_distance - 1, 2 * pool_distance - 1]

    def _factorized_reduction(self, x, out_filters, stride, is_training):
        """Reduces the shape of x without information loss due to striding."""
        assert out_filters % 2 == 0, (
            "Need even number of filters when using this factorized reduction.")
        if stride == 1:
            with tf.variable_scope("path_conv"):
                inp_c = get_C(x, self.data_format)
                w = create_weight("w", [1, 1, inp_c, out_filters])
                x = tf.nn.conv2d(x, w, [1, 1, 1, 1], "SAME",
                                 data_format=self.data_format)
                x = batch_norm(x, is_training, data_format=self.data_format)
                return x

        stride_spec = get_strides(stride, self.data_format)
        # Skip path 1
        path1 = tf.nn.avg_pool(
            x, [1, 1, 1, 1], stride_spec, "VALID", data_format=self.data_format)
        with tf.variable_scope("path1_conv"):
            inp_c = get_C(path1, self.data_format)
            w = create_weight("w", [1, 1, inp_c, out_filters // 2])
            path1 = tf.nn.conv2d(path1, w, [1, 1, 1, 1], "SAME",
                                 data_format=self.data_format)

        # Skip path 2
        # First pad with 0"s on the right and bottom, then shift the filter to
        # include those 0"s that were added.
        if self.data_format == "NHWC":
            pad_arr = [[0, 0], [0, 1], [0, 1], [0, 0]]
            path2 = tf.pad(x, pad_arr)[:, 1:, 1:, :]
            concat_axis = 3
        else:
            pad_arr = [[0, 0], [0, 0], [0, 1], [0, 1]]
            path2 = tf.pad(x, pad_arr)[:, :, 1:, 1:]
            concat_axis = 1

        path2 = tf.nn.avg_pool(
            path2, [1, 1, 1, 1], stride_spec, "VALID", data_format=self.data_format)
        with tf.variable_scope("path2_conv"):
            inp_c = get_C(path2, self.data_format)
            w = create_weight("w", [1, 1, inp_c, out_filters // 2])
            path2 = tf.nn.conv2d(path2, w, [1, 1, 1, 1], "SAME",
                                 data_format=self.data_format)

        # Concat and apply BN
        final_path = tf.concat(values=[path1, path2], axis=concat_axis)
        final_path = batch_norm(final_path, is_training,
                                data_format=self.data_format)

        return final_path

    def _model(self, images, is_training, reuse=False):
        '''Build model'''
        with tf.variable_scope(self.name, reuse=reuse):
            layers = []

            out_filters = self.out_filters
            with tf.variable_scope("stem_conv"):
                w = create_weight("w", [3, 3, 3, out_filters])
                x = tf.nn.conv2d(
                    images, w, [1, 1, 1, 1], "SAME", data_format=self.data_format)
                x = batch_norm(x, is_training, data_format=self.data_format)
                layers.append(x)

            def add_fixed_pooling_layer(layer_id, layers, out_filters, is_training):
                '''Add a fixed pooling layer every four layers'''
                out_filters *= 2
                with tf.variable_scope("pool_at_{0}".format(layer_id)):
                    pooled_layers = []
                    for i, layer in enumerate(layers):
                        with tf.variable_scope("from_{0}".format(i)):
                            x = self._factorized_reduction(
                                layer, out_filters, 2, is_training)
                        pooled_layers.append(x)
                    return pooled_layers, out_filters

            def post_process_out(out, optional_inputs):
                '''Form skip connection and perform batch norm'''
                with tf.variable_scope("skip"):
                    inputs = layers[-1]
                    if self.data_format == "NHWC":
                        inp_h = inputs.get_shape()[1].value
                        inp_w = inputs.get_shape()[2].value
                        inp_c = inputs.get_shape()[3].value
                        out.set_shape([None, inp_h, inp_w, out_filters])
                    elif self.data_format == "NCHW":
                        inp_c = inputs.get_shape()[1].value
                        inp_h = inputs.get_shape()[2].value
                        inp_w = inputs.get_shape()[3].value
                        out.set_shape([None, out_filters, inp_h, inp_w])
                    optional_inputs.append(out)
                    pout = tf.add_n(optional_inputs)
                    out = batch_norm(pout, is_training,
                                     data_format=self.data_format)
                layers.append(out)
                return out

            global layer_id
            layer_id = -1

            def get_layer_id():
                global layer_id
                layer_id += 1
                return 'layer_' + str(layer_id)

            def conv3(inputs):
                # res_layers is pre_layers that are chosen to form skip connection
                # layers[-1] is always the latest input
                with tf.variable_scope(get_layer_id()):
                    with tf.variable_scope('branch_0'):
                        out = conv_op(
                            inputs[0][0], 3, is_training, out_filters, out_filters, self.data_format, start_idx=None)
                    out = post_process_out(out, inputs[1])
                return out

            def conv3_sep(inputs):
                with tf.variable_scope(get_layer_id()):
                    with tf.variable_scope('branch_1'):
                        out = conv_op(
                            inputs[0][0], 3, is_training, out_filters, out_filters, self.data_format, start_idx=None, separable=True)
                    out = post_process_out(out, inputs[1])
                return out

            def conv5(inputs):
                with tf.variable_scope(get_layer_id()):
                    with tf.variable_scope('branch_2'):
                        out = conv_op(
                            inputs[0][0], 5, is_training, out_filters, out_filters, self.data_format, start_idx=None)
                    out = post_process_out(out, inputs[1])
                return out

            def conv5_sep(inputs):
                with tf.variable_scope(get_layer_id()):
                    with tf.variable_scope('branch_3'):
                        out = conv_op(
                            inputs[0][0], 5, is_training, out_filters, out_filters, self.data_format, start_idx=None, separable=True)
                    out = post_process_out(out, inputs[1])
                return out

            def avg_pool(inputs):
                with tf.variable_scope(get_layer_id()):
                    with tf.variable_scope('branch_4'):
                        out = pool_op(
                            inputs[0][0], is_training, out_filters, out_filters, "avg", self.data_format, start_idx=None)
                    out = post_process_out(out, inputs[1])
                return out

            def max_pool(inputs):
                with tf.variable_scope(get_layer_id()):
                    with tf.variable_scope('branch_5'):
                        out = pool_op(
                            inputs[0][0], is_training, out_filters, out_filters, "max", self.data_format, start_idx=None)
                    out = post_process_out(out, inputs[1])
                return out

            """@nni.mutable_layers(
            {
                layer_choice: [conv3(), conv3_sep(), conv5(), conv5_sep(), avg_pool(), max_pool()],
                fixed_inputs:[x],
                optional_inputs: [],
                optional_input_size: 1,
                layer_output: layer_0_out
            },
            {
                layer_choice: [conv3(), conv3_sep(), conv5(), conv5_sep(), avg_pool(), max_pool()],
                fixed_inputs:[layer_0_out],
                optional_inputs: [layer_0_out],
                optional_input_size: 1,
                layer_output: layer_1_out
            },
            {
                layer_choice: [conv3(), conv3_sep(), conv5(), conv5_sep(), avg_pool(), max_pool()],
                fixed_inputs:[layer_1_out],
                optional_inputs: [layer_0_out, layer_1_out],
                optional_input_size: 1,
                layer_output: layer_2_out
            },
            {
                layer_choice: [conv3(), conv3_sep(), conv5(), conv5_sep(), avg_pool(), max_pool()],
                fixed_inputs:[layer_2_out],
                optional_inputs: [layer_0_out, layer_1_out, layer_2_out],
                optional_input_size: 1,
                layer_output: layer_3_out
            }
            )"""
            layers, out_filters = add_fixed_pooling_layer(
                3, layers, out_filters, is_training)
            layer_0_out, layer_1_out, layer_2_out, layer_3_out = layers[-4:]
            """@nni.mutable_layers(
            {
                layer_choice: [conv3(), conv3_sep(), conv5(), conv5_sep(), avg_pool(), max_pool()],
                fixed_inputs: [layer_3_out],
                optional_inputs: [layer_0_out, layer_1_out, layer_2_out, layer_3_out],
                optional_input_size: 1,
                layer_output: layer_4_out
            },
            {
                layer_choice: [conv3(), conv3_sep(), conv5(), conv5_sep(), avg_pool(), max_pool()],
                fixed_inputs: [layer_4_out],
                optional_inputs: [layer_0_out, layer_1_out, layer_2_out, layer_3_out, layer_4_out],
                optional_input_size: 1,
                layer_output: layer_5_out
            },
            {
                layer_choice: [conv3(), conv3_sep(), conv5(), conv5_sep(), avg_pool(), max_pool()],
                fixed_inputs: [layer_5_out],
                optional_inputs: [layer_0_out, layer_1_out, layer_2_out, layer_3_out, layer_4_out, layer_5_out],
                optional_input_size: 1,
                layer_output: layer_6_out
            },
            {
                layer_choice: [conv3(), conv3_sep(), conv5(), conv5_sep(), avg_pool(), max_pool()],
                fixed_inputs: [layer_6_out],
                optional_inputs: [layer_0_out, layer_1_out, layer_2_out, layer_3_out, layer_4_out, layer_5_out, layer_6_out],
                optional_input_size: 1,
                layer_output: layer_7_out
            }
            )"""
            layers, out_filters = add_fixed_pooling_layer(
                7, layers, out_filters, is_training)
            layer_0_out, layer_1_out, layer_2_out, layer_3_out, layer_4_out, layer_5_out, layer_6_out, layer_7_out = layers[
                -8:]
            """@nni.mutable_layers(
            {
                layer_choice: [conv3(), conv3_sep(), conv5(), conv5_sep(), avg_pool(), max_pool()],
                fixed_inputs: [layer_7_out],
                optional_inputs: [layer_0_out, layer_1_out, layer_2_out, layer_3_out, layer_4_out, layer_5_out, layer_6_out, layer_7_out],
                optional_input_size: 1,
                layer_output: layer_8_out
            },
            {
                layer_choice: [conv3(), conv3_sep(), conv5(), conv5_sep(), avg_pool(), max_pool()],
                fixed_inputs: [layer_8_out],
                optional_inputs: [layer_0_out, layer_1_out, layer_2_out, layer_3_out, layer_4_out, layer_5_out, layer_6_out, layer_7_out, layer_8_out],
                optional_input_size: 1,
                layer_output: layer_9_out
            },
            {
                layer_choice: [conv3(), conv3_sep(), conv5(), conv5_sep(), avg_pool(), max_pool()],
                fixed_inputs: [layer_9_out],
                optional_inputs: [layer_0_out, layer_1_out, layer_2_out, layer_3_out, layer_4_out, layer_5_out, layer_6_out, layer_7_out, layer_8_out, layer_9_out],
                optional_input_size: 1,
                layer_output: layer_10_out
            },
            {
                layer_choice: [conv3(), conv3_sep(), conv5(), conv5_sep(), avg_pool(), max_pool()],
                fixed_inputs:[layer_10_out],
                optional_inputs: [layer_0_out, layer_1_out, layer_2_out, layer_3_out, layer_4_out, layer_5_out, layer_6_out, layer_7_out, layer_8_out, layer_9_out, layer_10_out],
                optional_input_size: 1,
                layer_output: layer_11_out
            }
            )"""

            x = global_avg_pool(layer_11_out, data_format=self.data_format)
            if is_training:
                x = tf.nn.dropout(x, self.keep_prob)
            with tf.variable_scope("fc"):
                if self.data_format == "NHWC":
                    inp_c = x.get_shape()[3].value
                elif self.data_format == "NCHW":
                    inp_c = x.get_shape()[1].value
                else:
                    raise ValueError(
                        "Unknown data_format {0}".format(self.data_format))
                w = create_weight("w", [inp_c, 10])
                x = tf.matmul(x, w)
        return x

    # override
    def _build_train(self):
        print("-" * 80)
        print("Build train graph")
        logits = self._model(self.x_train, is_training=True)
        log_probs = tf.nn.sparse_softmax_cross_entropy_with_logits(
            logits=logits, labels=self.y_train)
        self.loss = tf.reduce_mean(log_probs)

        self.train_preds = tf.argmax(logits, axis=1)
        self.train_preds = tf.to_int32(self.train_preds)
        self.train_acc = tf.equal(self.train_preds, self.y_train)
        self.train_acc = tf.to_int32(self.train_acc)
        self.train_acc = tf.reduce_sum(self.train_acc)

        tf_variables = [var
                        for var in tf.trainable_variables() if var.name.startswith(self.name)]
        self.num_vars = count_model_params(tf_variables)
        print("Model has {} params".format(self.num_vars))

        self.global_step = tf.Variable(
            0, dtype=tf.int32, trainable=False, name="global_step")

        self.train_op, self.lr, self.grad_norm, self.optimizer = get_train_ops(
            self.loss,
            tf_variables,
            self.global_step,
            clip_mode=self.clip_mode,
            grad_bound=self.grad_bound,
            l2_reg=self.l2_reg,
            lr_init=self.lr_init,
            lr_dec_start=self.lr_dec_start,
            lr_dec_every=self.lr_dec_every,
            lr_dec_rate=self.lr_dec_rate,
            lr_cosine=self.lr_cosine,
            lr_max=self.lr_max,
            lr_min=self.lr_min,
            lr_T_0=self.lr_T_0,
            lr_T_mul=self.lr_T_mul,
            num_train_batches=self.num_train_batches,
            optim_algo=self.optim_algo,
            sync_replicas=False,
            num_aggregate=self.num_aggregate,
            num_replicas=self.num_replicas)

    # override
    def _build_valid(self):
        if self.x_valid is not None:
            print("-" * 80)
            print("Build valid graph")
            logits = self._model(self.x_valid, False, reuse=True)
            self.valid_preds = tf.argmax(logits, axis=1)
            self.valid_preds = tf.to_int32(self.valid_preds)
            self.valid_acc = tf.equal(self.valid_preds, self.y_valid)
            self.valid_acc = tf.to_int32(self.valid_acc)
            self.valid_acc = tf.reduce_sum(self.valid_acc)

    # override
    def _build_test(self):
        print("-" * 80)
        print("Build test graph")
        logits = self._model(self.x_test, False, reuse=True)
        self.test_preds = tf.argmax(logits, axis=1)
        self.test_preds = tf.to_int32(self.test_preds)
        self.test_acc = tf.equal(self.test_preds, self.y_test)
        self.test_acc = tf.to_int32(self.test_acc)
        self.test_acc = tf.reduce_sum(self.test_acc)

    def build_model(self):

        self._build_train()
        self._build_valid()
        self._build_test()


"""
相机空间：右手系
"""
import numpy as np
import rtmath
from geometry import vector

TYPE_NORMAL = 1

class CUVNCamera(object):
	def __init__(self, vEye=vector([0, 0, -3, 1]), vAt=vector([0.23, 0, 0, 1]), vUp=vector([0, 1, 0, 0]), fFov=0.5*np.pi, fAspect=1.0, fNear=1.0, fFar=500.0):
		self.m_vEye = vEye  # 相机的位置
		self.m_vAt = vAt  # 注视目标的位置
		self.m_vUp = vUp  # 上向量
		self.m_mViewTrans = None
		self.m_fFov = fFov  # 竖直方向的张开角度
		self.m_fAspect = fAspect
		self.m_fNear = fNear  # 近裁剪平面
		self.m_fFar = fFar  # 远裁剪平面
		self.m_mPerspTrans = None

	def GetViewTrans(self):
		if self.m_mViewTrans is None:
			self.calViewTrans()
		return self.m_mViewTrans

	def calViewTrans(self):
		"""
		观察空间为右手系，这里算出U、N、V轴在世界空间的表示，叉乘顺序为UxV,VxN,NxU
		摄像机指向-N方向
		过程：1.平移  2.空间变换
		"""
		vN = rtmath.normalize(self.m_vEye - self.m_vAt)[:3]

		# 左手系中使用右手定则判断方向，方向相反
		vU = rtmath.normalize(-np.cross(self.m_vUp[:3], vN))
		vV = rtmath.normalize(-np.cross(vN, vU))

		mTemp = np.eye(4)
		mTemp[0, :3] = vU
		mTemp[1, :3] = vV
		mTemp[2, :3] = vN
		mTemp[0, 3] = -np.dot(self.m_vEye[:3], vU)
		mTemp[1, 3] = -np.dot(self.m_vEye[:3], vV)
		mTemp[2, 3] = -np.dot(self.m_vEye[:3], vN)
		self.m_mViewTrans = mTemp

	def GetProjectTrans(self):
		if self.m_mPerspTrans is None:
			self.calProjectTrans()
		return self.m_mPerspTrans

	def calProjectTrans(self):
		"""
		需要考虑Z分量需要取反
		:return:
		"""
		cot = 1. / np.tan(self.m_fFov / 2.)
		zRange = self.m_fFar - self.m_fNear
		mTemp = np.zeros((4, 4))
		mTemp[0, 0] = cot / self.m_fAspect
		mTemp[1, 1] = cot
		mTemp[2, 2] = (self.m_fNear + self.m_fFar) / zRange  # 取反
		mTemp[2, 3] = 2 * self.m_fFar * self.m_fNear / zRange
		mTemp[3, 2] = -1  # 取反
		self.m_mPerspTrans = mTemp

	def SetLookAt(self, vAt):
		self.m_vAt = vAt
		self.calViewTrans()

	def SetEye(self, vEye):
		self.m_vEye = vEye
		self.calViewTrans()

	def SetUp(self, vUp):
		self.m_vUp = vUp
		self.calViewTrans()

	def SetFov(self, fFov):
		self.m_fFov = fFov
		self.calProjectTrans()

	def SetAspect(self, fAspect):
		self.m_fAspect = fAspect
		self.calProjectTrans()

	def SetNear(self, fNear):
		self.m_fNear = fNear
		self.calProjectTrans()

	def SetFar(self, fFar):
		self.m_fFar = fFar
		self.calProjectTrans()

	def GetEye(self):
		return self.m_vEye


import requests
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.select import Select

def collect_proxies():
    driver = webdriver.Chrome(executable_path='chromedriver')
    url = 'https://free-proxy-list.net/'
    driver.get(url)
    proxies = []
    url = 'https://free-proxy-list.net/'
    timeout = 30
    try:
        WebDriverWait(driver, timeout).until(EC.visibility_of_element_located((By.ID, "proxylisttable")))
    except TimeoutException:
        driver.quit()

    selectXpath = '//*[@id="proxylisttable"]/tfoot/tr/th[7]/select'
    select_https = Select(driver.find_element(By.XPATH, selectXpath))

    #Change the filter to only show https proxies
    select_https.select_by_visible_text('yes')

    nextXpath = '//*[@id="proxylisttable_next"]/a'

    proxy_table_body = driver.find_element(By.XPATH, '//*[@id="proxylisttable"]/tbody')
    for pageNum in range(4):
        rows = proxy_table_body.find_elements(By.CSS_SELECTOR, 'tr')
        for tr in rows:
            td = tr.find_elements(By.CSS_SELECTOR, 'td')
            proxy = f'{td[0].text}:{td[1].text}'
            proxies.append(proxy)
        next_Page = driver.find_element(By.XPATH, nextXpath)
        next_Page.click()

    #webdriver.ActionChains(driver).move_to_element(select_https).click(element).perform()
    return proxies


class Solution:
  def specialArray(self, nums: List[int]) -> int:
    N = len(nums)

    if N == 0:
      return -1

    if N == 1:
      return 1 if nums[0] >= 1 else -1

    snums = sorted(nums, reverse=True)

    for i in range(1, N):
      num = snums[i]

      if num != snums[i - 1]:
        if i <= snums[i - 1] and i > num:
          return i

    return i + 1 if i + 1 <= snums[-1] else -1


"""A helper script to download ATT&CK releases in STIX/JSON format."""

import pathlib
from typing import List

import pooch
import typer
from loguru import logger

from mitreattack import release_info

app = typer.Typer(add_completion=False)

def download_stix(stix_version: str, domain: str, download_dir: str, release: str, known_hash: str):
    """Download an ATT&CK STIX release file.

    Parameters
    ----------
    stix_version : str
        Version of STIX to download. Options are "2.0" or "2.1"
    domain : str
        An ATT&CK domain from the following list ["enterprise", "mobile", "ics"]
    download_dir : str
        Directory to download the STIX files to.
    release : str
        ATT&CK release to download.
    known_hash : str
        SHA256 hash of the ATT&CK release.
    """
    release_download_dir = pathlib.Path(f"{download_dir}/v{release}")
    release_download_dir.mkdir(parents=True, exist_ok=True)
    fname = f"{domain}-attack.json"

    if stix_version == "2.0":
        download_url = f"https://raw.githubusercontent.com/mitre/cti/ATT%26CK-v{release}/{domain}-attack/{fname}"
    elif stix_version == "2.1":
        download_url = f"https://raw.githubusercontent.com/mitre-attack/attack-stix-data/master/{domain}-attack/{domain}-attack-{release}.json"

    pooch.retrieve(download_url, known_hash=known_hash, fname=fname, path=str(release_download_dir))

def download_domains(domains: List[str], download_dir: str, all_versions: bool, stix_version: str):
    """Download ATT&CK domains specified.

    Parameters
    ----------
    domains : List[str]
        List of domains to download.
    download_dir : str
        Directory to download the STIX files to.
    all_versions : bool
        Whether or not to download all versions of the domains.
    stix_version : str
        Version of STIX to download. Options are "2.0" or "2.1"
    """
    for domain in domains:
        if domain == "pre" and stix_version == "2.1":
            # there is no STIX 2.1 data for the PRE domain
            continue

        if stix_version == "2.0":
            stix_hash_data = release_info.STIX20
        elif stix_version == "2.1":
            stix_hash_data = release_info.STIX21

        releases = {}
        if domain == "enterprise":
            releases = stix_hash_data["enterprise"]
        elif domain == "mobile":
            releases = stix_hash_data["mobile"]
        elif domain == "ics":
            releases = stix_hash_data["ics"]
        elif domain == "pre":
            if stix_version == "2.0":
                releases = stix_hash_data["pre"]

        if all_versions:
            logger.info(f"Downloading STIX {stix_version} bundles for the {domain} domain to {download_dir}")
            for release, known_hash in releases.items():
                download_stix(
                    stix_version=stix_version,
                    domain=domain,
                    download_dir=download_dir,
                    release=release,
                    known_hash=known_hash,
                )
        else:
            if release_info.LATEST_VERSION in releases:
                logger.info(f"Downloading STIX {stix_version} bundle for the {domain} domain to {download_dir}")
                release = release_info.LATEST_VERSION
                known_hash = releases[release]
                download_stix(
                    stix_version=stix_version,
                    domain=domain,
                    download_dir=download_dir,
                    release=release,
                    known_hash=known_hash,
                )

@app.command()
def download_attack_stix(
    download_dir: str = typer.Option(
        "attack-releases", "--download-dir", "-d", help="Folder to save downloaded STIX data."
    ),
    all_versions: bool = typer.Option(False, "--all", "-a", help="Download all ATT&CK releases."),
    stix20: bool = typer.Option(True, help="Download STIX 2.0 data."),
    stix21: bool = typer.Option(False, help="Download STIX 2.1 data."),
):
    """Download the ATT&CK STIX data from GitHub in JSON format.

    By default, only the latest ATT&CK release will be downloaded in STIX 2.0 format.
    """
    domains = ["enterprise", "mobile", "ics", "pre"]

    if stix20:
        stix20_download_dir = f"{download_dir}/stix-2.0"
        pathlib.Path(stix20_download_dir).mkdir(parents=True, exist_ok=True)
        download_domains(
            domains=domains, download_dir=stix20_download_dir, all_versions=all_versions, stix_version="2.0"
        )

    if stix21:
        stix21_download_dir = f"{download_dir}/stix-2.1"
        pathlib.Path(stix21_download_dir).mkdir(parents=True, exist_ok=True)
        download_domains(
            domains=domains, download_dir=stix21_download_dir, all_versions=all_versions, stix_version="2.1"
        )


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('triumph_app', '0003_auto_20171115_0403'),
    ]

    operations = [
        migrations.AlterField(
            model_name='challenge',
            name='sequence',
            field=models.CharField(max_length=999, verbose_name=b'\xd0\x92\xd1\x8b\xd1\x80\xd0\xb0\xd0\xb6\xd0\xb5\xd0\xbd\xd0\xb8\xd0\xb5'),
            preserve_default=True,
        ),
    ]


import os
from typing import Dict, Optional, Union, Tuple

import torch
import torch.nn as nn
import torch.optim as optim

def get_device(
    device_type: str, device_index: Optional[Union[int, Tuple[int, ...]]] = None
) -> torch.device:
    if device_type == "cpu":
        return torch.device(device_type)
    elif device_type == "cuda":
        if isinstance(device_index, int):
            return torch.device(device_type + f":{device_index}")
        elif isinstance(device_index, list):
            raise NotImplementedError()
        else:
            raise ValueError("ConfigError: Invalid device index")
    else:
        raise ValueError("ConfigError: Invalid device type")

def get_optimizer(cfg, model):
    optimizer_params = cfg["train_config"]["parameters"]["optimizer"]
    optimizer_kind = optimizer_params.pop("kind")
    if optimizer_kind == "adam":
        return optim.Adam(model.parameters(), **optimizer_params)
    elif optimizer_kind == "sgd":
        return optim.SGD(model.parameters(), **optimizer_params)
    elif optimizer_kind == "adamw":
        return optim.AdamW(model.parameters(), **optimizer_params)
    else:
        raise NotImplementedError()

def get_loss(loss):
    if loss == "logit":
        return nn.BCEWithLogitsLoss()
    elif loss=="cross_entropy":
        return nn.CrossEntropyLoss()


import logging
from logging.handlers import RotatingFileHandler

import groupy
import forecastio
import giphypop
import requests
import bs4
from geopy.geocoders import Nominatim
from flask import Flask, request

application = Flask(__name__)
application.config.from_object('settings')

WEATHER_ICONS = {
  'clear-day': '☀',
  'clear-night': '🌑',
  'rain': '☔',
  'snow': '☃❄',
  'sleet': '☃❄',
  'wind': '🌬',
  'fog': '≈',
  'cloudy': '☁',
  'partly-cloudy-day': '⛅',
  'partly-cloudy-night': '☁🌑'
}

commands = {}

def command(cmd):
  def wrapped_command(function):
    def wrapper(*args, **kwargs):
      function(*args, **kwargs)
    if cmd not in commands:
      commands[cmd] = []
    commands[cmd].append(wrapper)
  return wrapped_command

@command('!weather')
def weather(bot, message, author=None, debug=False):
  api_key = application.config['FORECASTIO_API_KEY']

  if api_key is None:
    print('FORECASTIO_API_KEY not configured, not displaying weather information!')

  geolocator = Nominatim()
  location = geolocator.geocode(message or 'Austin, TX')
  forecast = forecastio.load_forecast(api_key, location.latitude, location.longitude)
  current = forecast.currently()
  icon = WEATHER_ICONS[current.icon] or ''

  post = '{0}  {1} {2}° (feels like {3}°)'.format(icon, current.summary, round(current.temperature), round(current.apparentTemperature))

  if debug:
    print(post)
  else:
    bot.post(post)

@command('!gif')
def gif(bot, message, author=None, debug=False):
  img = giphypop.translate(phrase=message, strict=True)

  if debug:
    print(img.media_url)
  else:
    bot.post(img.media_url)

@command('!slap')
def slap(bot, message, author=None, debug=False):
  if author is None:
    return

  slap = '{0} slaps {1} around a bit with a large trout'.format(author, message)

  if debug:
    print(slap)
  else:
    bot.post(slap)

@command('!h')
def horoscope(bot, message, author=None, debug=False):
  sign = message.lower()

  signs = [
    'aries',
    'taurus',
    'gemini',
    'cancer',
    'leo',
    'virgo',
    'libra',
    'scorpio',
    'sagittarius',
    'capricorn',
    'aquarius',
    'pisces'
  ]

  if sign not in signs:
    error = '{0} is not a star sign, zodiac thing, or whatever.'.format(message)
    if debug:
      print(error)
    else:
      bot.post()
    return

  url = 'http://www.horoscope.com/us/horoscopes/general/horoscope-general-daily-today.aspx?sign={0}'.format(signs.index(sign) + 1)
  page = requests.get(url)

  soup = bs4.BeautifulSoup(page.content, 'lxml')
  elements = soup.select('.block-horoscope-text')
  if len(elements) == 0:
    print('Unable to find horoscope for: {0}'.format(message))
    return
  horoscope = '{0}: {1}'.format(message, elements[0].getText().strip())
  if debug:
    print(horoscope)
  else:
    bot.post(horoscope)

@application.route('/pancakebot', methods=['POST'])
def hello():
  data = request.get_json()
  bot = groupy.Bot.list().first

  user = data['name']
  message = data['text']

  print('Received message: {0}'.format(data))

  for command in commands:
    if not message.startswith(command + ' ') and message != command:
      continue

    print('Executing command: {0}'.format(command))

    for callback in commands[command]:
      try:
        callback(bot, message[len(command):].strip(), author=user, debug=application.config['DEBUG'])
      except Exception as e:
        print('Error executing command: {0}'.format(command))
        print(e)

  return 'OK'

if __name__ == '__main__':
  application.run(host='0.0.0.0', port=5555)


from operator import methodcaller

class Node(object):
    """
    双链表中的链表节点对象
    """

    def __init__(self, key=None, value=None, freq=0):
        """
        Args:
            key:对应输入的key
            value:对应输入的value
            freq:被访问的频率
            pre:指向前一个节点的指针
            next:指向后一个节点的指针
        """
        self.key = key
        self.value = value
        self.freq = freq
        self.pre = None
        self.next = None

class LinkedList(object):
    """
    自定义的双向链表
    """

    def __init__(self):
        """
        Args:
            __head:双向链表的头结点
            __tail:双向链表的尾节点
        """
        self.__head = Node()
        self.__tail = Node()
        self.__head.next = self.__tail
        self.__tail.pre = self.__head

    def insertFirst(self, node):
        """
        将指定的节点插入到链表的第一个位置
        Args:
            node:将要插入的节点
        """
        node.next = self.__head.next
        self.__head.next.pre = node
        self.__head.next = node
        node.pre = self.__head

    def delete(self, node):
        """
        从链表中删除指定的节点

        Args:
            node:将要删除的节点
        """
        if self.__head.next == self.__tail:
            return
        node.pre.next = node.next
        node.next.pre = node.pre
        node.next = None
        node.pre = None

    def getLast(self):
        """
        从链表中获取最后一个节点
        Returns:
            双向链表中的最后一个节点，如果是空链表则返回None
        """
        if self.__head.next == self.__tail:
            return None
        return self.__tail.pre

    def isEmpty(self):
        """
        判断链表是否为空，除了head和tail没有其他节点即为空链表
        Returns:
            链表不空返回True，否则返回False
        """
        return self.__head.next == self.__tail

class LFUCache(object):
    """
    自定义的LFU缓存
    """

    def __init__(self, capacity):
        """
        Args:
            __capacity:缓存的最大容量
            __keyMap: key->Node 这种结构的字典
            __freqMap:freq->LinkedList 这种结构的字典
            __minFreq:记录缓存中最低频率
        """
        self.__capacity = capacity
        self.__keyMap = dict()
        self.__freqMap = dict()
        self.__minFreq = 0

    def get(self, key):
        """
        获取一个元素，如果key不存在则返回-1，否则返回对应的value
        同时更新被访问元素的频率
        Args:
            key:要查找的关键字
        Returns:
            如果没找到则返回-1，否则返回对应的value
        """
        #你的代码在这里#
        if key not in self.__keyMap:
            return -1

        self.__keyMap[key].freq += 1

        self.__freqMap[self.__keyMap[key].freq-1].delete(self.__keyMap[key])
        if self.__keyMap[key].freq not in self.__freqMap:
            self.__freqMap[self.__keyMap[key].freq] = LinkedList()
        self.__freqMap[self.__keyMap[key].freq].insertFirst(self.__keyMap[key])

        if self.__freqMap[self.__keyMap[key].freq-1].isEmpty():
            del self.__freqMap[self.__keyMap[key].freq-1]

        return self.__keyMap[key].value

    def put(self, key, value):
        """
        插入指定的key和value，如果key存在则更新value，同时更新频率
        如果key不存并且缓存满了，则删除频率最低的元素，并插入新元素
        否则，直接插入新元素
        Args:
            key:要插入的关键字
            value:要插入的值
        """
        #你的代码在这里#
        if key in self.__keyMap:
            self.__keyMap[key].value = value
            self.__keyMap[key].freq += 1

            self.__freqMap[self.__keyMap[key].freq -
                           1].delete(self.__keyMap[key])
            if self.__keyMap[key].freq not in self.__freqMap:
                self.__freqMap[self.__keyMap[key].freq] = LinkedList()
            self.__freqMap[self.__keyMap[key].freq].insertFirst(
                self.__keyMap[key])

            if self.__freqMap[self.__keyMap[key].freq-1].isEmpty():
                del self.__freqMap[self.__keyMap[key].freq-1]
        else:
            if len(self.__keyMap) >= self.__capacity:
                for (_, second) in self.__freqMap.items():
                    tmp = second.getLast()
                    second.delete(tmp)
                    del self.__keyMap[tmp.key]
                    break
                self.__keyMap[key] = Node(key, value, 1)
            else:
                self.__keyMap[key] = Node(key, value, 1)

        if 1 not in self.__freqMap:
            self.__freqMap[1] = LinkedList()
        self.__freqMap[1].insertFirst(self.__keyMap[key])

if __name__ == '__main__':
    operation = eval(input())
    data = eval(input())
    cache = eval("{}({})".format(operation.pop(0), data.pop(0)[0]))
    output = []
    for i, j in zip(operation, data):
        if i == 'put':
            methodcaller('put', j[0], j[1])(cache)
            output.append(None)
        elif i == 'get':
            output.append(methodcaller('get', j[0])(cache))
    print(output)
# [None, None, 1, None, -1, 3, None, -1, 3, 4]

实训专用/LFU缓存类.py


import wx
from mdbutton import MDButton

class DirPicker(wx.Panel):
    def __init__ (self, parent,
                  style = wx.TAB_TRAVERSAL,
                  labelText = 'Folder Path:',
                  initValue = '',
                  buttonText = 'Browse',
                  dialogTitle = 'Choose a folder',
                  pathCallback = None,
                  name = 'dirpicker'):
        super().__init__(parent, style=style, name=name)
        self.labelText = labelText
        self.initValue = initValue
        self.buttonText = buttonText
        self.dialogTitle = dialogTitle
        self.callback = pathCallback
        self.Font = parent.Font
        self.BackgroundColour = parent.BackgroundColour
        self.ForegroundColour = parent.ForegroundColour
        self.createPicker()

    def createPicker(self):
        vsizer = wx.BoxSizer(wx.VERTICAL)
        self.label = self.makeLabel()
        vsizer.Add(self.label, 0, wx.TOP, 16)
        hsizer = wx.BoxSizer(wx.HORIZONTAL)
        hsizer.Add(self.makeField(), 1, wx.ALIGN_BOTTOM|wx.RIGHT, 16)
        self.button = self.makeButton()
        hsizer.Add(self.button, 0)
        vsizer.Add(hsizer, 0, wx.EXPAND|wx.TOP, 8)
        self.SetAutoLayout(True)
        self.SetSizerAndFit(vsizer)
        self.Layout()

    def makeLabel(self):
        label = wx.StaticText(self, -1, self.labelText, style=wx.ALIGN_LEFT)
        label.BackgroundColour = self.BackgroundColour
        self.sizeText(label, self.labelText)
        return label

    def makeField(self):
        self.field = wx.TextCtrl(self, value=self.initValue,
            style=wx.TE_READONLY|wx.BORDER_NONE)
        self.field.BackgroundColour = self.BackgroundColour
        self.field.ForegroundColour = self.ForegroundColour
        self.field.Font = self.Font
        self.field.SetCanFocus(False)
        self.field.ToolTip = f'Path to: {self.labelText}'
        self.sizeText(self.field, self.initValue)
        line = wx.Panel(self, size=(-1, 1))
        line.BackgroundColour = self.ForegroundColour
        sizer = wx.BoxSizer(wx.VERTICAL)
        sizer.Add(self.field, 1, wx.EXPAND)
        sizer.Add(line, 0, wx.EXPAND)
        return sizer

    def makeButton(self):
        button = MDButton(self, self.buttonText)
        button.ToolTip = f'Click to browse to: {self.labelText}'
        button.Bind(wx.EVT_BUTTON, self.onBrowse)
        return button

    def setCallback(self, callbackFunc):
        self.callback = callbackFunc

    def onBrowse(self, ev=None):
        dirPath = wx.DirSelector(self.dialogTitle,
            style=wx.DD_DEFAULT_STYLE|wx.DD_DIR_MUST_EXIST)
        if dirPath:
            self.field.Value = dirPath
            if self.callback:
                self.callback(self.Name, dirPath)

    def sizeText(self, elem, value):
        width = self.CharWidth * (len(value) * 1.2 if value else 60) # (320, -1)
        elem.MinSize = (round(width), self.CharHeight * 1.2)


import lasagne.layers

class MaxPool(lasagne.layers.Pool2DLayer):

    # for convenience
    isdense, isbatchnorm, isconv, isdropout, isactivation = (False,) * 5
    ismaxpool = True

    def __init__(self, incoming_layer, pool_size, stride=None, pad=(0, 0),
                 ignore_border=True, name=None):

        lasagne.layers.Pool2DLayer.__init__(self, incoming_layer, pool_size,
                                            name=name, mode="max",
                                            stride=stride, pad=pad,
                                            ignore_border=ignore_border)

        self.inp_ndim = 4
        self.initialize_variables()

    def initialize_variables(self):

        self.pool_opts = {'ds': self.pool_size,
                          'st': self.stride,
                          'padding': self.pad,
                          'ignore_border': self.ignore_border}


#!/usr/bin/env python
# encoding: utf-8
'''
#-------------------------------------------------------------------
#                   CONFIDENTIAL --- CUSTOM STUDIOS
#-------------------------------------------------------------------
#
#                   @Project Name : 高级图片转换特效之毛玻璃
#
#                   @File Name    : effects_1.py
#
#                   @Programmer   : autofelix
#
#                   @Start Date   : 2022/01/14 13:14
#
#                   @Last Update  : 2022/01/14 13:14
#
#-------------------------------------------------------------------
'''
import cv2
import numpy as np

class picture:
    '''
     This is a main Class, the file contains all documents.
     One document contains paragraphs that have several sentences
     It loads the original file and converts the original file to new content
     Then the new content will be saved by this class
    '''
    def __init__(self):
        self.path = 'assets/picture.jpeg'

    def hello(self):
        '''
        This is a welcome speech
        :return: self
        '''
        print('*' * 50)
        print(' ' * 20 + '高级图片转换特效之毛玻璃')
        print(' ' * 5 + '作者: autofelix  Date: 2022-01-16 13:14')
        print(' ' * 5 + '主页: https://autofelix.blog.csdn.net')
        print('*' * 50)
        return self

    def run(self):
        # 读取原始图像
        src = cv2.imread(self.path)
        # 新建目标图像
        dst = np.zeros_like(src)
        # 获取图像行和列
        rows, cols = src.shape[:2]
        # 定义偏移量和随机数
        offsets = 5
        random_num = 0
        # 毛玻璃效果: 像素点邻域内随机像素点的颜色替代当前像素点的颜色
        for y in range(rows - offsets):
            for x in range(cols - offsets):
                random_num = np.random.randint(0, offsets)
                dst[y, x] = src[y + random_num, x + random_num]
        # 显示图像
        cv2.imshow('src', src)
        cv2.imshow('dst', dst)
        cv2.waitKey()
        cv2.destroyAllWindows()

if __name__ == '__main__':
    picture().hello().run()


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from util import get_data100k,read_process,get_edgelist,pa_index
import networkx as nx
from networkx.algorithms import bipartite

import numpy as np
import pandas as pd
import time
from collections import deque

import tensorflow as tf
from six import next
from sklearn import preprocessing
import sys
from scipy.sparse import lil_matrix
from scipy.sparse import coo_matrix

BATCH_SIZE = 1000
USER_NUM = 943
ITEM_NUM = 1682
df_train, df_test = get_data100k()
#df_train, df_test = get_data1m()
##preferential attachment
paUsers,paItems=pa_index(df_train,df_test)
################laplacian
la_movie,la_user,norm_la_movie,norm_la_user=laplacian_graph(df_train,df_test)

img=pd.read_csv('./data/cnn_100k.csv')
del img['Unnamed: 0']
img['path']-=1
img=img.set_index(np.arange(len(img)))
img_ma = np.zeros((ITEM_NUM,ITEM_NUM), dtype=np.float32) #np.asarray([[0 for x in range(USER_NUM)] for y in range(ITEM_NUM)],dtype=np.float16)
for index, row in img.iterrows():
    itemid=int(row['path'])
    img_ma[itemid]=row[1:]

AdjacencyUsers = np.zeros((USER_NUM,ITEM_NUM), dtype=np.float32) #np.asarray([[0 for x in range(ITEM_NUM)] for y in range(USER_NUM)],dtype=np.float16)
DegreeUsers_norm = np.zeros((USER_NUM,USER_NUM), dtype=np.float32)# np.asarray([[0 for x in range(1)] for y in range(USER_NUM)],dtype=np.float16)
AdjacencyItems = np.zeros((ITEM_NUM,USER_NUM), dtype=np.float32) #np.asarray([[0 for x in range(USER_NUM)] for y in range(ITEM_NUM)],dtype=np.float16)
DegreeItems_norm =  np.zeros((ITEM_NUM,ITEM_NUM), dtype=np.float32) #np.asarray([[0 for x in range(1)] for y in range(ITEM_NUM)],dtype=np.float16)
for index, row in df_train.iterrows():
    userid=int(row['user'])
    itemid=int(row['item'])
    AdjacencyUsers[userid][itemid]=row['rate']/5
    AdjacencyItems[itemid][userid]=row['rate']/5
    DegreeUsers_norm[userid][userid]+=1
    DegreeItems_norm[itemid][itemid]+=1

for i in range(len(DegreeUsers_norm)):
    for j in range(len(DegreeUsers_norm[i])):
        if DegreeUsers_norm[i][j] !=0:
            DegreeUsers_norm[i][j]=DegreeUsers_norm[i][j]**-0.5
            break

for i in range(len(DegreeItems_norm)):
    for j in range(len(DegreeItems_norm[i])):
        if DegreeItems_norm[i][j] !=0:
            DegreeItems_norm[i][j]=DegreeItems_norm[i][j]**-0.5
            break

class ShuffleIterator(object):

    def __init__(self, inputs, batch_size=10):
        self.inputs = inputs
        self.batch_size = batch_size
        self.num_cols = len(self.inputs)
        self.len = len(self.inputs[0])
        self.inputs = np.transpose(np.vstack([np.array(self.inputs[i]) for i in range(self.num_cols)]))

    def __len__(self):
        return self.len

    def __iter__(self):
        return self

    def __next__(self):
        return self.next()

    def next(self):
        ids = np.random.randint(0, self.len, (self.batch_size,))
        out = self.inputs[ids, :]
        return [out[:, i] for i in range(self.num_cols)]

class OneEpochIterator(ShuffleIterator):
    def __init__(self, inputs, batch_size=10):
        super(OneEpochIterator, self).__init__(inputs, batch_size=batch_size)
        if batch_size > 0:
            self.idx_group = np.array_split(np.arange(self.len), np.ceil(self.len / batch_size))
        else:
            self.idx_group = [np.arange(self.len)]
        self.group_id = 0

    def next(self):
        if self.group_id >= len(self.idx_group):
            self.group_id = 0
            raise StopIteration
        out = self.inputs[self.idx_group[self.group_id], :]
        self.group_id += 1
        return [out[:, i] for i in range(self.num_cols)]

def inferenceDense(phase,user_batch, item_batch,idx_user,idx_item, user_num, item_num,UReg=0.05,IReg=0.1):

    user_batch = tf.nn.embedding_lookup(idx_user, user_batch, name="embedding_user")
    item_batch = tf.nn.embedding_lookup(idx_item, item_batch, name="embedding_item")

    ul1mf=tf.layers.dense(inputs=user_batch, units=MFSIZE,activation=tf.nn.relu, kernel_initializer=tf.random_normal_initializer(stddev=0.01))
    il1mf=tf.layers.dense(inputs=item_batch, units=MFSIZE,activation=tf.nn.relu, kernel_initializer=tf.random_normal_initializer(stddev=0.01))
    InferInputMF=tf.multiply(ul1mf, il1mf)

    infer=tf.reduce_sum(InferInputMF, 1, name="inference")

    regularizer = tf.add(UW*tf.nn.l2_loss(ul1mf), IW*tf.nn.l2_loss(il1mf), name="regularizer")

    return infer, regularizer

def optimization(infer, regularizer, rate_batch, learning_rate=0.0005, reg=0.1):

    global_step = tf.train.get_global_step()
    assert global_step is not None
    cost_l2 = tf.nn.l2_loss(tf.subtract(infer, rate_batch))
    cost = tf.add(cost_l2, regularizer)
    train_op = tf.train.AdamOptimizer(learning_rate).minimize(cost, global_step=global_step)
    return cost, train_op

def clip(x):
    return np.clip(x, 1.0, 5.0)

def GraphRec_image(train, test,ver, Dataset='100k'):
    AdjacencyUsers = np.zeros((USER_NUM,ITEM_NUM), dtype=np.float32) #np.asarray([[0 for x in range(ITEM_NUM)] for y in range(USER_NUM)],dtype=np.float16)
    DegreeUsers = np.zeros((USER_NUM,1), dtype=np.float32)# np.asarray([[0 for x in range(1)] for y in range(USER_NUM)],dtype=np.float16)

    AdjacencyItems = np.zeros((ITEM_NUM,USER_NUM), dtype=np.float32) #np.asarray([[0 for x in range(USER_NUM)] for y in range(ITEM_NUM)],dtype=np.float16)
    DegreeItems =  np.zeros((ITEM_NUM,1), dtype=np.float32) #np.asarray([[0 for x in range(1)] for y in range(ITEM_NUM)],dtype=np.float16)
    for index, row in train.iterrows():
      userid=int(row['user'])
      itemid=int(row['item'])
      AdjacencyUsers[userid][itemid]=row['rate']/5.0
      AdjacencyItems[itemid][userid]=row['rate']/5.0
      DegreeUsers[userid][0]+=1
      DegreeItems[itemid][0]+=1

    DUserMax=np.amax(DegreeUsers)
    DItemMax=np.amax(DegreeItems)
    DegreeUsers=np.true_divide(DegreeUsers, DUserMax)
    DegreeItems=np.true_divide(DegreeItems, DItemMax)

    AdjacencyUsers=np.asarray(AdjacencyUsers,dtype=np.float32)
    AdjacencyItems=np.asarray(AdjacencyItems,dtype=np.float32)

    if ver=='ver1':
        UserFeatures= np.concatenate((np.identity(USER_NUM,dtype=np.bool_), AdjacencyUsers), axis=1)
        ItemFeatures= np.concatenate((np.identity(ITEM_NUM,dtype=np.bool_), AdjacencyItems,img_ma), axis=1)
    if ver=='ver2':
        UserFeatures= np.concatenate((np.identity(USER_NUM,dtype=np.bool_), AdjacencyUsers,DegreeUsers), axis=1)
        ItemFeatures= np.concatenate((np.identity(ITEM_NUM,dtype=np.bool_), AdjacencyItems,DegreeItems,img_ma), axis=1)
    if ver=='ver3':
        UserFeatures= np.concatenate((np.identity(USER_NUM,dtype=np.bool_), AdjacencyUsers,DegreeUsers,paUsers), axis=1)
        ItemFeatures= np.concatenate((np.identity(ITEM_NUM,dtype=np.bool_), AdjacencyItems,DegreeItems,paItems_img_ma), axis=1)
    if ver=='ver4':
        UserFeatures= np.concatenate((np.identity(USER_NUM,dtype=np.bool_), AdjacencyUsers,DegreeUsers_norm,paUsers), axis=1)
        ItemFeatures= np.concatenate((np.identity(ITEM_NUM,dtype=np.bool_), AdjacencyItems,DegreeItems_norm,paItems_img_ma), axis=1)
    if ver=='ver5':
        UserFeatures= np.concatenate((np.identity(USER_NUM,dtype=np.bool_), AdjacencyUsers,norm_la_user,paUsers), axis=1)
        ItemFeatures= np.concatenate((np.identity(ITEM_NUM,dtype=np.bool_), AdjacencyItems,norm_la_movie,paItems,img_ma), axis=1)
    if ver=='ver6':
        UserFeatures= np.concatenate((np.identity(USER_NUM,dtype=np.bool_), AdjacencyUsers,la_user,paUsers), axis=1)
        ItemFeatures= np.concatenate((np.identity(ITEM_NUM,dtype=np.bool_), AdjacencyItems,la_movie,paItems,img_ma), axis=1)

    UserFeaturesLength=UserFeatures.shape[1]
    ItemFeaturesLength=ItemFeatures.shape[1]

    samples_per_batch = len(train) // BATCH_SIZE
    iter_train = ShuffleIterator([train["user"],train["item"],train["rate"]],batch_size=BATCH_SIZE)
    iter_test = OneEpochIterator([test["user"],test["item"],test["rate"]],batch_size=10000)

    user_batch = tf.placeholder(tf.int32, shape=[None], name="id_user")
    item_batch = tf.placeholder(tf.int32, shape=[None], name="id_item")
    rate_batch = tf.placeholder(tf.float64, shape=[None])
    phase = tf.placeholder(tf.bool, name='phase')

    w_user = tf.constant(UserFeatures,name="userids", shape=[USER_NUM, UserFeatures.shape[1]],dtype=tf.float64)
    w_item = tf.constant(ItemFeatures,name="itemids", shape=[ITEM_NUM, ItemFeatures.shape[1]],dtype=tf.float64)

    infer, regularizer = inferenceDense(phase,user_batch, item_batch,w_user,w_item, user_num=USER_NUM, item_num=ITEM_NUM)
    global_step = tf.contrib.framework.get_or_create_global_step()
    _, train_op = optimization(infer, regularizer, rate_batch, learning_rate=LR, reg=0.09)

    init_op = tf.global_variables_initializer()
    config = tf.ConfigProto()
    config.gpu_options.per_process_gpu_memory_fraction = 0.5
    finalerror=-1
    with tf.Session(config=config) as sess:
        sess.run(init_op)
        print("{} {} {} {}".format("epoch", "train_error", "val_error", "elapsed_time"))
        errors = deque(maxlen=samples_per_batch)
        start = time.time()
        for i in range(EPOCH_MAX * samples_per_batch):
            #users, items, rates,y,m,d,dw,dy,w = next(iter_train)
            users, items, rates = next(iter_train)
            _, pred_batch = sess.run([train_op, infer], feed_dict={user_batch: users,
                                                                   item_batch: items,
                                                                   rate_batch: rates,
                                                                   phase:True})
            pred_batch = clip(pred_batch)
            errors.append(np.power(pred_batch - rates, 2))
            if i % samples_per_batch == 0:
                train_err = np.sqrt(np.mean(errors))
                test_err2 = np.array([])
                degreelist=list()
                predlist=list()
                for users, items, rates in iter_test:
                    pred_batch = sess.run(infer, feed_dict={user_batch: users,
                                                            item_batch: items,
                                                            phase:False})

                    pred_batch = clip(pred_batch)
                    test_err2 = np.append(test_err2, np.power(pred_batch - rates, 2))
                end = time.time()
                test_err = np.sqrt(np.mean(test_err2))
                finalerror=test_err
                print("{:3d},{:f},{:f},{:f}(s)".format(i // samples_per_batch, train_err, test_err, end - start))
                start = end

MFSIZE=50
UW=0.05
IW=0.02
LR=0.00003
EPOCH_MAX = 300
tf.reset_default_graph()
GraphRec_image(df_train, df_test,ver='ver1',Dataset='100k')


import logging
import requests, json

from kalliope.core.NeuronModule import (NeuronModule,
                                        MissingParameterException,
                                        InvalidParameterException)

logging.basicConfig()
logger = logging.getLogger("kalliope")

class Cryptocurrency(NeuronModule):
    """
    Class used to interact with diagral alarm system
    """
    def __init__(self, **kwargs):

        super(Cryptocurrency, self).__init__(**kwargs)

        # parameters
        self.currency = kwargs.get('currency', None)
        self.target = kwargs.get('target', None)

        logger.debug("CryptoCurrency launch for currency %s", self.currency)

        # check parameters
        if self._is_parameters_ok():
            result = dict( )

            payload = {'fsym': self.currency, 'tsyms': self.target}
            logger.debug(payload)
            response = requests.get('https://min-api.cryptocompare.com/data/price', params=payload)

            content = response.json()
            result[self.currency] = dict()
            result[self.currency][self.target] = content[self.target]

            logger.debug(result)
            self.say(result)

    def _is_parameters_ok(self):
        """
        Check if received parameters are ok to perform operations in the neuron.
        :return: True if parameters are ok, raise an exception otherwise.

        .. raises:: MissingParameterException, InvalidParameterException
        """
        if self.currency is None:
            raise MissingParameterException("CryptoCurrency needs an currency")
        if self.target is None:
            raise MissingParameterException("CryptoCurrency needs a target currency")
        return True


__author__ = 'rmozes1'
import sys
import numpy as np

class Status:
	INVALID_MOVE = -1.01
	SUCCESS = 0.1
	WIN = 1
	LOSE = -1

class TTTBoard:
	def __init__(self, cols=5, rows=5, seq_size=4):
		self.cols = cols
		self.rows = rows
		self.board = [[0 for i in range(rows)] for j in range(cols)]
		self.seq_size = seq_size
		self.next_player = 1

	def get(self):
		return np.reshape(np.array(self.board),(1,self.cols*self.rows))

	def set(self,state):
		self.board = [[state[0],state[1],state[2]],[state[3],state[4],state[5]],[state[6],state[7],state[8]]]

	def reset(self):
		  self.board = [[0 for i in range(self.rows)] for j in range(self.cols)]

	def get_new_player(self):
		self.next_player += 1
		return self.next_player - 1

	def win_row(self, player, pos_x, pos_y):
		tmp_seq_size = 0
		for i in range(max(0, pos_x - self.seq_size + 1), min(self.cols, pos_x + self.seq_size)):
			if self.board[pos_y][i] == player:
				tmp_seq_size += 1
				if tmp_seq_size == self.seq_size:
					return True
			else:
				tmp_seq_size = 0
		return False

	def win_col(self, player, pos_x, pos_y):
		tmp_seq_size = 0
		for j in range(max(0, pos_y - self.seq_size + 1), min(self.rows, pos_y + self.seq_size)):
			if self.board[j][pos_x] == player:
				tmp_seq_size += 1
				if tmp_seq_size == self.seq_size:
					return True
			else:
				tmp_seq_size = 0
		return False

	def win_diagonal(self, player, pos_x, pos_y):
		tmp_seq_size = 0
		diff = pos_x - pos_y;
		x_range = range(max(0, pos_x - self.seq_size + 1), min(self.cols, pos_x + self.seq_size))
		for i in x_range:
			j = i - diff
			if j in range(0, self.rows):
				if self.board[j][i] == player:
					tmp_seq_size += 1
					if tmp_seq_size == self.seq_size:
						return True
				else:
					tmp_seq_size = 0
		return False

	def win_reversed_diagonal(self, player, pos_x, pos_y):
		tmp_seq_size = 0
		sum = pos_x + pos_y
		x_range = range(max(0, pos_x - self.seq_size + 1), min(self.cols, pos_x + self.seq_size))
		for i in x_range:
			j = sum - i
			if j in range(0, self.rows):
				if self.board[j][i] == player:
					tmp_seq_size += 1
					if tmp_seq_size == self.seq_size:
						return True
				else:
					tmp_seq_size = 0
		return False

	def game_over(self):
		for i in range (self.rows):
			for j in range ( self.cols):
				if self.board[i][j] == 0:
					return False
		return True

	def play_random(self,player):

		pos = np.random.randint(self.cols*self.rows)
		pos_x = int(pos % self.cols)
		pos_y = int(pos / self.cols)
		while self.board[pos_y][pos_x] != 0:
			pos = np.random.randint(self.cols*self.rows)
			pos_x = int(pos % self.cols)
			pos_y = int(pos / self.cols)

		return pos,self.play(player,pos)

	def play(self, player, pos):

		pos_x = int(pos % self.cols)
		pos_y = int(pos / self.cols)
		if self.board[pos_y][pos_x] != 0:
			return Status.INVALID_MOVE

		self.board[pos_y][pos_x] = player

		if (self.win_row(player, pos_x, pos_y) or self.win_col(player, pos_x, pos_y) or
				self.win_diagonal(player, pos_x, pos_y) or self.win_reversed_diagonal(player, pos_x, pos_y)):
			return Status.WIN

		return Status.SUCCESS

	def draw(self):

		# draw top line
		sys.stdout.write(' -')
		for j in range(self.cols):
				sys.stdout.write('----')
		sys.stdout.write('\n')

		# draw board
		for i in range(self.rows):
			for j in range(self.cols):
				if self.board[i][j] == 1:
					sign = 'X'
				if self.board[i][j] == -1:
					sign = 'O'
				if self.board[i][j] == 0:
					sign = ' '
				sys.stdout.write(' | ' + sign)
			sys.stdout.write(' |')
			sys.stdout.write('\n -')
			for j in range(self.cols):
				sys.stdout.write('----')
			sys.stdout.write('\n')


import sys
sys.stdin = open("stack1_괄호검사.txt")

T = int(input())

for test_case in range(1, T+1):
    words = input()

    parenthesis = ['[', '{', '(', ']', '}', ')']
    check = []
    flag = 1

    for word in words:
        if word not in parenthesis:
            continue
        elif word == '[':
            check.append(word)
        elif word == '{':
            check.append(word)
        elif word == '(':
            check.append(word)
        elif word == ']':
            if len(check) >= 1 and check[-1] == '[':
                check.pop()
            else:
                flag = 0
                break
        elif word == '}':
            if len(check) >= 1 and check[-1] == '{':
                check.pop()
            else:
                flag = 0
                break
        elif word == ')':
            if len(check) >= 1 and check[-1] == '(':
                check.pop()
            else:
                flag = 0
                break
        else:
            break

    print('#{}'.format(test_case), end=' ')
    if flag == 0:
        print(0)
    elif flag == 1:
        if len(check) == 0:
            print(1)
        else:
            print(0)


# -*- coding: utf-8 -*-
"""
Created on Thu Dec 19 20:45:25 2019

@author: ar1
"""
import gc
import time
import numpy as np
import pandas as pd
from multiprocessing import Pool
from functools import partial

import os
for dirname, _, filenames in os.walk('C:/Personal/Kaggle/ASHRAE/ashrae-energy-prediction/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

## Function to read the data sets
def read_data(path):
    """
    Reads the data from the path and
    prints the shape and sample rows of the dataframe
    """
    df = pd.read_csv(path)
    print(df.shape)
    print(df.head())
    return df

def parse_timestamp(df, col):
    """
    Converts the timestamp column from object to datetime type
    """
    df[col] = pd.to_datetime(df[col])
    return df

def merge_datasets(train, weather_train, building_metadata):
    """
    Merges the 3 dataframes train, weather_train and building_metadata
    to a single dataframe and removes the duplicate rows if any
    """
    print("Merging weather data with building metadata")
    weather_build_meta = pd.merge(weather_train, building_metadata, on='site_id', how='left')
    print(weather_build_meta.shape)

    print("Dropping duplicates, if any")
    weather_build_meta = weather_build_meta.drop_duplicates()
    print(weather_build_meta.shape)

    print("Merging with train data")
    final_data = pd.merge(weather_build_meta, train, on = ['building_id', 'timestamp'], how='outer')
    print(final_data.shape)

    print("Dropping duplicates, if any")
    final_data = final_data.drop_duplicates()
    print(final_data.shape)

    return final_data

def parallelize_dataframe(df, group_list, func, **kwargs):
    """
    Parallelize the dataframe based on groups in a grouped dataframe,
    runs the specified function on each of the groups in parallel and
    combines it to a dataframe
    """
    df_split = df.groupby(group_list)
    pool = Pool(os.cpu_count())
#    if args is not None:
#        func_x=partial(func, **kwargs)
#    else:
    func_x = partial(func, **kwargs)
    ret_list = pool.map(func_x, [group for name, group in df_split])
    df = pd.DataFrame()
    print("for loop")
    for dat in ret_list:
        df = df.append(dat, ignore_index=True)
    pool.close()
    pool.join()
    return df

def fill_attributes(df, attr_list):
    print(df['building_id'].unique()[0])
    for attr in attr_list:
        print(attr)
        attr_value_list = df[attr].dropna().unique().tolist()
        if len(attr_value_list) == 1:
            df[attr] = df[attr].fillna(attr_value_list[0])
        else:
            print("No unique values to fill the attributes")
    return df

def fill_numerical_data(df):
    print("Forward Fill")
    df[['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
       'wind_direction', 'wind_speed']] = df.groupby(['building_id','meter'])['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
       'wind_direction', 'wind_speed'].fillna(method='ffill')

    print("Backward Fill")
    df[['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
       'wind_direction', 'wind_speed']] = df.groupby(['building_id','meter'])['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
       'wind_direction', 'wind_speed'].fillna(method='bfill')

    return df

def create_features(df, limit_sqft):
    print(df['building_id'].unique()[0])
    ## Features from timestamp
    df['year'] = df['timestamp'].dt.year
    df['month'] = df['timestamp'].dt.month
    df['day'] = df['timestamp'].dt.day
    df['hour'] = df['timestamp'].dt.hour

    df['age'] = df['year'] - df['year_built']

    df['square_feet_profile'] = np.where((df['square_feet'] >= limit_sqft['min']) & (df['square_feet'] < limit_sqft['25%']), 'small',
                                         np.where((df['square_feet'] >= limit_sqft['25%']) & (df['square_feet'] < limit_sqft['50%']), 'medium',
                                         np.where((df['square_feet'] >= limit_sqft['50%']) & (df['square_feet'] < limit_sqft['75%']), 'big',
                                         np.where((df['square_feet'] >= limit_sqft['75%']) & (df['square_feet'] < limit_sqft['max']), 'huge', np.nan))))

    return df

def sine_transform(input_df, normalize_var):
    """Transform a input DF to Sine transform"""
    col_names = input_df.columns
    if len(col_names) == 1:
        transformed_array = np.sin(2 * np.pi * input_df[col_names[0]] / normalize_var)
        transformed_df = pd.DataFrame({f'sine_{col_names[0]}': transformed_array})
    else:
        transformed_df = None
    return transformed_df

def cosine_transform(input_df, normalize_var):
    """Transform a input DF to Cosine transform"""
    col_names = input_df.columns
    if len(col_names) == 1:
        transformed_array = np.cos(2 * np.pi * input_df[col_names[0]] / normalize_var)
        transformed_df = pd.DataFrame({f'cosine_{col_names[0]}': transformed_array})
    else:
        transformed_df = None
    return transformed_df

def create_sin_cos_features(df):
    print(df['building_id'].unique()[0])
    df['month_sin'] = sine_transform(df[['month']], 12)
    df['month_cos'] = cosine_transform(df[['month']], 12)

    df['day_sin'] = sine_transform(df[['day']], 31)
    df['day_cos'] = cosine_transform(df[['day']], 31)

    df['hour_sin'] = sine_transform(df[['hour']], 23)
    df['hour_cos'] = cosine_transform(df[['hour']], 23)

    df['wind_dir_sin'] = sine_transform(df[['wind_direction']], 360)
    df['wind_dir_cos'] = cosine_transform(df[['wind_direction']], 360)

    return df

def main():
    start = time.time()
    #global meter_grp
    ## Reading Training Data
    test = read_data('C:/Personal/Kaggle/ASHRAE/ashrae-energy-prediction/input/test.csv')
    weather_test = read_data('C:/Personal/Kaggle/ASHRAE/ashrae-energy-prediction/input/weather_test.csv')
    building_metadata = read_data('C:/Personal/Kaggle/ASHRAE/ashrae-energy-prediction/input/building_metadata.csv')

    ## Parsing Timestamps
    test = parse_timestamp(test, 'timestamp')
    weather_test = parse_timestamp(weather_test, 'timestamp')

    test_final = merge_datasets(test, weather_test, building_metadata)
    print(test_final.head())
    print(test_final.columns)
    print(test_final.shape)

    del test, weather_test, building_metadata
    print(gc.collect())

    for meter in test_final.meter.unique():
        print ("Meter " + str(meter))
        test_final_0 = test_final[test_final['meter'] == meter].reset_index(drop=True)
        print(test_final_0.shape)

        print("Filling attribute data")
        test_final_0[['building_id', 'site_id', 'primary_use', 'square_feet', 'year_built']] = parallelize_dataframe(test_final_0[['building_id', 'site_id', 'primary_use', 'square_feet', 'year_built']], ['building_id'], fill_attributes, attr_list=['site_id', 'primary_use', 'square_feet', 'year_built'])
        print(test_final_0.isnull().sum())

        ## Filling Numerical data
        print("Filling numerical data")

        test_final_0[['building_id', 'meter', 'air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
           'wind_direction', 'wind_speed']] = parallelize_dataframe(test_final_0[['building_id', 'meter', 'air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
           'wind_direction', 'wind_speed']], ['building_id','meter'], fill_numerical_data)

        print(test_final_0.isnull().sum())

    #    print("Forward Fill")
    #    test_final_0[['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
    #       'wind_direction', 'wind_speed']] = test_final_0.groupby(['building_id','meter'])['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
    #       'wind_direction', 'wind_speed'].fillna(method='ffill')
    #    print(test_final_0.isnull().sum())
    #
    #    print("Backward Fill")
    #    test_final_0[['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
    #       'wind_direction', 'wind_speed']] = test_final_0.groupby(['building_id','meter'])['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
    #       'wind_direction', 'wind_speed'].fillna(method='bfill')
    #    print(test_final_0.isnull().sum())
    #

        ## Creating Features
        print("Creating features")
        #global limit_sqft
        limit_sqft = test_final['square_feet'].describe()
        test_final_0 = parallelize_dataframe(test_final_0, ['building_id'], create_features, limit_sqft=limit_sqft)
        print(test_final_0.head())

        print("Creating cyclical features")
        test_final_0 = parallelize_dataframe(test_final_0, ['building_id'], create_sin_cos_features)
        print(test_final_0.head())

        test_final_0.to_csv('C:/Personal/Kaggle/ASHRAE/ashrae-energy-prediction\output/test_final_'+str(meter)+'.csv', index=False)

    end = time.time()
    print(end-start)

    return test_final_0

if __name__ == "__main__":
    msg = main()
    print(msg)


import string
from math import ceil, floor
from .eds import EDS
from .utilities import FailedValidationError, get_name, decode, format_bytes
from ..can import MessageType

PDO1_TX = 0x1A00
PDO1_RX = 0x1600
PDO2_TX = 0x1A01
PDO2_RX = 0x1601
PDO3_TX = 0x1A02
PDO3_RX = 0x1602
PDO4_TX = 0x1A03
PDO4_RX = 0x1603

def parse(cob_id: int, data: bytes, eds: EDS):
    """
    PDO mappings come from the eds file and is dependent on the type (
    Receiving/transmission PDO). Mapping value is made up of index subindex
    and size. For Example 0x31010120 Means 3101sub01 size 32bit

    The eds mapping is determined by the cob_id passed ot this function. That
    indicated which PDO record to look up in the EDS file.
    """
    msg_type = MessageType.cob_id_to_type(cob_id)
    pdo_type = {
        MessageType.PDO1_TX: PDO1_TX,
        MessageType.PDO1_RX: PDO1_RX,
        MessageType.PDO2_TX: PDO2_TX,
        MessageType.PDO2_RX: PDO2_RX,
        MessageType.PDO3_TX: PDO3_TX,
        MessageType.PDO3_RX: PDO3_RX,
        MessageType.PDO4_TX: PDO4_TX,
        MessageType.PDO4_RX: PDO4_RX,
        MessageType.UKNOWN: None
    }[msg_type]

    if(not pdo_type or msg_type.supertype is not MessageType.PDO):
        raise FailedValidationError(data,
                                    cob_id - MessageType.PDO1_TX.value[0],
                                    cob_id,
                                    __name__,
                                    f"Unable to determine pdo type with given"
                                    f" cob_id {hex(cob_id)}, expected value"
                                    f" between {MessageType.PDO1_TX.value[0]}"
                                    f" and {MessageType.PDO4_RX.value[1] + 1}")

    if len(data) > 8 or len(data) < 1:
        raise FailedValidationError(data,
                                    cob_id - MessageType.PDO1_TX.value[0],
                                    cob_id,
                                    __name__,
                                    f"Invalid payload length {len(data)} "
                                    f"expected between 1 and 8")
    try:
        eds_elements = eds[hex(pdo_type)][0]
    except (TypeError, IndexError):
        raise FailedValidationError(data,
                                    cob_id - MessageType.PDO1_TX.value[0],
                                    cob_id,
                                    __name__,
                                    f"Unable to find eds data for pdo type "
                                    f"{hex(pdo_type)}")

    # default_value could be 2 or '0x02', this is meant to work with both
    if (c in string.hexdigits for c in str(eds_elements.default_value)):
        num_elements = int(str(eds_elements.default_value), 16)
    else:
        num_elements = int(str(eds_elements.default_value))

    if num_elements < 0x40:
        return parse_pdo(num_elements, pdo_type, cob_id, eds, data)

    if num_elements in (0xFE, 0xFF):
        if len(data) != 8:
            raise FailedValidationError(data,
                                        cob_id - MessageType.PDO1_TX.value[0],
                                        cob_id,
                                        __name__,
                                        f"Invalid payload length {len(data)} "
                                        f"expected 8")
        return parse_mpdo(num_elements, pdo_type, eds, data, cob_id)

    raise FailedValidationError(data,
                                cob_id - MessageType.PDO1_TX.value[0],
                                cob_id,
                                __name__,
                                f"Invalid pdo mapping detected in eds file at "
                                f"[{pdo_type}sub0]")

def parse_pdo(num_elements, pdo_type, cob_id, eds, data):
    """
    Parse pdo message. Message will include num_elements elements. Elements
    are processed in reverse order, from rightmost to leftmost
    """
    output_string = ""
    data_start = 0
    for i in range(num_elements, 0, -1):
        try:
            eds_record = eds[hex(pdo_type)][i]
        except (TypeError, IndexError):
            raise FailedValidationError(data,
                                        cob_id - MessageType.PDO1_TX.value[0],
                                        cob_id,
                                        __name__,
                                        f"Unable to find eds data for pdo type"
                                        f" {hex(pdo_type)} index {i}")

        pdo_definition = int(eds_record.default_value, 16).to_bytes(4, "big")

        index = pdo_definition[0:3]
        size = pdo_definition[3]
        mask = 1
        for j in range(1, size):
            mask = mask << 1
            mask += 1

        # Possible exceptions from get_name are not caught because they indicate
        # an issue with the PDO definition in the OD file, which should be
        # checked when the file is loaded
        eds_details = get_name(eds, index)
        num_bytes = ceil(size / 8)

        start = len(data) - num_bytes - floor(data_start / 8)
        end = len(data) - floor(data_start / 8)
        masked_data = int.from_bytes(data[start:end], "big") & mask
        masked_data = masked_data >> data_start % 8
        masked_data = masked_data.to_bytes(num_bytes, "big")
        output_string = f"{eds_details[1]} -" \
                        f" {decode(eds_details[0], masked_data)}" + \
                        output_string
        if i > 1:
            output_string = " " + output_string
        data_start += size

    return output_string

def parse_mpdo(num_elements, pdo_type, eds, data, cob_id):
    mpdo = MPDO(data)
    if mpdo.is_source_addressing and num_elements != 0xFE:
        raise FailedValidationError(data,
                                    cob_id - MessageType.PDO1_TX.value[0],
                                    cob_id,
                                    __name__,
                                    f"MPDO type and definition do not match. "
                                    f"Check eds file at [{pdo_type}sub0]")

    try:
        eds_details = get_name(eds, mpdo.index)
    except KeyError as e:
        raise FailedValidationError(data,
                                    cob_id - MessageType.PDO1_TX.value[0],
                                    cob_id,
                                    __name__,
                                    f"MPDO provided type index does not exist. "
                                    f"Check provided index {str(e)}")

    except ValueError:
        raise FailedValidationError(data,
                                    cob_id - MessageType.PDO1_TX.value[0],
                                    cob_id,
                                    __name__,
                                    f"MPDO provided type index is missing "
                                    f"attributes. Check OD file provided index "
                                    f"[{format_bytes(mpdo.index)}")

    return f"{eds_details[1]} - {decode(eds_details[0], mpdo.data)}"

class MPDO:
    """

 .. code-block:: python

     +-------------+---------+----------------+
     |   f   addr  |    m    |       d        |
     |   7   6_0   |         |                |
     +-------------+---------+----------------+
            0        1     3  4             7

 Definitions
 ===========
 * **f**: address type
  0. Source addressing
  1. Destination addressing

 * **addr**: node-ID of the MPDO consumer in destination addressing or MPDO
 producer in source addressing. 0. Shall be reserved in source addressing
 mode. Shall address all CANopen devices in the network that are configured
 for MPDO reception in destination addressing mode. 1..127. Shall address the
 CANopen device in the network with the very same node-ID

 * **m**: multiplexer. It represents the index/sub-index of the process data
 to be transferred by the MPDO. In source addressing this shall be used to
 identify the data from the transmitting CANopen device or in destination
 addressing addressing to identity the data on the receiving CANopen device.

 * **d**: process data. Data length lower than 4 bytes is filled up to fit
 32-bit
    """

    def __init__(self, raw_sdo: bytes):
        self.__is_source_addressing = raw_sdo[0] & 0x8 == 0x8
        self.__is_destination_addressing = not self.__is_source_addressing
        self.__addr = raw_sdo[0] & 0x7F
        self.__index = raw_sdo[1:4]
        self.__data = raw_sdo[4:8]

    @property
    def is_source_addressing(self):
        return self.__is_source_addressing

    @property
    def is_destination_addressing(self):
        return self.__is_destination_addressing

    @property
    def addr(self):
        return self.__addr

    @property
    def index(self):
        return self.__index

    @property
    def data(self):
        return self.__data


### Diffusion Map Wrapper/Diffuse Function -- Python Implementation ###

import numpy as np
from scipy.sparse import csc_matrix
from scipy.sparse.linalg import eigsh

# Class to format the output of the diffuse function for the diffusion map

class Dmap(object):
    def __init__(self, X, phi0, eigenvals, eigenmult,
                 psi, phi, neigen, epsilon):
        self.X = X
        self.phi0 = phi0
        self.eigenvals = eigenvals
        self.eigenmult = eigenmult
        self.psi = psi
        self.phi = phi
        self.neigen = neigen
        self.epsilon = epsilon

# D - input distance matrix (represented as a 2D-list)
# neigen - the number of diffusion coordinates to return
# eps - a tuning parameter
# delta - a filtering parameter for creating a sparse matrix (Asp)
def diffuse(D,neigen,eps,delta=10^-10):
    n = D.shape[0]
    K = np.exp(-1 * (D**2) / eps)
    ## next two lines added to match Jaehyeok's DataLinker code
    v = K.sum(axis=1)
    K = K/np.outer(v,v.T)
    ##
    v = K.sum(axis=1)**0.5
    A = K / np.outer(v, v.T)

    ind = np.array([[row,col]
        for col in range(len(A))
            for row in range(len(A[0]))
                if A[row][col] > delta
    ])
    row = ind[..., 0]
    col = ind[..., 1]
    data = A[row, col]
    Asp = csc_matrix((data, (row, col)), shape=(n, n)).toarray()

    neff = min(neigen+1,n)
    eigenvals, eigenvecs = eigsh(Asp, k=neff, which="LA", ncv=n)
    # Python eigsh sorts eigenvalues in order of increasing value,
    # whereas R arpack sorts them in order of decreasing value.
    # This line will correct for this
    (eigenvals, eigenvecs) = (eigenvals[::-1], eigenvecs[..., ::-1])

    psi = eigenvecs / (eigenvecs[..., 0:1].dot(np.ones((1, neff))))
    phi = eigenvecs * (eigenvecs[..., 0:1].dot(np.ones((1, neff))))

    lam = eigenvals[1:] / (1 - eigenvals[1:])
    lam = np.outer(np.array([1]*n), lam.T)
    X = psi[..., 1:neigen+1] * lam[..., 0:neigen]

    y = Dmap(X=X, phi0=phi[...,0], eigenvals=eigenvals[1:],
             eigenmult=lam[0, 0:neigen], psi=psi, phi=phi,
             neigen=neigen, epsilon=eps)
    return y

## csvPath - path to get to csv data file *from the directory this file is in*
## k - number of nearest neighbors to use in computation
#
#def diffusionMap(csvPath, k, neigen, eps=1):
#
#	# Read in and format data
#	statistics = np.array(readFile(csvPath))
#	statScale = preprocessing.scale(statistics)
#	D = squareform(pdist(statScale))
#
#	nrow = D.shape[0]
#	sigma = np.array([0.0]*nrow)
#	for i in range(nrow):
#		j = np.argsort(D[..., i])[k + 1]
#		sigma[i] = pdist(np.array([statScale[i,...], statScale[j,...]]))
#
#	S = np.outer(np.array(1 / (sigma**0.5)), np.array(1 / (sigma**0.5)))
#	D = D * S
#	dmap = diffuse(D, neigen, eps)


from urllib import request
from urllib.request import Request

from bs4 import BeautifulSoup

from app.twitter_learning_journal.cachers.webpage_cacher import WebpageCacher
from app.twitter_learning_journal.services.logging_service import LoggingService
from app.twitter_learning_journal.transformers.transform_str import remove_ignore_characters_from_str

logger = LoggingService('Blogs')

def classify_blogs(details):
    print(f'total details: {len(details)}')
    detail_count = 1

    unclassified_blogs = []
    classified_blogs = []
    _details = [detail for detail in details if detail.url]

    for detail in _details:
        is_other = False
        for url in detail.url.split('|'):
            if is_other:
                break

            for other_url_type in not_blog_urls:
                if is_other:
                    break

                if other_url_type in url:
                    logger.info(f'@@@@ URL looks like not blog: {url}')
                    detail.is_fully_classified = False
                    detail.type = 'other'
                    unclassified_blogs.append(detail)
                    is_other = True

    _details = [detail for detail in _details if detail not in unclassified_blogs]

    for detail in _details:
        total_count = 0

        for url in detail.url.split('|'):
            url = url.replace('www.google.com/amp/s/', '')

            if not url:
                continue

            webpage_cacher = WebpageCacher(url)

            if not webpage_cacher.is_cached():
                _request = Request(url)
                _request.headers = headers

                try:
                    html = request.urlopen(_request).read().decode('utf8')
                except Exception as e:
                    print(f'could not open url: {url}')
                    continue

                webpage_cacher.entity = html
                webpage_cacher.cache()
            else:
                try:
                    html = webpage_cacher.get()
                except:
                    _request = Request(url)
                    _request.headers = headers

                    try:
                        html = request.urlopen(_request).read().decode('utf8')
                    except Exception as e:
                        print(f'could not open url: {url}')
                        continue

                    webpage_cacher.entity = html
                    webpage_cacher.cache()

            html = remove_auxiiary_tags(html)

            _raw = BeautifulSoup(html).get_text()
            _raw = remove_ignore_characters_from_str(_raw)
            _raw = ' '.join(_raw.split())

            raw_split = _raw.split()
            words = len(raw_split)

            devation = default_domain_devation
            is_found = False

            if words == 0:
                print('Removed/Redirected URL')
                continue

            # I feel like this is a common problem that could be optimized
            for key in domain_deviations.keys():
                if key in url:
                    is_found = True
                    devation = domain_deviations[key]

            if not is_found:
                print()

            if words > (devation * 1.25):
                words = words - devation

            print(f'counted words: {words}')
            total_count += words

        detail_count += 1

        print(f'processed: {detail_count}')

        if total_count == 0:
            continue

        detail.count = total_count
        classified_blogs.append(detail)

    return classified_blogs, unclassified_blogs

def remove_auxiiary_tags(html):
    soup = BeautifulSoup(html)
    for tag in soup(['script', 'style', 'head']):
        tag.decompose()
    return ' '.join(soup.stripped_strings)

headers = {
    'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0.1; SM-G920V Build/MMB29K) '
                  'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.98 Mobile Safari/537.36'
}

default_domain_devation = 300

domain_deviations = {
    'cucumber.io': 200,
    'blog.jessitron.com': 250,
    '12factor.net': 25,
    'interviewcake.com': 550,
    'yanado.com': 300,
    'medium.freecodecamp.org': 200,
    'deque.blog': 300,
    'shmula.com': 500,
    'fearlesssalarynegotiation.com': 500,
    'gizmodo.com': 150,
    'expertenough.com': 150,
    'github.io': 100,
    'employeebenefitadviser.com': 250,
    'pragprog.com': 150,
    'treyhunner.com': 500,
    'scientistlive.com': 700,
    'fastcodesign.com': 350,
    'caroli.org': 250,
    'Caroli.org': 250,
    'coreyhaines.com': 700,
    'tjelvarolsson.com': 150,
    'aits.org': 400,
    'techbeacon.com': 1700,
    'curl.trillworks.com': 25,
    'oswalpalash.com': 25,
    'theoatmeal.com': 25,
    'opensource.googleblog.com': 300,
    'lean.org': 1050,
    'nyti.ms': 450,

    'quora.com': 150,
    'hackernoon.com': 175,
    'insidebigdata.com': 415,
    'fcc.im': 250,
    'lornemitchell.com': 400,
    'medium.com': 350,
    'wikipedia.org': 1250,
    'ASP.NET': default_domain_devation,
    'visualstudio.com': 1500,
    'devbridge.com': 200,
    'buff.ly': 600,
    'ben-evans.com': 200,
    'runlean.ly': 200,
    'dev.to': 200,
    'agileuprising.com': 200,
    'techcrunch.com': 100,
    'tobeagile.com': 225,
    'scrumalliance.org': 1650,
    'increment.com': 650,
    'solutionsiq.com': 550,
    'wordpress.com': 200,
    'eleganthack.com': 650,
    'brodzinski.com': 425,
    'blog.cleancoder.com': 650,
    'blog.juandelgado.es': 150,
    'blackswanfarming.com': 400,
    'thght.works': 400,
    'zumsteg.net': 350,
    'ribbonfarm.com': 1600,
    'curiosity.com': 250,
    'blogspot.com': 250,
    'builttoadapt.io': 150,
    'm.signalvnoise.com': 250,
    'mcfunley.com': 200,
    'dbader.org': 800,
    'martinfowler.com': 300,
    'thoughtworks.com': 300,
    'extremeuncertainty.com': 250,
    'facebook.com': 100,
    'michaelnygard.com': 150,
    'vitsoe.com': 150,
    'a16z.com': 150,
    'engineering.semantics3.com': 200,
    'blog.coffeeandcode.com': 150,
    'motherboard.vice.com': 150,
    'andrewchen.co': 150,
    'inc.com': 200,
    'startupsventurecapital.com': 200,
    'michaelfeathers.silvrback.com': 50,
    'blog.wingman-sw.com': 2000,
    'nataliewarnert.com': 350,
    'solutionsiq.in': 225,
    'itrevolution.com': 400,
    'linkedin.com': 50,
    'lnkd.in': 50,
    'mountaingoatsoftware.com': 500,
    'producthabits.com': 150,
    'keybase.io': 250,
    'agilealliance.org': 350,
    'codemash.org': 250,
    'stackoverflow.com': 200,
    'medium': 200,
    'fastcompany.com': 450,
    'blog.thedigitalcatonline.com': 750,
    'sandimetz.com': 150,
    'stickyminds.com': 400,
    'laughingmeme.org': 500,
    'ronjeffries.com': 250,
    'luis-goncalves.com': 850,
    'lucidchart.com': 200,
    'simpleisbetterthancomplex.com': 300,
    'jasonrudolph.com': 150,
    'engineering.onshift.com': 125,
    'eng.lyft.com': 100,
    'spikesandstories.com': 500,
    'labs.spotify.com': 1800,
    'lifehacker.com': 300,
    'wired.com': 550,
    'meowni.ca': 300,
    'microsoft.com': 500,
    'svpg.com': 200,
    'worldpositive.com': 1200,
    'aws.amazon.com': 4700,

    'leanqa.wordpress.com': 0,

    'snap-ci.com': default_domain_devation,
    'sumo.ly': default_domain_devation,
    'bit.ly': default_domain_devation,
    'shar.es': default_domain_devation,
    'go.shr.lc': default_domain_devation,
    'dlvr.it': default_domain_devation,
    'getpocket.com': default_domain_devation,
    'leanuxmas.com': default_domain_devation,
    'fb.me': default_domain_devation,
    'tinyurl.com': default_domain_devation,
    'ow.ly': default_domain_devation,

}

not_blog_urls = {
    'youtu.be',
    'youtube.com',
    'vimeo.com',

    'slideshare.net',

    # podcast
    'pca.st',

    'gist.github.com',

    'meetup.com',

    # conference
    'sched.co',

    # words out...
    'dev.to/dev3l',
    'softwaredev3loper.wordpress.com',
}


from turtle import Turtle

WIDTH = 1
HEIGHT = 1

class Ball(Turtle):

    def __init__(self):
        super().__init__()
        self.shape("circle")
        self.color("white")
        self.shapesize(stretch_wid=WIDTH, stretch_len=HEIGHT)
        self.penup()
        self.x_move = 10
        self.y_move = 10
        self.move_speed = 0.1

    def move(self):
        """Move the ball on the screen"""
        new_x = self.xcor() + self.x_move
        new_y = self.ycor() + self.y_move
        self.goto(new_x, new_y)

    def bounce_x(self):
        """
        Change the movement of the ball about the x-axis (bounced with paddles)
        in the opposite direction and increase ball's speed after bounce
        """
        self.x_move *= -1
        self.move_speed *= 0.9

    def bounce_y(self):
        """
        Change the movement of the ball about the y-axis in the opposite direction
        """
        self.y_move *= -1

    def reset_position(self):
        """return ball to the start position and change ball's movement in the opposite direction"""
        self.goto(0, 0)
        self.move_speed = 0.1
        self.bounce_x()


from django.contrib.auth import get_user_model
from django.db.models import Count
from django import forms
from django.utils.encoding import force_text

from ads.models import LineItem

User = get_user_model()

COUNTRIES_CHOICES = [(x[0], x[1]) for x in User.objects.exclude(country_code__isnull=True).values_list('country_code', 'country_name').annotate(count=Count('country_code')).order_by('-count')]
CITIES_CHOICES = [(x[0], x[0]) for x in User.objects.exclude(city__isnull=True).values_list('city').annotate(count=Count('city')).order_by('-count')]

class ArrayMultipleSelected(forms.SelectMultiple):
    def format_value(self, value):
        if value is None and self.allow_multiple_selected:
            return []
        if not isinstance(value, (tuple, list)):
            # This means it should be a comma delimited list of items so parse it
            value = value.split(',')
        return [force_text(v) if v is not None else '' for v in value]

class GeoAdder(forms.ModelForm):
    def __init__(self, *args, **kwargs):
        super(GeoAdder, self).__init__(*args, **kwargs)
        self.fields['targeting_countries'].widget = ArrayMultipleSelected(
            attrs={'class': 'multi-select-input'}, choices=COUNTRIES_CHOICES
        )
        self.fields['targeting_cities'].widget = ArrayMultipleSelected(
            attrs={'class': 'multi-select-input'}, choices=CITIES_CHOICES
        )

    class Meta:
        model = LineItem
        exclude = []