# Trying to reproduce results from
# paper :    arXiv : 1412.7828v2 [q-bio.QM] 4 Jan 2015
# Experiment name : Protein secondary structure prediction using
# LSTM networks.

# Model :
#  -- Standard stacked bidirectional LSTM with 3 layers.
#  -- (300 or 500) LSTM units in each layer
#  -- There is a FFN between h_rec and h with a skip connection. h_rec = ffn(h) + h
#  -- FFN is a two layer ReLU network with 300 or 500 units,
#  -- Introduce a FFN to combine output from forward and backward RNN
#  -- Has a ReLU with 200 or 400 hidden units.
#  -- The concatenation is regularized with 50% dropout.

import tensorflow as tf
import tensorflow
import numpy as np
from tensorflow.contrib import rnn
import pandas as pd
from tensorflow.contrib.rnn import *
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import tensor_shape
from tensorflow.python.framework import tensor_util
from tensorflow.python.layers import base as base_layer
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import clip_ops
from tensorflow.python.ops import init_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import nn_ops
from tensorflow.python.ops import partitioned_variables
from tensorflow.python.ops import random_ops
from tensorflow.python.ops import variable_scope as vs
from tensorflow.python.ops import variables as tf_variables
from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.util import nest
from tensorflow.python.ops.rnn_cell_impl import *
# from tensorflow.python.ops.rnn_cell_impl import _linear

model = RNNCell()
print(tf.__version__)

_BIAS_VARIABLE_NAME = "bias"
_WEIGHTS_VARIABLE_NAME = "kernel"

def _concat(prefix, suffix, static=False):
  """Concat that enables int, Tensor, or TensorShape values.
  This function takes a size specification, which can be an integer, a
  TensorShape, or a Tensor, and converts it into a concatenated Tensor
  (if static = False) or a list of integers (if static = True).
  Args:
    prefix: The prefix; usually the batch size (and/or time step size).
      (TensorShape, int, or Tensor.)
    suffix: TensorShape, int, or Tensor.
    static: If `True`, return a python list with possibly unknown dimensions.
      Otherwise return a `Tensor`.
  Returns:
    shape: the concatenation of prefix and suffix.
  Raises:
    ValueError: if `suffix` is not a scalar or vector (or TensorShape).
    ValueError: if prefix or suffix was `None` and asked for dynamic
      Tensors out.
  """
  if isinstance(prefix, ops.Tensor):
    p = prefix
    p_static = tensor_util.constant_value(prefix)
    if p.shape.ndims == 0:
      p = array_ops.expand_dims(p, 0)
    elif p.shape.ndims != 1:
      raise ValueError("prefix tensor must be either a scalar or vector, "
                       "but saw tensor: %s" % p)
  else:
    p = tensor_shape.as_shape(prefix)
    p_static = p.as_list() if p.ndims is not None else None
    p = (constant_op.constant(p.as_list(), dtype=dtypes.int32)
         if p.is_fully_defined() else None)
  if isinstance(suffix, ops.Tensor):
    s = suffix
    s_static = tensor_util.constant_value(suffix)
    if s.shape.ndims == 0:
      s = array_ops.expand_dims(s, 0)
    elif s.shape.ndims != 1:
      raise ValueError("suffix tensor must be either a scalar or vector, "
                       "but saw tensor: %s" % s)
  else:
    s = tensor_shape.as_shape(suffix)
    s_static = s.as_list() if s.ndims is not None else None
    s = (constant_op.constant(s.as_list(), dtype=dtypes.int32)
         if s.is_fully_defined() else None)

  if static:
    shape = tensor_shape.as_shape(p_static).concatenate(s_static)
    shape = shape.as_list() if shape.ndims is not None else None
  else:
    if p is None or s is None:
      raise ValueError("Provided a prefix or suffix of None: %s and %s"
                       % (prefix, suffix))
    shape = array_ops.concat((p, s), 0)
  return shape

def _zero_state_tensors(state_size, batch_size, dtype):
  """Create tensors of zeros based on state_size, batch_size, and dtype."""
  def get_state_shape(s):
    """Combine s with batch_size to get a proper tensor shape."""
    c = _concat(batch_size, s)
    c_static = _concat(batch_size, s, static=True)
    size = array_ops.zeros(c, dtype=dtype)
    size.set_shape(c_static)
    return size
  return nest.map_structure(get_state_shape, state_size)

class RNNCell(base_layer.Layer):
  """Abstract object representing an RNN cell.
  Every `RNNCell` must have the properties below and implement `call` with
  the signature `(output, next_state) = call(input, state)`.  The optional
  third input argument, `scope`, is allowed for backwards compatibility
  purposes; but should be left off for new subclasses.
  This definition of cell differs from the definition used in the literature.
  In the literature, 'cell' refers to an object with a single scalar output.
  This definition refers to a horizontal array of such units.
  An RNN cell, in the most abstract setting, is anything that has
  a state and performs some operation that takes a matrix of inputs.
  This operation results in an output matrix with `self.output_size` columns.
  If `self.state_size` is an integer, this operation also results in a new
  state matrix with `self.state_size` columns.  If `self.state_size` is a
  (possibly nested tuple of) TensorShape object(s), then it should return a
  matching structure of Tensors having shape `[batch_size].concatenate(s)`
  for each `s` in `self.batch_size`.
  """

  def __call__(self, inputs, state, scope=None):
    """Run this RNN cell on inputs, starting from the given state.
    Args:
      inputs: `2-D` tensor with shape `[batch_size x input_size]`.
      state: if `self.state_size` is an integer, this should be a `2-D Tensor`
        with shape `[batch_size x self.state_size]`.  Otherwise, if
        `self.state_size` is a tuple of integers, this should be a tuple
        with shapes `[batch_size x s] for s in self.state_size`.
      scope: VariableScope for the created subgraph; defaults to class name.
    Returns:
      A pair containing:
      - Output: A `2-D` tensor with shape `[batch_size x self.output_size]`.
      - New state: Either a single `2-D` tensor, or a tuple of tensors matching
        the arity and shapes of `state`.
    """
    if scope is not None:
      with vs.variable_scope(scope,
                             custom_getter=self._rnn_get_variable) as scope:
        return super(RNNCell, self).__call__(inputs, state, scope=scope)
    else:
      with vs.variable_scope(vs.get_variable_scope(),
                             custom_getter=self._rnn_get_variable):
        return super(RNNCell, self).__call__(inputs, state)

  def _rnn_get_variable(self, getter, *args, **kwargs):
    variable = getter(*args, **kwargs)
    trainable = (variable in tf_variables.trainable_variables() or
                 (isinstance(variable, tf_variables.PartitionedVariable) and
                  list(variable)[0] in tf_variables.trainable_variables()))
    if trainable and variable not in self._trainable_weights:
      self._trainable_weights.append(variable)
    elif not trainable and variable not in self._non_trainable_weights:
      self._non_trainable_weights.append(variable)
    return variable

  @property
  def state_size(self):
    """size(s) of state(s) used by this cell.
    It can be represented by an Integer, a TensorShape or a tuple of Integers
    or TensorShapes.
    """
    raise NotImplementedError("Abstract method")

  @property
  def output_size(self):
    """Integer or TensorShape: size of outputs produced by this cell."""
    raise NotImplementedError("Abstract method")

  def build(self, _):
    # This tells the parent Layer object that it's OK to call
    # self.add_variable() inside the call() method.
    pass

  def zero_state(self, batch_size, dtype):
    """Return zero-filled state tensor(s).
    Args:
      batch_size: int, float, or unit Tensor representing the batch size.
      dtype: the data type to use for the state.
    Returns:
      If `state_size` is an int or TensorShape, then the return value is a
      `N-D` tensor of shape `[batch_size x state_size]` filled with zeros.
      If `state_size` is a nested list or tuple, then the return value is
      a nested list or tuple (of the same structure) of `2-D` tensors with
      the shapes `[batch_size x s]` for each s in `state_size`.
    """
    with ops.name_scope(type(self).__name__ + "ZeroState", values=[batch_size]):
      state_size = self.state_size
      return _zero_state_tensors(state_size, batch_size, dtype)

def _linear(args,
            output_size,
            bias,
            bias_initializer=None,
            kernel_initializer=None):
  """Linear map: sum_i(args[i] * W[i]), where W[i] is a variable.
  Args:
    args: a 2D Tensor or a list of 2D, batch x n, Tensors.
    output_size: int, second dimension of W[i].
    bias: boolean, whether to add a bias term or not.
    bias_initializer: starting value to initialize the bias
      (default is all zeros).
    kernel_initializer: starting value to initialize the weight.
  Returns:
    A 2D Tensor with shape [batch x output_size] equal to
    sum_i(args[i] * W[i]), where W[i]s are newly created matrices.
  Raises:
    ValueError: if some of the arguments has unspecified or wrong shape.
  """
  if args is None or (nest.is_sequence(args) and not args):
    raise ValueError("`args` must be specified")
  if not nest.is_sequence(args):
    args = [args]

  # Calculate the total size of arguments on dimension 1.
  total_arg_size = 0
  shapes = [a.get_shape() for a in args]
  for shape in shapes:
    if shape.ndims != 2:
      raise ValueError("linear is expecting 2D arguments: %s" % shapes)
    if shape[1].value is None:
      raise ValueError("linear expects shape[1] to be provided for shape %s, "
                       "but saw %s" % (shape, shape[1]))
    else:
      total_arg_size += shape[1].value

  dtype = [a.dtype for a in args][0]

  # Now the computation.
  scope = vs.get_variable_scope()
  with vs.variable_scope(scope) as outer_scope:
    weights = vs.get_variable(
        _WEIGHTS_VARIABLE_NAME, [total_arg_size, output_size],
        dtype=dtype,
        initializer=kernel_initializer)
    if len(args) == 1:
      res = math_ops.matmul(args[0], weights)
    else:
      res = math_ops.matmul(array_ops.concat(args, 1), weights)
    if not bias:
      return res
    with vs.variable_scope(outer_scope) as inner_scope:
      inner_scope.set_partitioner(None)
      if bias_initializer is None:
        bias_initializer = init_ops.constant_initializer(0.0, dtype=dtype)
      biases = vs.get_variable(
          _BIAS_VARIABLE_NAME, [output_size],
          dtype=dtype,
          initializer=bias_initializer)
    return nn_ops.bias_add(res, biases)

class newLSTMCell(RNNCell):
  """Long short-term memory unit (LSTM) recurrent network cell.
  The default non-peephole implementation is based on:
    http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf
  S. Hochreiter and J. Schmidhuber.
  "Long Short-Term Memory". Neural Computation, 9(8):1735-1780, 1997.
  The peephole implementation is based on:
    https://research.google.com/pubs/archive/43905.pdf
  Hasim Sak, Andrew Senior, and Francoise Beaufays.
  "Long short-term memory recurrent neural network architectures for
   large scale acoustic modeling." INTERSPEECH, 2014.
  The class uses optional peep-hole connections, optional cell clipping, and
  an optional projection layer.
  """

  def __init__(self, num_units,
               use_peepholes=False, cell_clip=None,
               initializer=None, num_proj=None, proj_clip=None,
               num_unit_shards=None, num_proj_shards=None,
               forget_bias=1.0, state_is_tuple=True,
               activation=None, reuse=None):
    """Initialize the parameters for an LSTM cell.
    Args:
      num_units: int, The number of units in the LSTM cell
      use_peepholes: bool, set True to enable diagonal/peephole connections.
      cell_clip: (optional) A float value, if provided the cell state is clipped
        by this value prior to the cell output activation.
      initializer: (optional) The initializer to use for the weight and
        projection matrices.
      num_proj: (optional) int, The output dimensionality for the projection
        matrices.  If None, no projection is performed.
      proj_clip: (optional) A float value.  If `num_proj > 0` and `proj_clip` is
        provided, then the projected values are clipped elementwise to within
        `[-proj_clip, proj_clip]`.
      num_unit_shards: Deprecated, will be removed by Jan. 2017.
        Use a variable_scope partitioner instead.
      num_proj_shards: Deprecated, will be removed by Jan. 2017.
        Use a variable_scope partitioner instead.
      forget_bias: Biases of the forget gate are initialized by default to 1
        in order to reduce the scale of forgetting at the beginning of
        the training.
      state_is_tuple: If True, accepted and returned states are 2-tuples of
        the `c_state` and `m_state`.  If False, they are concatenated
        along the column axis.  This latter behavior will soon be deprecated.
      activation: Activation function of the inner states.  Default: `tanh`.
      reuse: (optional) Python boolean describing whether to reuse variables
        in an existing scope.  If not `True`, and the existing scope already has
        the given variables, an error is raised.
    """
    super(LSTMCell, self).__init__(_reuse=reuse)
    if not state_is_tuple:
      logging.warn("%s: Using a concatenated state is slower and will soon be "
                   "deprecated.  Use state_is_tuple=True.", self)
    if num_unit_shards is not None or num_proj_shards is not None:
      logging.warn(
          "%s: The num_unit_shards and proj_unit_shards parameters are "
          "deprecated and will be removed in Jan 2017.  "
          "Use a variable scope with a partitioner instead.", self)

    self._num_units = num_units
    self._use_peepholes = use_peepholes
    self._cell_clip = cell_clip
    self._initializer = initializer
    self._num_proj = num_proj
    self._proj_clip = proj_clip
    self._num_unit_shards = num_unit_shards
    self._num_proj_shards = num_proj_shards
    self._forget_bias = forget_bias
    self._state_is_tuple = state_is_tuple
    self._activation = activation or math_ops.tanh

    if num_proj:
      self._state_size = (
          LSTMStateTuple(num_units, num_proj)
          if state_is_tuple else num_units + num_proj)
      self._output_size = num_proj
    else:
      self._state_size = (
          LSTMStateTuple(num_units, num_units)
          if state_is_tuple else 2 * num_units)
      self._output_size = num_units

  @property
  def state_size(self):
    return self._state_size

  @property
  def output_size(self):
    return self._output_size

  def call(self, inputs, state):
    """Run one step of LSTM.
    Args:
      inputs: input Tensor, 2D, batch x num_units.
      state: if `state_is_tuple` is False, this must be a state Tensor,
        `2-D, batch x state_size`.  If `state_is_tuple` is True, this must be a
        tuple of state Tensors, both `2-D`, with column sizes `c_state` and
        `m_state`.
    Returns:
      A tuple containing:
      - A `2-D, [batch x output_dim]`, Tensor representing the output of the
        LSTM after reading `inputs` when previous state was `state`.
        Here output_dim is:
           num_proj if num_proj was set,
           num_units otherwise.
      - Tensor(s) representing the new state of LSTM after reading `inputs` when
        the previous state was `state`.  Same type and shape(s) as `state`.
    Raises:
      ValueError: If input size cannot be inferred from inputs via
        static shape inference.
    """
    num_proj = self._num_units if self._num_proj is None else self._num_proj
    sigmoid = math_ops.sigmoid

    if self._state_is_tuple:
      (c_prev, m_prev) = state
    else:
      c_prev = array_ops.slice(state, [0, 0], [-1, self._num_units])
      m_prev = array_ops.slice(state, [0, self._num_units], [-1, num_proj])

    dtype = inputs.dtype
    input_size = inputs.get_shape().with_rank(2)[1]
    if input_size.value is None:
      raise ValueError("Could not infer input size from inputs.get_shape()[-1]")
    scope = vs.get_variable_scope()
    with vs.variable_scope(scope, initializer=self._initializer) as unit_scope:
      if self._num_unit_shards is not None:
        unit_scope.set_partitioner(
            partitioned_variables.fixed_size_partitioner(
                self._num_unit_shards))
      # i = input_gate, j = new_input, f = forget_gate, o = output_gate
      lstm_matrix = _linear([inputs, m_prev], 4 * self._num_units, bias=True)
      i, j, f, o = array_ops.split(
          value=lstm_matrix, num_or_size_splits=4, axis=1)
      # Diagonal connections
      if self._use_peepholes:
        with vs.variable_scope(unit_scope) as projection_scope:
          if self._num_unit_shards is not None:
            projection_scope.set_partitioner(None)
          w_f_diag = vs.get_variable(
              "w_f_diag", shape=[self._num_units], dtype=dtype)
          w_i_diag = vs.get_variable(
              "w_i_diag", shape=[self._num_units], dtype=dtype)
          w_o_diag = vs.get_variable(
              "w_o_diag", shape=[self._num_units], dtype=dtype)

      if self._use_peepholes:
        c = (sigmoid(f + self._forget_bias + w_f_diag * c_prev) * c_prev +
             sigmoid(i + w_i_diag * c_prev) * self._activation(j))
      else:
        c = (sigmoid(f + self._forget_bias) * c_prev + sigmoid(i) *
             self._activation(j))

      if self._cell_clip is not None:
        # pylint: disable=invalid-unary-operand-type
        c = clip_ops.clip_by_value(c, -self._cell_clip, self._cell_clip)
        # pylint: enable=invalid-unary-operand-type
      if self._use_peepholes:
        m = sigmoid(o + w_o_diag * c) * self._activation(c)
      else:
        m = sigmoid(o) * self._activation(c)

      if self._num_proj is not None:
        with vs.variable_scope("projection") as proj_scope:
          if self._num_proj_shards is not None:
            proj_scope.set_partitioner(
                partitioned_variables.fixed_size_partitioner(
                    self._num_proj_shards))
          m = _linear(m, self._num_proj, bias=False)

        if self._proj_clip is not None:
          # pylint: disable=invalid-unary-operand-type
          m = clip_ops.clip_by_value(m, -self._proj_clip, self._proj_clip)
          # pylint: enable=invalid-unary-operand-type

    new_state = (LSTMStateTuple(c, m) if self._state_is_tuple else
                 array_ops.concat([c, m], 1))
    return m, new_state

"""
Source : arXiv:1403.1347v1  [q-bio.QM]  6 Mar 2014
:Deep Supervised and Convolutional Generative Stochastic Network for Protein Secondary Structure Prediction

The resulting training data including both feature and la-
bels has 57 channels (22 for PSSM, 22 for sequence, 2 for
terminals,  8  for  secondary  structure  labels,  2  for  solvent
accessibility  labels),  and  the  overall  channel  size  is  700.
"""

"""
Source : http://www.princeton.edu/~jzthree/datasets/ICML2014/dataset_readme.txt
It is currently in numpy format as a (N protein x k features) matrix. You can reshape it to (N protein x 700 amino acids x 57 features) first.

The 57 features are:
"[0,22): amino acid residues, with the order of 'A', 'C', 'E', 'D', 'G', 'F',
'I', 'H', 'K', 'M', 'L', 'N', 'Q', 'P', 'S', 'R', 'T', 'W', 'V', 'Y', 'X','NoSeq'"
"[22,31): Secondary structure labels, with the sequence of 'L', 'B', 'E', 'G', 'I', 'H',
'S', 'T','NoSeq'"
"[31,33): N- and C- terminals;"
"[33,35): relative and absolute solvent accessibility, used only for training.
(absolute accessibility is thresholded at 15; relative accessibility is normalized by the largest accessibility
value in a protein and thresholded at 0.15; original solvent accessibility is computed by DSSP)"
"[35,57): sequence profile. Note the order of amino acid residues is ACDEFGHIKLMNPQRSTVWXY and
it is different from the order for amino acid residues"

The last feature of both amino acid residues and secondary structure labels just mark end of the protein sequence.
"[22,31) and [33,35) are hidden during testing."

"The dataset division for the first ""cullpdb+profile_6133.npy.gz"" dataset is"
"[0,5600) training"
"[5605,5877) test "
"[5877,6133) validation"
"""

"""
cb513+profile_split1.npy.gz
cullpdb+profile_6133.npy.gz
cullpdb+profile_6133_filtered.npy.gz
"""

data = np.load('./data/cullpdb+profile_6133.npy.gz')
print(data.shape)
data = np.reshape(data, [6133, 700, 57])
print(data.shape)

# print(data.info())
cv_data = data[5600:5877, :]
train_data = np.load('./data/cullpdb+profile_6133_filtered.npy')
test_data = np.load('./data/cb513+profile_split1.npy')
train_data = np.reshape(train_data, [-1, 700, 57])
test_data = np.reshape(test_data, [-1, 700, 57])
print(train_data.shape)
print(test_data.shape)

print(train_data.shape)
# print(cv_data.shape)
print(test_data.shape)
# Split the train data
train_data_residues = train_data[:, :,  0:21]
train_data_secstruc = train_data[:, :, 22:30]
train_data_nctermin = train_data[:, :, 31:33]
train_data_rlabsolv = train_data[:, :, 33:35]
train_data_sequepro = train_data[:, :, 35:57]

# Checking shapes
print("Train data residues shape : ", train_data_residues.shape)
print("Train data secondary structue : ",train_data_secstruc.shape)
print("Train data n and c terminals : ", train_data_nctermin.shape)
print("Train data relative and absolute solvability : ", train_data_rlabsolv.shape)
print("Train data sequence profile : ", train_data_sequepro.shape)

train_data_input = train_data[:, :, np.r_[0:21, 36:57]]
train_data_otput = train_data[:, :, 23:31]
test_data_input = test_data[:, :, np.r_[0:21, 36:57]]
test_data_otput = test_data[:, :, 23:31]
# Checking shapes
# print("Train data input  shape : ", train_data_input.shape)
# print("Train data output shape : ", train_data_otput.shape)

learning_rate = 0.1
n_epochs = 1000
num_classes = 8
hidden_units = 300

class BrnnForPssp():

    def __init__(self, learning_rate, num_classes, hidden_units):

        # Initialize data and variables
        self.weights = tf.Variable(tf.random_uniform([hidden_units*2, num_classes], minval=-0.5, maxval=0.5))
        self.biases  = tf.Variable(tf.zeros([num_classes]))
        self.x = tf.placeholder("float", [None, 700, 42])
        self.y = tf.placeholder("float", [None, 700, 8])

        # Do the prediction

        # Remember to change activation to ReLU
        self.fw_rnn_cell1 = newLSTMCell(hidden_units, forget_bias=1.0)
        self.fw_rnn_cell2 = newLSTMCell(hidden_units, forget_bias=1.0)
        self.fw_rnn_cell3 = newLSTMCell(hidden_units, forget_bias=1.0)
        self.bw_rnn_cell1 = newLSTMCell(hidden_units, forget_bias=1.0)
        self.bw_rnn_cell2 = newLSTMCell(hidden_units, forget_bias=1.0)
        self.bw_rnn_cell3 = newLSTMCell(hidden_units, forget_bias=1.0)
        # self.fw_rnn_cells = [self.fw_rnn_cell1, self.fw_rnn_cell2, self.fw_rnn_cell3]
        # self.bw_rnn_cells = [self.bw_rnn_cell1, self.bw_rnn_cell2, self.bw_rnn_cell3]
        self.fw_rnn_cells = [self.fw_rnn_cell1]
        self.bw_rnn_cells = [self.bw_rnn_cell1]
        self.outputs, self.states_fw, self.states_bw = rnn.stack_bidirectional_dynamic_rnn(
                                                            self.fw_rnn_cells,
                                                            self.bw_rnn_cells,
                                                            self.x,
                                                            dtype=tf.float32)
        # self.output.shape is (?, 700, 600)
        self.outputs_reshaped = tf.reshape(self.outputs, [-1, 2*hidden_units])
        self.y_reshaped = tf.reshape(self.y, [-1, num_classes])
        # check importantFunctions.py : line-40 to see how it works
        # reference link  is :
        # https://stackoverflow.com/questions/38051143/no-broadcasting-for-tf-matmul-in-tensorflow
        self.y_predicted = tf.nn.softmax(tf.matmul(self.outputs_reshaped, self.weights) + self.biases)

        # Define the loss function
        self.loss = tf.nn.softmax_cross_entropy_with_logits(logits=self.y_predicted, labels=self.y_reshaped)

        # Define the trainer and optimizer
        self.optimizer = tf.train.GradientDescentOptimizer(learning_rate)
        self.trainer = self.optimizer.minimize(self.loss)

        # creating session and initializing variables
        self.sess = tf.Session()
        self.init = tf.global_variables_initializer()
        self.sess.run(self.init)

        # get accuracy
        self.get_equal = tf.equal(tf.argmax(self.y_reshaped, 1), tf.argmax(self.y_predicted, 1))
        self.accuracy = tf.reduce_mean(tf.cast(self.get_equal, tf.float32))

    def predict(self, x, y):
        result = self.sess.run(self.y_predicted, feed_dict={self.x: x, self.y: y})
        return result

    def optimize(self, x, y):
        print("Accuracy on next-to-be-trained batch : ", self.sess.run(self.accuracy, feed_dict={self.x: x, self.y: y}))
        result = self.sess.run(self.trainer, feed_dict={self.x: x, self.y: y})
        print("Accuracy obtained after training     : ", self.sess.run(self.accuracy, feed_dict={self.x: x, self.y: y}))

    def cross_validate(self, x, y):
        result = self.sess.run(self.accuracy, feed_dict={self.x: x, self.y: y})
        return result

    def build_graph(self, x, y):
        writer = tf.summary.FileWriter('./graphs/lstmForPSSP',self.sess.graph)

model = BrnnForPssp(learning_rate=learning_rate, num_classes=8, hidden_units=hidden_units)
print("Successfully created the model")

for i in range(n_epochs):
    if i % 10 == 0:
        x = test_data_input
        y = test_data_otput
        print(i, "td - ", model.cross_validate(x=x, y=y))
    j = i%50 + 1
    x = train_data_input[j*100:j*100+100, :]
    y = train_data_otput[j*100:j*100+100, :]
    print("Iteration number & batch number: ", i, j)
    model.optimize(x=x, y=y)


import glob
import os
import sys
import pdb
import os.path as osp

sys.path.append(os.getcwd())

import math
import joblib
import numpy as np
from isaacgym import gymapi, gymutil, gymtorch
import torch
from poselib.poselib.skeleton.skeleton3d import SkeletonMotion, SkeletonState
from poselib.poselib.visualization.common import plot_skeleton_motion_interactive, plot_skeleton_state
from pacer.utils.motion_lib import MotionLib
from pacer.utils.motion_lib_smpl import MotionLib as MotionLibSMPL
from uhc.smpllib.smpl_local_robot import Robot
from mujoco_py import load_model_from_path, load_model_from_xml, MjSim, MjViewer
from poselib.poselib.skeleton.skeleton3d import SkeletonTree
from pacer.utils import torch_utils

def clamp(x, min_value, max_value):
    return max(min(x, max_value), min_value)

# simple asset descriptor for selecting from a list

class AssetDesc:
    def __init__(self, file_name, flip_visual_attachments=False):
        self.file_name = file_name
        self.flip_visual_attachments = flip_visual_attachments

robot_cfg = {
                "mesh": False,
                "rel_joint_lm": False,
                "upright_start": True,
                "remove_toe": False,
                "real_weight_porpotion": True,
                "model": "smpl",
                "body_params": {},
                "joint_params": {},
                "geom_params": {},
                "actuator_params": {},
}
smpl_robot = Robot(
    robot_cfg,
    data_dir="data/smpl",
    masterfoot=False,
)
test_good = f"test_good.xml"
smpl_robot.write_xml(test_good)
sk_tree = SkeletonTree.from_mjcf(test_good)

asset_descriptors = [
    AssetDesc(test_good, False),
]

# parse arguments
args = gymutil.parse_arguments(
    description="Joint monkey: Animate degree-of-freedom ranges",
    custom_parameters=[{
        "name":
        "--asset_id",
        "type":
        int,
        "default":
        0,
        "help":
        "Asset id (0 - %d)" % (len(asset_descriptors) - 1)
    }, {
        "name": "--speed_scale",
        "type": float,
        "default": 1.0,
        "help": "Animation speed scale"
    }, {
        "name": "--show_axis",
        "action": "store_true",
        "help": "Visualize DOF axis"
    }])

if args.asset_id < 0 or args.asset_id >= len(asset_descriptors):
    print("*** Invalid asset_id specified.  Valid range is 0 to %d" %
          (len(asset_descriptors) - 1))
    quit()

# initialize gym
gym = gymapi.acquire_gym()

# configure sim
sim_params = gymapi.SimParams()
sim_params.dt = dt = 1.0 / 60.0
sim_params.up_axis = gymapi.UP_AXIS_Z
sim_params.gravity = gymapi.Vec3(0.0, 0.0, -9.81)
if args.physics_engine == gymapi.SIM_FLEX:
    pass
elif args.physics_engine == gymapi.SIM_PHYSX:
    sim_params.physx.solver_type = 1
    sim_params.physx.num_position_iterations = 6
    sim_params.physx.num_velocity_iterations = 0
    sim_params.physx.num_threads = args.num_threads
    sim_params.physx.use_gpu = args.use_gpu

sim_params.use_gpu_pipeline = False
if args.use_gpu_pipeline:
    print("WARNING: Forcing CPU pipeline.")

sim = gym.create_sim(args.compute_device_id, args.graphics_device_id,
                     args.physics_engine, sim_params)
if sim is None:
    print("*** Failed to create sim")
    quit()

# add ground plane
plane_params = gymapi.PlaneParams()
plane_params.normal = gymapi.Vec3(0.0, 0.0, 1.0)
gym.add_ground(sim, plane_params)

# create viewer
viewer = gym.create_viewer(sim, gymapi.CameraProperties())
if viewer is None:
    print("*** Failed to create viewer")
    quit()

# load asset
# asset_root = "pacer/data/assets"
asset_root = "./"
asset_file = asset_descriptors[args.asset_id].file_name

asset_options = gymapi.AssetOptions()
asset_options.fix_base_link = True
# asset_options.flip_visual_attachments = asset_descriptors[
#     args.asset_id].flip_visual_attachments
asset_options.use_mesh_materials = True

print("Loading asset '%s' from '%s'" % (asset_file, asset_root))
asset = gym.load_asset(sim, asset_root, asset_file, asset_options)

# set up the env grid
num_envs = 1
num_per_row = 6
spacing = 0
env_lower = gymapi.Vec3(-spacing, spacing, 0)
env_upper = gymapi.Vec3(spacing, spacing, spacing)

# position the camera
cam_pos = gymapi.Vec3(0, -10.0, 3)
cam_target = gymapi.Vec3(0, 0, 0)
gym.viewer_camera_look_at(viewer, None, cam_pos, cam_target)

# cache useful handles
envs = []
actor_handles = []

num_dofs = gym.get_asset_dof_count(asset)
print("Creating %d environments" % num_envs)
for i in range(num_envs):
    # create env
    env = gym.create_env(sim, env_lower, env_upper, num_per_row)
    envs.append(env)

    # add actor
    pose = gymapi.Transform()
    pose.p = gymapi.Vec3(0.0, 0, 0.0)
    pose.r = gymapi.Quat(0, 0.0, 0.0, 1)

    actor_handle = gym.create_actor(env, asset, pose, "actor", i, 1)
    actor_handles.append(actor_handle)

    # set default DOF positions
    dof_states = np.zeros(num_dofs, dtype=gymapi.DofState.dtype)
    gym.set_actor_dof_states(env, actor_handle, dof_states, gymapi.STATE_ALL)

# Setup Motion
body_ids = []
key_body_names = ["R_Ankle", "L_Ankle", "R_Wrist", "L_Wrist"]
for body_name in key_body_names:
    body_id = gym.find_actor_rigid_body_handle(envs[0], actor_handles[0],
                                               body_name)
    assert (body_id != -1)
    body_ids.append(body_id)

body_ids = np.array(body_ids)
_dof_body_ids = [
    1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21,
    22, 23
]
_dof_offsets = np.linspace(0, 23 * 3, 24).astype(int)
_dof_obs_size = 138
_num_actions = 69

# motion_file = "data/amass/pkls/amass_isaac_locomotion_upright.pkl"
# motion_file = "data/amass/pkls/amass_isaac_slowalk_upright.pkl"
# motion_file = "data/amass/pkls/amass_isaac_slowalk_upright_slim.pkl"
# motion_file = "data/amass/pkls/amass_isaac_slowalk_upright_slim_double.pkl"
# motion_file = "data/amass/pkls/amass_isaac_run_upright_slim_double.pkl"
# motion_file = "data/amass/pkls/amass_isaac_walk_upright_test_slim.pkl"

import ipdb; ipdb.set_trace()
motion_file = "output/renderings/rough_terrain.pkl"
motion_data = joblib.load(motion_file)
joblib.dump(motion_data, motion_file)

motion_keys = list(joblib.load(motion_file).keys())
motion_lib = MotionLib(motion_file=motion_file,
                       dof_body_ids=_dof_body_ids,
                       dof_offsets=_dof_offsets,
                       key_body_ids=body_ids,
                       device=args.compute_device_id)

current_dof = 0
speeds = np.zeros(num_dofs)
fps = 60
time_step = 0
dt = 1 / fps
rigidbody_state = gym.acquire_rigid_body_state_tensor(sim)
rigidbody_state = gymtorch.wrap_tensor(rigidbody_state)
rigidbody_state = rigidbody_state.reshape(num_envs, -1, 13)

gym.subscribe_viewer_keyboard_event(viewer, gymapi.KEY_LEFT, "previous")
gym.subscribe_viewer_keyboard_event(viewer, gymapi.KEY_RIGHT, "next")
gym.subscribe_viewer_keyboard_event(viewer, gymapi.KEY_G, "add")
gym.subscribe_viewer_keyboard_event(viewer, gymapi.KEY_P, "print")
motion_id = 0
motion_acc = set()
while not gym.query_viewer_has_closed(viewer):
    # step the physics

    motion_len = motion_lib.get_motion_length(motion_id).item()
    motion_time = time_step % motion_len

    motion_res = motion_lib.get_motion_state_smpl(
        torch.tensor([motion_id]).to(args.compute_device_id),
        torch.tensor([motion_time]).to(args.compute_device_id))

    root_pos, root_rot, dof_pos, root_vel, root_ang_vel, dof_vel, key_pos, smpl_params, limb_weights, pose_aa, rb_pos, rb_rot, body_vel, body_ang_vel = \
                motion_res["root_pos"], motion_res["root_rot"], motion_res["dof_pos"], motion_res["root_vel"], motion_res["root_ang_vel"], motion_res["dof_vel"], \
                motion_res["key_pos"], motion_res["motion_bodies"], motion_res["motion_limb_weights"], motion_res["motion_aa"], motion_res["rg_pos"], motion_res["rb_rot"], motion_res["body_vel"], motion_res["body_ang_vel"]

    if args.show_axis:
        gym.clear_lines(viewer)
    root_states = torch.cat([root_pos, root_rot, root_vel, root_ang_vel],
                        dim=-1).cpu().repeat(num_envs, 1)
    gym.set_actor_root_state_tensor(sim, gymtorch.unwrap_tensor(root_states))

    gym.simulate(sim)
    gym.refresh_rigid_body_state_tensor(sim)
    gym.fetch_results(sim, True)

    dof_pos = dof_pos.cpu().numpy()

    dof_states['pos'] = dof_pos
    speed = speeds[current_dof]

    # clone actor state in all of the environments
    for i in range(num_envs):
        gym.set_actor_dof_states(envs[i], actor_handles[i], dof_states,
                                 gymapi.STATE_POS)

        if args.show_axis:
            # get the DOF frame (origin and axis)
            dof_handle = gym.get_actor_dof_handle(envs[i], actor_handles[i], current_dof)
            frame = gym.get_dof_frame(envs[i], dof_handle)

            # draw a line from DOF origin along the DOF axis
            p1 = frame.origin
            p2 = frame.origin + frame.axis * 0.7
            color = gymapi.Vec3(1.0, 0.0, 0.0)
            gymutil.draw_line(p1, p2, color, gym, viewer, envs[i])

    # update the viewer
    gym.step_graphics(sim)
    gym.draw_viewer(viewer, sim, True)

    # Wait for dt to elapse in real time.
    # This synchronizes the physics simulation with the rendering rate.
    gym.sync_frame_time(sim)
    time_step += dt
    for evt in gym.query_viewer_action_events(viewer):
        if evt.action == "previous" and evt.value > 0:
            motion_id = (motion_id - 1) % len(motion_keys)
            print(
                f"Motion ID: {motion_id}. Motion length: {motion_len:.3f}. Motion Name: {motion_keys[motion_id]}"
            )
        elif evt.action == "next" and evt.value > 0:
            motion_id = (motion_id + 1) % len(motion_keys)
            print(
                f"Motion ID: {motion_id}. Motion length: {motion_len:.3f}. Motion Name: {motion_keys[motion_id]}"
            )
        elif evt.action == "add" and evt.value > 0:
            motion_acc.add(motion_keys[motion_id])
            print(f"Adding motion {motion_keys[motion_id]}")
        elif evt.action == "print" and evt.value > 0:
            print(motion_acc)
            time_step = 0

gym.destroy_viewer(viewer)
gym.destroy_sim(sim)


def solution(numbers):
    answer = []

    for i in range(len(numbers)):
        for j in range(i+1, len(numbers)):
            nums = numbers[i] + numbers[j]
            if nums not in answer:
                answer.append(nums)

    answer.sort()

    return answer

print(solution([2,1,3,4,1]))

두 개 뽑아서 더하기.py


from django.db import models
from rest_framework import serializers
from eventos.models import Evento, Tipo_Evento, SubTipo_Evento, Periodo, Convocatoria
from authentication.serializers import UsuarioSerializer
from authentication.models import Usuario
from datetime import datetime
import json

# class EventoSerializer(serializers.PrimaryKeyRelatedField, serializers.ModelSerializer):
class EventoSerializer(serializers.ModelSerializer):
    #autor = UsuarioSerializer()
    convocatoria = models.ForeignKey(Tipo_Evento)
    created_at = serializers.DateTimeField(format="%Y-%m-%d %H:%M:%S", read_only=True)

    class Meta:
        model = Evento
        fields = ('id', 'contenido', 'convocatoria', 'autor', 'titulo', 'created_at', 'updated_at')
        read_only_fields = ('created_at', 'updated_at')

    def create_evento(self, validated_data):
        return Evento.objects.create_evento(**validated_data)

    def update(self, evento, validated_data):
        try:
            # comprobando contenido
            if 'contenido' in validated_data.keys():
                new_contenido = validated_data.get('contenido')
                if not isinstance(new_contenido, basestring):
                    raise NameError("Error en el contenido")
                else:
                    evento.contenido = new_contenido

            # comprobando convocatoria
            # if 'convocatoria' in validated_data.keys():
            #     new_convocatoria = validated_data.get('convocatoria')
            #     try:
            #         convocatoria = Convocatoria.objects.get(codigo=new_convocatoria)
            #     except Tipo_Evento.DoesNotExist:
            #         raise NameError("Convocatoria incorrecto")
            #     evento.convocatoria = convocatoria

            # comprobando tipo
            if 'tipo' in validated_data.keys():
                new_tipo = validated_data.get('tipo')
                try:
                    tipo = SubTipo_Evento.objects.get(codigo=new_tipo)
                except SubTipo_Evento.DoesNotExist:
                    raise NameError("Tipo Evento incorrecto")
                evento.tipo = tipo

            # comprobando autor
            # if 'autor' in validated_data.keys():
            #     new_autor = validated_data.get('autor')
            #     autor = Usuario.objects.get(email=new_autor)
            #     if not isinstance(autor, Usuario):
            #         raise NameError("Autor incorrecto")
            #     else:
            #         evento.autor = autor

            # comprobando titulo
            if 'titulo' in validated_data.keys():
                new_titulo = validated_data.get('titulo')
                if not isinstance(new_titulo, basestring):
                    raise NameError("El titulo del evento no tiene formato correcto")
                else:
                    evento.titulo = new_titulo

            # comprobando desde
            if 'desde' in validated_data.keys():
                try:
                    new_desde = datetime.strptime(validated_data.get('desde')[:19], '%Y-%m-%dT%H:%M:%S')
                except:
                    raise NameError("Error en formato fecha desde")

            # comprobando hasta
            if 'hasta' in validated_data.keys():
                try:
                    new_hasta = datetime.strptime(validated_data.get('hasta')[:19], '%Y-%m-%dT%H:%M:%S')
                except:
                    raise NameError("Error en formato fecha hasta")

            # Actualizo el periodo
            if 'hasta' in validated_data.keys() or 'desde' in validated_data.keys():
                periodo = Periodo.objects.get(evento=evento)
                if 'hasta' in validated_data.keys():
                    periodo.end = new_hasta
                if 'desde' in validated_data.keys():
                    periodo.start = new_desde
                periodo.save()
            evento.save()
            return dict(status=True, data=EventoSerializer(Evento.objects.get(contenido=evento.contenido)).data)

        except NameError as e:
            return dict(status=False, message=e.message)
        except:
            return dict(status=False, message="Error en los parametros")

    def delete(self, evento):
        evento.delete()
        return dict(status=True)

class Tipo_EventoSerializer(serializers.ModelSerializer):

    class Meta:
        model = Tipo_Evento
        fields = ('id', 'codigo', 'nombre',)

    def create(self, validated_data):
        return Tipo_Evento.objects.create_tipo_evento(**validated_data)

    def update(self, tipo_evento, validated_data):
        try:
            # comprobando codigo
            if 'codigo' in validated_data.keys():
                new_codigo = validated_data.get('codigo')
                res = Tipo_Evento.objects.filter(codigo=new_codigo)
                if res.count() != 0:
                    raise NameError("El tipo de evento ya existe")
                elif not isinstance(new_codigo, basestring):
                    raise NameError("El codigo del tipo de evento no tiene formato correcto")
                else:
                    tipo_evento.codigo = new_codigo

            # comprobando nombre
            if 'nombre' in validated_data.keys():
                new_nombre = validated_data.get('nombre')
                if not isinstance(new_nombre, basestring):
                    raise NameError("El nombre del tipo de evento no tiene formato correcto")
                else:
                    tipo_evento.nombre = new_nombre

            tipo_evento.save()

            return dict(status=True, data=tipo_evento)

        except NameError as e:
            return dict(status=False, message=e.message)
        except:
            return dict(status=False, message="Error en los parametros")

    def delete(self, tipo_evento):
        tipo_evento.delete()
        return dict(status=True)

class SubTipo_EventoSerializer(serializers.ModelSerializer):

    class Meta:
        model = SubTipo_Evento
        fields = ('id', 'codigo', 'nombre',)

    def update(self, subtipo, validated_data):
        try:
            # comprobando codigo
            if 'codigo' in validated_data.keys():
                new_codigo = validated_data.get('codigo')
                res = SubTipo_Evento.objects.filter(codigo=new_codigo)
                if res.count() != 0:
                    raise NameError("El departamento ya existe")
                elif not isinstance(new_codigo, basestring):
                    raise NameError("El codigo del subtipo no tiene formato correcto")
                else:
                    subtipo.codigo = new_codigo

            # comprobando nombre
            if 'nombre' in validated_data.keys():
                new_nombre = validated_data.get('nombre')
                if not isinstance(new_nombre, basestring):
                    raise NameError("El nombre del subtipo no tiene formato correcto")
                else:
                    subtipo.nombre = new_nombre

            subtipo.save()

            return dict(status=True, data=subtipo)

        except NameError as e:
            return dict(status=False, message=e.message)
        except:
            return dict(status=False, message="Error en los parametros")

    def delete(self, subtipo):
        subtipo.delete()
        return dict(status=True)


########################################################################
# import python-library
########################################################################
# from import
import keras.models
from keras import backend as K
from keras import regularizers
from keras.layers import Input, Dense, BatchNormalization, Activation, Conv2D, PReLU, Flatten, MaxPool2D, Dropout
from keras.models import Model
import tensorflow as tf
import tensorflow_addons as tfa

########################################################################
# keras model
########################################################################
def get_model(input_dim, lr):
    """
    define the keras model
    the model based on the simple dense auto encoder
    (128*128*128*128*8*128*128*128*128)
    """

def get_model(input_dim, lr):
    """
    define the keras model
    the model based on the simple dense auto encoder
    (128*128*128*128*8*128*128*128*128)
    """
    regularizer = regularizers.L2(1e-4)
    x = Input(shape=(input_dim,))

    h = Dense(128, kernel_regularizer=regularizer)(x)
    h = BatchNormalization()(h)
    h = PReLU()(h)

    h = Dense(128, kernel_regularizer=regularizer)(h)
    h = BatchNormalization()(h)
    h = PReLU()(h)

    h = Dense(128, kernel_regularizer=regularizer)(h)
    h = BatchNormalization()(h)
    h = PReLU()(h)

    h = Dense(128, kernel_regularizer=regularizer)(h)
    h = BatchNormalization()(h)
    h = PReLU()(h)

    h = Dense(8)(h)
    h = BatchNormalization()(h)
    h = PReLU()(h)

    h = Dense(128, kernel_regularizer=regularizer)(h)
    h = BatchNormalization()(h)
    h = PReLU()(h)

    h = Dense(128, kernel_regularizer=regularizer)(h)
    h = BatchNormalization()(h)
    h = PReLU()(h)

    h = Dense(128, kernel_regularizer=regularizer)(h)
    h = BatchNormalization()(h)
    h = PReLU()(h)

    h = Dense(128, kernel_regularizer=regularizer)(h)
    h = BatchNormalization()(h)
    h = PReLU()(h)

    h = Dense(input_dim)(h)

    model = Model(inputs=x, outputs=h)

    model.compile(optimizer=tfa.optimizers.AdamW(weight_decay=lr),
                  loss='mean_squared_error')

    return model

#########################################################################

def load_model(file_path):
    return keras.models.load_model(file_path, compile=False)

def clear_session():
    K.clear_session()


__all__ = ['simple_test']

import inspect
import sys
import traceback

import reframe.utility.osext as osext
from reframe.core.exceptions import ReframeSyntaxError, SkipTestError, what
from reframe.core.fields import make_convertible
from reframe.core.fixtures import FixtureRegistry
from reframe.core.logging import getlogger, time_function
from reframe.core.pipeline import RegressionTest
from reframe.utility.versioning import VersionValidator

# NOTE: we should consider renaming this module in 4.0; it practically takes
# care of the registration and instantiation of the tests.

def _setvars(registry, variables):
    unset_vars = {}
    for test in registry:
        for name, val in variables.items():
            if '.' in name:
                testname, varname = name.split('.', maxsplit=1)
            else:
                testname, varname = test.__name__, name

            if testname == test.__name__:
                # Treat special values
                if val == '@none':
                    val = None
                else:
                    val = make_convertible(val)

                if not test.setvar(varname, val):
                    unset_vars.setdefault(test.__name__, [])
                    unset_vars[test.__name__].append(varname)

    return unset_vars

class TestRegistry:
    '''Regression test registry.

    The tests are stored in a dictionary where the test class is the key
    and the constructor arguments for the different instantiations of the
    test are stored as the dictionary value as a list of (args, kwargs)
    tuples.
    '''

    def __init__(self):
        self._tests = {}
        self._unset_vars = {}

    @property
    def unset_vars(self):
        return self._unset_vars

    @classmethod
    def create(cls, test, *args, **kwargs):
        obj = cls()
        obj.add(test, *args, **kwargs)
        return obj

    def add(self, test, *args, **kwargs):
        self._tests.setdefault(test, [])
        self._tests[test].append((args, kwargs))

    def setvars(self, variables):
        return _setvars(self, variables)

    @time_function
    def instantiate_all(self, reset_sysenv=0, external_vars=None):
        '''Instantiate all the registered tests.

        :param reset_sysenv: Reset valid_systems and valid_prog_environs after
            instantiating the tests. Bit 0 resets the valid_systems, bit 1
            resets the valid_prog_environs.

        :param external_vars: Test variables to set in the instantiated
            fixtures.

        '''

        # We first instantiate the leaf tests and then walk up their
        # dependencies to instantiate all the fixtures. Fixtures can only
        # establish their exact dependencies at instantiation time, so the
        # dependency graph grows dynamically.

        leaf_tests = []
        for test, variants in self._tests.items():
            for args, kwargs in variants:
                try:
                    kwargs['reset_sysenv'] = reset_sysenv
                    leaf_tests.append(test(*args, **kwargs))
                except SkipTestError as e:
                    getlogger().warning(
                        f'skipping test {test.__qualname__!r}: {e}'
                    )
                except Exception:
                    exc_info = sys.exc_info()
                    getlogger().warning(
                        f"skipping test {test.__qualname__!r}: "
                        f"{what(*exc_info)} "
                        f"(rerun with '-v' for more information)"
                    )
                    getlogger().verbose(traceback.format_exc())

        # Instantiate fixtures

        # Do a level-order traversal of the fixture registries of all leaf
        # tests, instantiate all fixtures and generate the final set of
        # candidate tests; the leaf tests are consumed at the end of the
        # traversal and all instantiated tests (including fixtures) are stored
        # in `final_tests`.
        unset_vars = {}
        final_tests = []
        fixture_registry = FixtureRegistry()
        while leaf_tests:
            tmp_registry = FixtureRegistry()
            while leaf_tests:
                c = leaf_tests.pop()
                reg = getattr(c, '_rfm_fixture_registry', None)
                final_tests.append(c)
                if reg:
                    tmp_registry.update(reg)

            # Instantiate the new fixtures and update the registry
            new_fixtures = tmp_registry.difference(fixture_registry)
            if external_vars:
                self._unset_vars.update(
                    _setvars(new_fixtures.uninst_tests(), external_vars)
                )

            leaf_tests = new_fixtures.instantiate_all()
            fixture_registry.update(new_fixtures)

        return final_tests

    def __iter__(self):
        '''Iterate over the registered test classes.'''
        return iter(self._tests.keys())

    def __contains__(self, test):
        return test in self._tests

def _register_test(cls, *args, **kwargs):
    '''Register a test and its construction arguments into the registry.'''

    mod = inspect.getmodule(cls)
    if not hasattr(mod, '_rfm_test_registry'):
        mod._rfm_test_registry = TestRegistry.create(cls, *args, **kwargs)
    else:
        mod._rfm_test_registry.add(cls, *args, **kwargs)

def _validate_test(cls):
    if not issubclass(cls, RegressionTest):
        raise ReframeSyntaxError('the decorated class must be a '
                                 'subclass of RegressionTest')

    if (cls.is_abstract()):
        getlogger().warning(
            f'skipping test {cls.__qualname__!r}: '
            f'test has one or more undefined parameters'
        )
        return False

    conditions = [VersionValidator(v) for v in cls._rfm_required_version]
    if (cls._rfm_required_version and
        not any(c.validate(osext.reframe_version()) for c in conditions)):

        getlogger().warning(f"skipping incompatible test "
                            f"'{cls.__qualname__}': not valid for ReFrame "
                            f"version {osext.reframe_version().split('-')[0]}")
        return False

    return True

def simple_test(cls):
    '''Class decorator for registering tests with ReFrame.

    The decorated class must derive from
    :class:`reframe.core.pipeline.RegressionTest`. This decorator is also
    available directly under the :mod:`reframe` module.

    .. versionadded:: 2.13
    '''
    if _validate_test(cls):
        for n in range(cls.num_variants):
            _register_test(cls, variant_num=n)

    return cls


from flask import Flask, request, render_template, session, redirect
from sklearn.metrics import confusion_matrix
import pandas as pd
import pickle

# load data
test = pd.read_csv("test.csv")

# preprocess
x_test1 = pd.read_csv('processed_test.csv')
y_test = test['price'].to_numpy().ravel()

#load model
clf = pickle.load( open('model.pkl','rb'))

app = Flask(__name__)

@app.route("/")
def home():
    return render_template('index.html')

@app.route("/report")
def report():
    '''Print prediction metric on test.csv'''
    y_hat = clf.predict(x_test1)

    cm = confusion_matrix(y_test, y_hat)
    cm = pd.DataFrame(cm, columns=['TrueClass_1','TrueClass_2','TrueClass_3','TrueClass_4'], index=['PredictClass_1','PredictClass_2','PredictClass_3','PredictClass_4'])
    #acc = clf.score(x_test, y_test)
    #print('confusion matrix:', '\n', cm, '\n', '='*80,  'prediction accuracy', acc)
    return cm.to_html(header="true", table_id="Confusion_matrix")

@app.route("/fancyreport")
def fancyreport():
    return render_template('report.html')

@app.route('/html')
def html():
    """Returns some custom HTML"""
    return """
    <title>This is a Hello World World Page</title>

Hello

World

    """

if __name__ == '__main__':
    app.run(host='127.0.0.1', port=8080, debug=True)


from functools import reduce
import sys
from typing import (
    Dict,
    List,
    Set,
    Tuple,
)

Cell = Tuple[int, int]
Path = Dict[Cell, Set[Cell]]

class HeightMap:
    def __init__(self, raw_values: List[str]) -> None:
        self.values: List[List[int]] = []
        self.low_points: List[Tuple[int, int]] = []
        self.risks: List[int] = []
        self.rows: int = 0
        self.cols: int = 0
        self.basins: Path = {}
        self.basin_sizes: List[int] = []

        self._init_values(raw_values)
        self._init_lows()
        self._init_risks()

    def _init_values(self, raw: List[str]) -> None:
        for l in raw:
            self.values.append([int(raw) for raw in l.strip()])

    def _init_lows(self):
        vals = self.values
        self.rows, self.cols = len(vals), len(vals[0])

        for r in range(self.rows):
            for c in range(self.cols):
                cur = vals[r][c]

                left = float("inf") if c == 0 else vals[r][c - 1]
                right = float("inf") if c == self.cols - 1 else vals[r][c + 1]
                up = float("inf") if r == 0 else vals[r - 1][c]
                down = float("inf") if r == self.rows - 1 else vals[r + 1][c]

                if cur < left and cur < right and cur < up and cur < down:
                    self.low_points.append((r, c))

    def _init_risks(self):
        for r, c in self.low_points:
            self.risks.append(self.values[r][c] + 1)

    def calc_basins(self):
        def dfs(r: int, c: int, val: int, path: Set[Cell]):
            cur = self.values[r][c]
            if cur == -1:
                return

            path.add((r, c))
            dirs = [(1, 0), (0, 1), (-1, 0), (0, -1)]
            self.values[r][c] = -1
            for dr, dc in dirs:
                nr, nc = r + dr, c + dc
                if not (0 <= nr < self.rows and 0 <= nc < self.cols):
                    continue
                nxt = self.values[nr][nc]
                if nxt <= val or nxt == 9:
                    continue
                dfs(nr, nc, self.values[nr][nc], path)
            self.values[r][c] = cur

        for r, c in self.low_points:
            self.basins[(r, c)] = set()
            path = self.basins[(r, c)]
            dfs(r, c, self.values[r][c], path)
            self.basin_sizes.append(len(path))

        self.basin_sizes.sort(reverse=True)

    def print_basins(self):
        for (r, c), path in self.basins.items():
            print(f"for {self.values[r][c]} @ ({r},{c}) -> total of {len(path)}")
            for pr, pc in path:
                print(f"    {self.values[pr][pc]} @ ({pr},{pc})")

def read_input(input_file: str) -> List[str]:
    lines: List[str] = []
    with open(input_file) as f:
        lines = f.readlines()
    return lines

def main(input_file: str) -> None:
    raw_vals = read_input(input_file)
    h_map = HeightMap(raw_vals)
    #  print(sum(h_map.risks))
    h_map.calc_basins()
    #  h_map.print_basins()
    print(reduce(lambda a, b: a * b, h_map.basin_sizes[:3]))

if __name__ == "__main__":
    main(sys.argv[1])


""" Simple demonstration of the new 3D visualization tool for Madagascar.
The program displays x-, y-, z- slices of a numpy array in 3D, allows user
to interactively drag the slices, includes useful features such as colorbar
and axis legend, and can output the figure to .png file with various resolution.

python seamai_data_demo.py --filepath seamai_data/data_train.npz
python seamai_data_demo.py --filepath seamai_data/data_test_1.npz

"""
import argparse

import numpy as np
from vispy.color import get_colormap, Colormap, Color
from vispy.gloo.util import _screenshot

from seismic_canvas import (SeismicCanvas, volume_slices, XYZAxis, Colorbar)
# Create the parser
my_parser = argparse.ArgumentParser(description='file path to viz')

# Add the arguments
my_parser.add_argument('--filepath',
                      # metavar='path',
                       type=str,
                       help='the path to file')

if __name__ == '__main__':

  # Execute the parse_args() method
  args = my_parser.parse_args()
 #volume = np.load('./data_train.npz')["data"]
  volume = np.load(args.filepath)["data"] #
  volume = volume.transpose(1,2,0)
  #print(volume.min(),volume.max())
  print(volume.shape) #(1006, 782, 590) z,x,y ==> (782,590,1006)
  axis_scales = (0.7, 0.5, 1) # anisotropic axes (stretch z-axis)

  # Colormaps.
  cmap='seismic'; clim=(-1500, 1500)   #-5195.5234 5151.7188
  # Get visual nodes ready.
  visual_nodes = volume_slices(volume,
    cmaps=cmap, clims=clim,
    # x_pos=32, y_pos=25, z_pos=93)
    x_pos=volume.shape[0]-1, y_pos=volume.shape[1]-1, z_pos=0) #(782, 590, 1006)
  xyz_axis = XYZAxis()
  colorbar = Colorbar(cmap=cmap, clim=clim, label_str='Data',
                      label_size=8, tick_size=6)

  # Run the canvas.
  canvas = SeismicCanvas(title='Data Interactive Demo',
                         visual_nodes=visual_nodes,
                         xyz_axis=xyz_axis,
                         colorbar=colorbar,
                         # Set the option below=0 will hide the colorbar region
                         # colorbar_region_ratio=0,
                         axis_scales=axis_scales,
                         # Manual camera setting below.
                         # auto_range=False,
                         # scale_factor=972.794,
                         # center=(434.46, 545.63, 10.26),
                         fov=30,
                         elevation=36,
                         azimuth=45,
                         zoom_factor=1.2 # >1: zoom in; <1: zoom out
                         )
  canvas.measure_fps()

  canvas.app.run()


import fastentrypoints
from setuptools import setup, find_packages
from setuptools.command.install import install

install_requirements = ['psutil']

version = '0.8.0'

try:
    import importlib
except ImportError:
    install_requirements.append('importlib')

try:
    import PyQt5
except ImportError:
    raise RuntimeError("Please install PyQt5 via your system package manager")

class InstallSystemdServiceFileInstall(install):
    """
    Installs systemd service user file during install.
    """

    def install_systemd_service(self):
        import os

        if os.path.isdir('/etc/systemd/user/'):
            from pkg_resources import Requirement, resource_filename
            from shutil import copy2
            service_path = resource_filename(Requirement.parse("actor"),
                                             "actor/static/actor.service")
            copy2(service_path, '/etc/systemd/user/')

    def run(self):
        install.run(self)
        self.install_systemd_service()

setup(
    name='actor',
    version=version,
    description='Actor the activity enforcer',
    long_description=open('README.md').read(),
    author='Tomas Babej',
    author_email='tomasbabej@gmail.com',
    license='AGPLv3',
    url='https://github.com/tbabej/actor',
    download_url='https://github.com/tbabej/actor/downloads',
    packages=find_packages(exclude=['tests']),
    include_package_data=True,
    install_requires=install_requirements,
    classifiers=[
        'Development Status :: 4 - Beta',
    ],
    entry_points={
        'console_scripts': [
            'actor = actor.core.client:main',
            'actor-daemon = actor.core.actord:main',
            'actor-debug = actor.core.actord:main_debug',
        ]
    },
    cmdclass={
        'install': InstallSystemdServiceFileInstall,
    }
)


from inspect import getsource
def inline(func):
    src = inspect.getsource(func).split('\n')
    for cut,ch in enumerate( src[1] ):
        if ch not in " \t":
            break
    return '\n'.join(ln[cut:] for ln in src[1:])

def foo(x):
    x *= 5
    A = [x]
    print(A)
    return [ x * A[0], A ]

def my_func():
    fn = lambda x : x+5*2
    print( inline(fn) )
    # print( inline(foo) )

def unpack(ns):
    return rf"""
for k,v in {ns}.items():
    exec(f"{{k}} = {{v}}")
"""

ns = { "asd":5, "x":"'q8weu'", "a":[123,5] }
print( unpack('ns') )
exec( unpack("ns") )
print( locals() )


#Uses python3
'''
    author: kamran mostafavi
    This program determines if the input graph is bipartite. In a bipartite graph, the end points of each edge has nodes that are different say
    one node is black and the next node is white. In mathematical terms if two nodes that are in the same level (i.e: distance from origin) are
    connected, then the graph is not bipartite.
'''
import sys
import queue
class bfs:
    def breathFirstSearch(self, adj, n, s, t):
        dist = [-1 for _ in range(n)] #init distance to infinity for all vertices
        dist[s]=0
        prev = [-1 for _ in range(n)]
        q=[]
        q.append(s)
        while len(q) > 0:
            u=q[0]
            q.remove(q[0])
            for v in adj[u]:
                if dist[v] == -1:
                    q.append(v)
                    dist[v]=dist[u]+1
                    prev[v]=u
                elif dist[v]==dist[u]:  #check to see if not bipartite (u and v are nodes that are connected and at same level
                    return 0
        return 1

def bipartite(adj,n):
    '''
        determine if the graph is bipartite - that is if two nodes in the same level/distance are connected the graph is not bipartite, otherwise it is.
        return 0 if not, 1 if bipartite
    '''
    g=bfs()
    return g.breathFirstSearch(adj,n,0,n-1)

if __name__ == '__main__':
    input = sys.stdin.read()
    data = list(map(int, input.split()))
    n, m = data[0:2]
    data = data[2:]
    edges = list(zip(data[0:(2 * m):2], data[1:(2 * m):2]))
    adj = [[] for _ in range(n)]
    for (a, b) in edges:
        adj[a - 1].append(b - 1)
        adj[b - 1].append(a - 1)
    print(bipartite(adj,n))


# your code goes here
from random import randint

class TrieNode(object):
    def __init__(self):
        self.left = None
        self.right = None
        self.parent = None
        # How many times this character appeared in the addition process
        self.counter = 0

class Trie(object):
    def __init__(self, root: TrieNode):
        self.root = root

    # def update(self, root):
    #     if (root.right and root.left):
    #         root.counter = root.right.counter + root.left.counter
    #     elif (root.left):
    #         root.counter = root.left.counter
    #     elif (root.right):
    #         root.counter = root.right.counter

    #     if (root.parent):
    #         self.update(root.parent)

    def insert(self, binary_num):
        self.insert_rec(binary_num, 0, self.root)

    def insert_rec(self, binary_num, level, current_node):
        if level >= len(binary_num):
            new_node = TrieNode()
            new_node.parent = current_node
            current_node.right = new_node
            current_node.counter += 1
            new_node.counter = 1
            return
        current_val = binary_num[level]
        current_node.counter += 1

        if current_val == '1':
            if current_node.right is None:
                new_node = TrieNode()
                new_node.parent = current_node
                current_node.right = new_node

            self.insert_rec(binary_num, level + 1, current_node.right)
        elif current_val == '0':
            if current_node.left is None:
                new_node = TrieNode()
                new_node.parent = current_node
                current_node.left = new_node

            self.insert_rec(binary_num, level + 1, current_node.left)

        # current_node.counter = 1
        # if current_node.left:
        #     current_node.counter += current_node.left.counter
        # if current_node.right:
        #     current_node.counter += current_node.right.counter

    def query(self, binary_num_query, binary_num_less_than):
        return self.query_rec(binary_num_query, binary_num_less_than, 0, self.root)

    def query_rec(self, binary_num_query, binary_num_less_than, level, current_node):
        if level >= len(binary_num_query) or current_node is None:
            return 0

        q = binary_num_query[level]
        k = binary_num_less_than[level]
        ans = 0

        # print(level, q, k)

        # if level == len(binary_num_query) - 1 and int(q) < int(k):
        #     return current_node.counter

        if q == '1' and k == '1':
            # we take right branch for sure
            ans += 0 if current_node.right is None else current_node.right.counter
            ans += self.query_rec(binary_num_query,
                                  binary_num_less_than, level + 1, current_node.left)

        elif q == '1' and k == '0':
            # we cannot take left branch, but we can investigate the right one
            ans += self.query_rec(binary_num_query,
                                  binary_num_less_than, level + 1, current_node.right)

        elif q == '0' and k == '1':
            # we take left branch for sure
            ans += 0 if current_node.left is None else current_node.left.counter
            ans += self.query_rec(binary_num_query,
                                  binary_num_less_than, level + 1, current_node.right)

        elif q == '0' and k == '0':
            # we cannot take righ branch, but we can investigate the left one
            ans += self.query_rec(binary_num_query,
                                  binary_num_less_than, level + 1, current_node.left)

        return ans

def xorLessK(arr, n, k):
    count = 0

    # check all subarrays
    for i in range(n):
        tempXor = 0
        for j in range(i, n):
            tempXor ^= arr[j]
            if (tempXor < k):
                count += 1

    return count

def get_binary_rep(number):
    INTEGER_LENGTH = 32

    binary_rep = bin(number)
    binary_rep = binary_rep[2:]

    binary_rep = ['0'] * (INTEGER_LENGTH - len(binary_rep)) + list(binary_rep)

    return binary_rep

def solve(arr, n, k):
    prev = 0

    root = TrieNode()
    trie = Trie(root)

    K = get_binary_rep(k)

    ans = 0

    for val in arr:
        prev = prev ^ val
        binary_rep = get_binary_rep(prev)
        # query here
        ans += trie.query(binary_rep, K)
        trie.insert(binary_rep)

    ans += trie.query(get_binary_rep(0), K)
    # print(trie.root.counter, trie.root.right, trie.root.left.counter)
    return ans

def generate_test_cases():

    trials = 1000
    while(trials > 0):
        print(trials)
        n = randint(0, 4)
        k = randint(0, n)

        arr = list()
        for i in range(n):
            arr.append(randint(0, 10))

        print("Took input")
        sol_naive = xorLessK(arr, n, k)
        print("solved naive")
        sol_real = solve(arr, n, k)

        if sol_naive != sol_real:
            print(n, k, arr)
            print("Naive: ", sol_naive, " Mine: ", sol_real)
            break
        trials -= 1

def main():
    n, k = map(int, input().split())

    arr = list()
    for _ in range(n):
        arr.append(int(input()))

    print(solve(arr, n, k))
    # generate_test_cases()

if __name__ == '__main__':
    main()


try:
    import json
except ImportError:
    import simplejson as json

from pyhole import plugin
from pyhole import utils

class Redmine(plugin.Plugin):
    """Provide access to the Redmine API"""

    def __init__(self, irc):
        self.irc = irc
        self.name = self.__class__.__name__
        self.disabled = False

        try:
            self.redmine = utils.get_config("Redmine")
            self.redmine_domain = self.redmine.get("domain")
            self.redmine_key = self.redmine.get("key")
            self.redmine_url = "https://%s:password@%s" % (
                    self.redmine_key, self.redmine_domain)
        except Exception:
            self.disabled = True

    @plugin.hook_add_command("rbugs")
    @utils.spawn
    def rbugs(self, params=None, **kwargs):
        """Redmine bugs for a user (ex: .rbugs <login>)"""
        if params and not self.disabled:
            login = params.split(" ", 1)[0]
            user_id = self._find_user(login)

            i = 0
            issues = self._find_issues(user_id)
            for i, issue in enumerate(issues):
                if i <= 4:
                    self._find_issue(issue["id"])
                else:
                    self.irc.reply("[...] truncated last %d bugs" % (
                            len(issues) - i))
                    break
            else:
                if i <= 0:
                    self.irc.reply("No Redmine bugs found for '%s'" % login)
        else:
            self.irc.reply(self.rbugs.__doc__)

    @plugin.hook_add_keyword("rm")
    @utils.spawn
    def keyword_rm(self, params=None, **kwargs):
        """Retrieve Redmine bug information (ex: RM12345)"""
        if params and not self.disabled:
            params = utils.ensure_int(params)
            if params:
                self._find_issue(params)

    @plugin.hook_add_msg_regex("https?:\/\/redmine\..*/issues")
    def _watch_for_rm_bug_url(self, params=None, **kwargs):
        """Watch for Redmine bug URLs"""
        try:
            line = kwargs["full_message"].split("/")
            for i, word in enumerate(line):
                if word == "issues":
                    bug_id = line[i + 1].split(" ", 1)[0]
                    self.keyword_rm(bug_id)
        except TypeError:
            return

    def _find_issues(self, user_id):
        """Find all issues for a Redmine user"""
        url = "%s/issues.json?assigned_to_id=%s" % (
                self.redmine_url, user_id)
        response = self.irc.fetch_url(url, self.name)
        if not response:
            return

        return json.loads(response.read())["issues"]

    def _find_user(self, login):
        """Find a specific Redmine user"""
        for user in self._find_users():
            if login == user["login"]:
                return user["id"]

        for user in self._find_users(100):
            if login == user["login"]:
                return user["id"]

    def _find_users(self, offset=None):
        """Find all Redmine users"""
        if offset:
            url = "%s/users.json?limit=100&offset=%d" % (
                    self.redmine_url, offset)
        else:
            url = "%s/users.json?limit=100" % self.redmine_url
        response = self.irc.fetch_url(url, self.name)
        if not response:
            return

        return json.loads(response.read())["users"]

    def _find_issue(self, issue_id):
        """Find and display a Redmine issue"""
        url = "%s/issues/%s.json" % (self.redmine_url, issue_id)
        response = self.irc.fetch_url(url, self.name)
        if not response:
            return

        try:
            issue = json.loads(response.read())["issue"]
        except Exception:
            return

        self.irc.reply("RM %s #%s: %s [Status: %s, Assignee: %s] "
                "https://%s/issues/show/%s" % (
                issue["tracker"]["name"], issue["id"], issue["subject"],
                issue["status"]["name"],
                issue.get("assigned_to", {}).get("name", "N/A"),
                self.redmine_domain, issue["id"]))


from jpype import *
import tempfile
import os

stanford_path = '/root/NLP/stanford-corenlp/'
jars = ['ejml-0.23.jar', 'javax.activation-api-1.2.0.jar', 'javax.activation-api-1.2.0-sources.jar', 'javax.json-api-1.0-sources.jar', 'javax.json.jar', 'jaxb-api-2.4.0-b180830.0359.jar', 'jaxb-api-2.4.0-b180830.0359-sources.jar', 'jaxb-core-2.3.0.1.jar', 'jaxb-core-2.3.0.1-sources.jar', 'jaxb-impl-2.4.0-b180830.0438.jar', 'jaxb-impl-2.4.0-b180830.0438-sources.jar', 'joda-time-2.9-sources.jar', 'joda-time.jar', 'jollyday-0.4.9-sources.jar', 'jollyday.jar', 'protobuf.jar', 'slf4j-api.jar', 'slf4j-simple.jar', 'stanford-corenlp-3.9.2.jar', 'stanford-corenlp-3.9.2-javadoc.jar', 'stanford-corenlp-3.9.2-models.jar', 'stanford-corenlp-3.9.2-sources.jar', 'xom-1.2.10-src.jar', 'xom.jar']

classpath=':'.join([stanford_path + jar for jar in jars])+':'+'/root/NLP'
jpath = getDefaultJVMPath()

def JVM_init():
    if not isJVMStarted():
        startJVM(jpath, '-ae', '-Djava.class.path=%s' % (classpath))

    if not isThreadAttachedToJVM():
        attachThreadToJVM()

def parse(text):
    JVM_init()
    BiohubNLP = JClass('BiohubNLP')
    start, end, match = BiohubNLP.Parse(text)
    start = start[:]
    end = end[:]
    match = match[:]
    return start, end, match

if __name__ == "__main__":
    #print(parse("I will leave for the US next month."))
    print(parse("yesterday"))


import json
import pandas as pd
import numpy as np
import copy
import os
import geopandas as gpd

def get_pre_ripa_data(cols_cfgs, **cfgs):

    base_url, description_url, race_url = [
      "http://seshat.datasd.org/pd/vehicle_stops_2014_datasd_v1.csv",
      "http://seshat.datasd.org/pd/vehicle_stops_search_details_2014_datasd.csv",
      "http://seshat.datasd.org/pd/vehicle_stops_race_codes.csv"
    ]

    years, races, cols, outdir = copy.deepcopy(cfgs['years']), cfgs['races'], cfgs['pre-RIPA_cols'], cfgs['outdir']
    if not os.path.exists(outdir):
        os.makedirs(outdir)

    stop_df = pd.DataFrame()

    # if years includes only 2019, no pre-ripa data needed
    if min(years) > 2018:
        return

    # 2018 data is located in the 2017 csv file, handle case when 2017 is not in years but 2018 is
    if 2018 in years:
        years.remove(2018)
        if 2017 not in years:
            years.append(2017)

    if 2019 in years:
        years.remove(2019)

    # map how race is indicated in config file to how it is indicated in dataset (e.g., white -> W)
    race_mapper = {'white':'W','black':'B','hispanic':'H'}
    if races[0] == 'all':
        race_filter = []
    else:
        race_filter = [race_mapper[r] for r in races]

    # indicate which columns belong to the basic details dataset, and which belong to search details dataset
    main_cols = list(set(cols_cfgs['pre-ripa'].keys()).intersection(set(cols)))
    detail_cols = list(set(cols_cfgs['pre-ripa_details'].keys()).intersection(set(cols)))

    for year in set(years):

        # Get urls for stop data from year
        url = base_url.replace('2014',str(year))
        description_url_c = description_url.replace('2014',str(year))
        df = pd.read_csv(url,usecols = main_cols)

        default_date = '{}-12-31 23:39:00'.format(year)
        df['date_time'].fillna(default_date)

        # handle case when either 2017 or 2018 is in dataset, but not both
        if year == 2017:
            if 2017 not in cfgs['years']:
                df = df[df['date_time'].apply(lambda x: pd.to_datetime(x).year) == 2018]
            elif 2018 not in cfgs['years']:
                df = df[df['date_time'].apply(lambda x: pd.to_datetime(x).year) == 2017]

        # if race is to be included and not all races are needed, filter by neede races
        # join race description to basic details dataset
        if 'subject_race' in df.columns:
            if races[0] != 'all':
                df = df[df.subject_race.isin(race_filter)]
            df_race = pd.read_csv(race_url)
            df = df.merge(df_race,left_on = 'subject_race',right_on='Race Code')

        # handle when no columns from details dataset are specified in config file
        if len(detail_cols) > 1:
            df_details = pd.read_csv(description_url_c,usecols=detail_cols)
            df = df.merge(df_details,on='stop_id')
        stop_df = pd.concat([stop_df,df])

    stop_df.reset_index(inplace=True,drop=True)
    file_path = os.path.join(outdir,'pre_ripa.csv')
    stop_df.to_csv(file_path,index=False)

def get_ripa_data(cols_cfgs, **cfgs):

    years, races, r_cols, outdir = copy.deepcopy(cfgs['years']), cfgs['races'], cfgs['RIPA_cols'], cfgs['outdir']
    stop_df = pd.DataFrame()

    if not os.path.exists(outdir):
        os.makedirs(outdir)

    # If no data from 2018 or 2019 is needed, no data from ripa datasets are needed
    if max(years) < 2018:
        return

    filter_years = list(set(years).intersection(set([2018,2019])))

    RIPA_urls = [
     "http://seshat.datasd.org/pd/ripa_stops_datasd_v1.csv",
     "http://seshat.datasd.org/pd/ripa_actions_taken_datasd.csv",
     "http://seshat.datasd.org/pd/ripa_contraband_evid_datasd.csv",
     "http://seshat.datasd.org/pd/ripa_prop_seize_type_datasd.csv",
     "http://seshat.datasd.org/pd/ripa_race_datasd.csv",
     "http://seshat.datasd.org/pd/ripa_stop_reason_datasd.csv",
     "http://seshat.datasd.org/pd/ripa_stop_result_datasd.csv"
    ]

    # map how race is indicated in config file to how it is indicated in dataset (e.g., white -> W)
    race_mapper = {'white':'White','black':'Black/African American',
    'hispanic':'Hispanic/Latino/a','other':['Native American','Pacific Islander'],
    'asian':['Middle Eastern or South Asian','Asian']}
    # create list of races to filter by
    race_filter = []
    if races[0] != 'all':
        for r in races:
            if r in ['asian','other']:
                race_filter.append(race_mapper[r.lower()][0])
                race_filter.append(race_mapper[r.lower()][1])
            else:
                race_filter.append(race_mapper[r.lower()])

    for url, cols in zip(RIPA_urls,cols_cfgs.items()):
        # specify which cols are needed from each dataset
        usecols  = list(set(r_cols).intersection(set(cols[1].keys())))
        df = pd.read_csv(url,usecols=usecols)
        # If years don't include both 2018 and 2019, filter out unneeded year
        if ('date_stop' in df.columns) and (len(filter_years) != 2):
            df = df[df.date_stop.apply(lambda x: pd.to_datetime(x).year).isin(filter_years)]
        # filter out unneeded races
        if 'race' in df.columns:
            if races[0] != 'all':
                df = df[df.race.isin(race_filter)]
        if stop_df.empty:
            stop_df = df.copy()
        else:
            stop_df = stop_df.merge(df,on=['stop_id','pid'])

    stop_df.reset_index(inplace=True,drop=True)

    file_path = os.path.join(outdir,'ripa.csv')

    stop_df.to_csv(file_path,index=False)

# Read in service area data and write to file
def get_service_area_info(**cfgs):

    pd_beats_url = 'http://seshat.datasd.org/sde/pd/pd_beats_datasd.geojson'
    sa = gpd.read_file(pd_beats_url)
    outdir = cfgs['outdir']
    if not os.path.exists(outdir):
        os.makedirs(outdir)
    file_path = os.path.join(outdir,'service_area.geojson')
    sa.to_file(file_path,driver="GeoJSON")

def get_stops_data(cols_cfgs, **cfgs):
    get_pre_ripa_data(cols_cfgs, **cfgs)
    get_ripa_data(cols_cfgs, **cfgs)
    get_service_area_info(**cfgs)


from flask import Flask, render_template, jsonify, request
from name import find_name, is_name
from nlp_fin import rest

import nltk.classify.util
from nltk.classify import NaiveBayesClassifier
from nltk.corpus import names
from nltk.corpus import stopwords
import json
import math
import re
from nltk.tokenize import sent_tokenize, word_tokenize

app = Flask(__name__)
app.debug = True

@app.route('/')
def index():
    return render_template("home.html")

@app.route('/change_name', methods=['POST'])
def requester():
	if request.json:
		json = request.get_json(silent=True)
		print(json)
		text = json["text"]
		name = find_name(text)
		data = jsonify({'name': name})
		print(text)
		return data

@app.route('/api/check_name', methods=['POST', 'GET'])
def checker():
	if request.json:
		json = request.get_json(silent=True)
		print(json)
		text = json["name"]
		result = is_name(text)
		print(result)
		if(result):
			return("1")
		else:
			return("0")

@app.route('/result', methods=['POST', 'GET'])
def result() :
	if request.method == 'POST' :
		word = str(request.form['text']).lower()
		print(word)
		fin = rest(word)
		print(str(fin))
		return str(fin)
		# rest(word)"ram with 4gb and budget with 12000rs."

if __name__ == '__main__':
    app.run()


import os
import pathlib
import shutil

from PyQt5 import QtWidgets
from PyQt5 import QtGui
from PyQt5.QtGui import QStandardItemModel, QStandardItem
from PyQt5.QtWidgets import QAction, QMenu, QDesktopWidget, QMessageBox, QPushButton
from datetime import datetime
from ui_files.mainWindow import Ui_MainWindow

from create_archive.createArchive import create
from create_archive.operation import Operation
from fileSystem.filehandler import FileHandler
from archive_process.compress import Compress
from archive_process.extract import Extract
from running.popup import Popup

class RunHandler(QtWidgets.QMainWindow):
    def __init__(self, parent=None):
        super().__init__(parent)
        self.model_treeView = None
        self.drag_and_drop_activate = False
        self.ops_data = []
        self.ops_file_path = os.path.join(os.path.dirname(__file__), '..', 'ops', 'ops_file.ops')
        self.temp_file_path = os.path.join(os.path.expanduser('~/.archiveManager'), 'temp')
        self.parent = Ui_MainWindow()
        self.parent.setupUi(self)

        self.compress = Compress(self)
        self.extract = Extract(self)
        self.createArchive = create(self)
        self.operation = Operation()
        self.fileHandler = FileHandler(self)
        self.popup = Popup(self)

        self.createArchive.parent.cancel_button.clicked.connect(self.close_create_archive_form)

        self.icon_type = {'folder': ':/icons/icons/folder.svg',
                          'file': ':/icons/icons/empty-page.svg'}
        self.folder_list = []
        self.file_list = []

        self.parent.lineEdit_path.setTextMargins(0, 0, 0, 0)
        icon = QtGui.QIcon(self.icon_type['folder'])
        self.parent.lineEdit_path.addAction(icon, QtWidgets.QLineEdit.LeadingPosition)
        self.parent.pushButton_compress.setVisible(False)
        self.parent.widget_drag_and_drop.setVisible(False)

        self.mainLoop()

        # Open Window Center
        qtRectangle = self.frameGeometry()
        centerPoint = QDesktopWidget().availableGeometry().center()
        qtRectangle.moveCenter(centerPoint)
        self.move(qtRectangle.topLeft())
        qtRectangle = self.frameGeometry()
        centerPoint = QDesktopWidget().availableGeometry().center()
        qtRectangle.moveCenter(centerPoint)
        self.move(qtRectangle.topLeft())

    def mainLoop(self):
        self.custom_toolBar()
        self.parent.pushButton_add_folder.clicked.connect(self.compress.add_folder_clicked)
        self.parent.pushButton_add_file.clicked.connect(self.compress.add_file_clicked)

        self.parent.pushButton_compress.clicked.connect(self.compress.compress_file)
        self.parent.pushButton_extract.clicked.connect(self.extract.move_zip_folder)

    def custom_toolBar(self):
        # TODO Menu Action List
        self.menu = QMenu()
        self.new_archive_action = QAction("New Archive", self)
        self.new_archive_action.triggered.connect(self.create_archive_file)
        self.open_archive_action = QAction("Open Archive", self)
        self.open_archive_action.triggered.connect(self.extract.open_archive_file)

        # Added menu item after create menu
        self.menu.addAction(self.new_archive_action)
        self.menu.addSeparator()
        self.menu.addAction(self.open_archive_action)

        self.parent.menu_button.setMenu(self.menu)

    def create_archive_file(self):
        # self.clearQTreeWidget(self.parent.treeView)
        self.createArchive.show()
        self.createArchive.add_location()
        self.createArchive.parent.create_button.clicked.connect(self.connect_path)

        # Clicked new_archive_create. changed type to pushButton_compress_extract.
        self.parent.pushButton_compress.setVisible(True)
        self.parent.pushButton_extract.setVisible(False)

    def close_create_archive_form(self):
        self.parent.pushButton_compress.setVisible(False)
        self.parent.pushButton_extract.setVisible(True)
        self.createArchive.close()

    def connect_path(self):
        self.createArchive.set_archive_detail()
        self.parent.widget_drag_and_drop.setVisible(True)
        self.drag_and_drop_activate = True
        self.setAcceptDrops(True)
        # COMMENT enabled to lineEdit_path edit or write
        self.parent.lineEdit_path.setText(self.operation.save_path)
        self.parent.lineEdit_path.setReadOnly(False)
        self.parent.pushButton_add_file.setEnabled(True)
        self.parent.pushButton_add_folder.setEnabled(True)

    def write_treeView(self, target_path=None):
        # Klasör yapısını göstermek için bir QFileSystemModel oluştur
        self.model_treeView = QStandardItemModel()
        self.model_treeView.setHorizontalHeaderLabels(['Name', 'Size', 'Type', 'Modified'])

        self.parent.treeView.setModel(self.model_treeView)

        self.populate_tree(target_path, self.model_treeView.invisibleRootItem())

        # Sütun genişliklerini ayarla
        self.parent.treeView.setColumnWidth(0, 300)
        self.parent.treeView.setColumnWidth(1, 100)
        self.parent.treeView.setColumnWidth(2, 200)
        self.parent.treeView.setColumnWidth(3, 180)

    def populate_tree(self, folder_path, parent_item):
        for item_name in os.listdir(folder_path):
            item_path = os.path.join(folder_path, item_name)
            item = QStandardItem(item_name)

            if os.path.isdir(item_path):
                item.setIcon(QtGui.QIcon(self.icon_type['folder']))
                self.populate_tree(item_path, item)
                parent_item.appendRow(item)
            else:
                item.setIcon(QtGui.QIcon(self.icon_type['file']))

                size_item = QStandardItem(str(self.fileHandler.file_size(mode=False, total=str(os.path.getsize(item_path)))))
                type_item = QStandardItem(str(self.fileHandler.select_file_type(os.path.splitext(item_name)[1], mode=False)))
                modified_item = QStandardItem(
                    str(datetime.fromtimestamp(
                        pathlib.Path(item_path).stat().st_mtime
                    )).split('.')[0]
                )

                parent_item.appendRow([item, size_item, type_item, modified_item])

    def read_ops_file(self):
        if os.path.exists(self.ops_file_path):
            with open(self.ops_file_path, 'r') as ops:
                self.ops_data = ops.readlines()

            self.ops_data = [item.replace('\n', '') for item in self.ops_data]

    def clearQTreeWidget(self, tree):
        # yeni arşiv oluşturmak istendiğinde tüm itemlarım temizlenmesi sağlanır.
        self.createArchive.parent.archive_name.clear()
        self.parent.lineEdit_path.clear()
        self.createArchive.parent.comboBox_archive_location.clear()
        self.parent.pushButton_add_folder.setEnabled(False)
        self.parent.pushButton_add_file.setEnabled(False)

    def dragEnterEvent(self, event):
        if event.mimeData().hasUrls():
            event.accept()
        else:
            event.ignore()

    def dropEvent(self, event):
        if self.drag_and_drop_activate:
            self.parent.pushButton_compress.setEnabled(True)
            if event.mimeData().hasUrls():
                drop_urls = [url.toLocalFile() for url in event.mimeData().urls()]
                self.compress.move_folder_or_files(drop_urls)

    def closeEvent(self, event: QtGui.QCloseEvent) -> None:
        if os.path.exists(self.temp_file_path):
            shutil.rmtree(self.temp_file_path)
        else:
            pass


game = [[0,0,0],[0,0,0],[0,0,0]] #list of lists

def game_board(game_map,player=0, row=0, column=0,just_display=False):
	#takes multiple parameters
	try:
		print ("   0  1  2")
		## header for row select
		if not just_display:
			game_map[row][column] = player
		# if just display is false then it will use this function
		# and this block makes the actual modification to the map
		for count, rows in enumerate(game_map):
			print (count , rows)
		# this for loop will just display the actual gameboard
			if count == 2 :
				print()
			# prints an empty line when the board is finished printing for readability
		return game_map ## returns the modified map
	except IndexError as e:
		print("You didnt enter your row/column as 0 1 or 2")
	except Exception as e:
		print("A serious fault occured")

game=game_board(game, just_display = True)
#sets the game variable to the new modified map
#without the need for a global variable
game=game_board(game, player=1,row=2, column=1)

game=game_board(game,just_display = True)


from sympy.parsing.mathematica import mathematica

import sympy as sp
from sympy.functions.special.bessel import besselj
from sympy.parsing.sym_expr import SymPyExpression
from sympy import latex

def get_ylm_FTs():
    results = []
    with open("SphericalHarmonicsResults_10.txt", 'r') as file:
        for line in file:
            # Split the line into columns based on tabs
            columns = line.strip().split('\t')
            l = int(columns[0])
            m = int(columns[1])
            result = mathematica(columns[2], {'BesselJ[n, rho]':'besselj(n, rho)'})
            results.append([l,m,result])
    replacements = {'pi':'jnp.pi',
                'cos':'jnp.cos',
                'sin':'jnp.sin',
                'sqrt':'jnp.sqrt',
                'I':'1j'}
    def replace_all(text, replacements):
        for i in replacements:
            text = text.replace(str(i), str(replacements[i]))
        return text
    jax_exprs = {}
    for l, m, res in results:
        jax_exprs[(l,m)]=replace_all(str(res), replacements)
    return jax_exprs


from django.core.management.base import BaseCommand, CommandError
from newsroom.models import Article, Author
from django.utils import timezone
from django.db.models import Count

def process(year, author_no=1):
    date_from = timezone.datetime(year=year, month=1, day=1)
    date_to = timezone.datetime(year=year, month=12, day=31)
    articles = Article.objects.published().filter(published__gte=date_from). \
               filter(published__lte=date_to)
    authors = Author.objects.all()
    author_field = "author_01"
    if author_no == 2:
        author_field = "author_02"
    elif author_no == 3:
        author_field = "author_03"
    elif author_no == 4:
        author_field = "author_04"
    elif author_no == 5:
        author_field = "author_04"

    count = articles.values(author_field).annotate(total=Count('author_01')).\
        order_by('-total')
    result = []
    for item in count:
        pk = pk=item[author_field]
        if pk:
            author = str(Author.objects.get(pk=item[author_field]))
        else:
            author = "NONE"
        num = item["total"]
        result.append([author, num])
    return result

class Command(BaseCommand):
    help = 'Count first authorships by author over a specified period of time'

    def add_arguments(self, parser):
        parser.add_argument('year', type=int,
                            help="Year to count for.")

        parser.add_argument('author_no', nargs='?', type=int,
                            help="Author number", default=1)

    def handle(self, *args, **options):
        year = options["year"]
        author_no = options["author_no"]
        print("CountAuthors for year {0}. Author # {1}.". \
              format(str(year),str(author_no)))
        author_list = process(year, author_no)
        for item in author_list:
            print(item[0], item[1])