from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import abc
import tensorflow as tf
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import nn_ops
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import sparse_tensor
from tensorflow.python.ops import random_ops
from tensorflow.python.ops import sparse_ops
from tensorflow.python.ops.losses import losses as core_losses

# The smallest probability that is used to derive smallest logit for invalid or
# padding entries.
_EPSILON = 1e-10

def label_valid_fn(labels):
  """Returns a boolean `Tensor` for label validity."""
  labels = ops.convert_to_tensor(labels)
  return math_ops.greater_equal(labels, 0.)

def sort_by_scores(scores, features_list, topn=None):
  """Sorts example features according to per-example scores.

  Args:
    scores: A `Tensor` of shape [batch_size, list_size] representing the
      per-example scores.
    features_list: A list of `Tensor`s with the same shape as scores to be
      sorted.
    topn: An integer as the cutoff of examples in the sorted list.

  Returns:
    A list of `Tensor`s as the list of sorted features by `scores`.
  """
  scores = ops.convert_to_tensor(scores)
  scores.get_shape().assert_has_rank(2)
  batch_size, list_size = array_ops.unstack(array_ops.shape(scores))
  if topn is None:
    topn = list_size
  topn = math_ops.minimum(topn, list_size)
  _, indices = nn_ops.top_k(scores, topn, sorted=True)
  list_offsets = array_ops.expand_dims(
      math_ops.range(batch_size) * list_size, 1)
  # The shape of `indices` is [batch_size, topn] and the shape of
  # `list_offsets` is [batch_size, 1]. Broadcasting is used here.
  gather_indices = array_ops.reshape(indices + list_offsets, [-1])
  output_shape = array_ops.stack([batch_size, topn])
  # Each feature is first flattened to a 1-D vector and then gathered by the
  # indices from sorted scores and then re-shaped.
  return [
      array_ops.reshape(
          array_ops.gather(array_ops.reshape(feature, [-1]), gather_indices),
          output_shape) for feature in features_list
  ]

def organize_valid_indices(is_valid, shuffle=True, seed=None):
  """Organizes indices in such a way that valid items appear first.

  Args:
    is_valid: A boolen `Tensor` for entry validity with shape [batch_size,
      list_size].
    shuffle: A boolean indicating whether valid items should be shuffled.
    seed: An int for random seed at the op level. It works together with the
      seed at global graph level together to determine the random number
      generation. See `tf.set_random_seed`.

  Returns:
    A tensor of indices with shape [batch_size, list_size, 2]. The returned
    tensor can be used with `tf.gather_nd` and `tf.scatter_nd` to compose a new
    [batch_size, list_size] tensor. The values in the last dimension are the
    indices for an element in the input tensor.
  """
  is_valid = ops.convert_to_tensor(is_valid)
  is_valid.get_shape().assert_has_rank(2)
  output_shape = array_ops.shape(is_valid)

  if shuffle:
    values = random_ops.random_uniform(output_shape, seed=seed)
  else:
    values = (array_ops.ones_like(is_valid, dtypes.float32) * array_ops.reverse(
        math_ops.to_float(math_ops.range(output_shape[1])), [-1]))

  rand = array_ops.where(is_valid, values, array_ops.ones(output_shape) * -1e-6)
  # shape(indices) = [batch_size, list_size]
  _, indices = nn_ops.top_k(rand, output_shape[1], sorted=True)
  # shape(batch_ids) = [batch_size, list_size]
  batch_ids = array_ops.ones_like(indices) * array_ops.expand_dims(
      math_ops.range(output_shape[0]), 1)
  return array_ops.concat(
      [
          array_ops.expand_dims(batch_ids, 2),  #[[0,...0], [1, ..., 1]]
          array_ops.expand_dims(indices, 2)
      ],  # shuffle之后的indices，dim0=batch  dim1=shuffle后的index,例如[0, 1, 2, 3] 变为[2,3,0,1]，为list的长度
      axis=2)

def shuffle_valid_indices(is_valid, seed=None):
  """Returns a shuffle of indices with valid ones on top."""
  return organize_valid_indices(is_valid, shuffle=True, seed=seed)

def reshape_first_ndims(tensor, first_ndims, new_shape):
  """Reshapes the first n dims of the input `tensor` to `new shape`.

  Args:
    tensor: The input `Tensor`.
    first_ndims: A int denoting the first n dims.
    new_shape: A list of int representing the new shape.

  Returns:
    A reshaped `Tensor`.
  """
  assert tensor.get_shape().ndims is None or tensor.get_shape(
  ).ndims >= first_ndims, (
      'Tensor shape is less than {} dims.'.format(first_ndims))
  new_shape = array_ops.concat(
      [new_shape, array_ops.shape(tensor)[first_ndims:]], 0)
  if isinstance(tensor, sparse_tensor.SparseTensor):
    return sparse_ops.sparse_reshape(tensor, new_shape)

  return array_ops.reshape(tensor, new_shape)

def approx_ranks(logits, alpha=10.):
  r"""Computes approximate ranks given a list of logits.

  Given a list of logits, the rank of an item in the list is simply
  one plus the total number of items with a larger logit. In other words,

    rank_i = 1 + \sum_{j \neq i} I_{s_j > s_i},

  where "I" is the indicator function. The indicator function can be
  approximated by a generalized sigmoid:

    I_{s_j < s_i} \approx 1/(1 + exp(-\alpha * (s_j - s_i))).

  This function approximates the rank of an item using this sigmoid
  approximation to the indicator function. This technique is at the core
  of "A general approximation framework for direct optimization of
  information retrieval measures" by Qin et al.

  Args:
    logits: A `Tensor` with shape [batch_size, list_size]. Each value is the
      ranking score of the corresponding item.
    alpha: Exponent of the generalized sigmoid function.

  Returns:
    A `Tensor` of ranks with the same shape as logits.
  """
  list_size = array_ops.shape(logits)[1]
  x = array_ops.tile(array_ops.expand_dims(logits, 2), [1, 1, list_size])
  y = array_ops.tile(array_ops.expand_dims(logits, 1), [1, list_size, 1])
  pairs = math_ops.sigmoid(alpha * (y - x))
  return math_ops.reduce_sum(pairs, -1) + .5

def inverse_max_dcg(labels,
                    gain_fn=lambda labels: math_ops.pow(2.0, labels) - 1.,
                    rank_discount_fn=lambda rank: 1. / math_ops.log1p(rank),
                    topn=None):
  """Computes the inverse of max DCG.

  Args:
    labels: A `Tensor` with shape [batch_size, list_size]. Each value is the
      graded relevance of the corresponding item.
    gain_fn: A gain function. By default this is set to: 2^label - 1.
    rank_discount_fn: A discount function. By default this is set to:
      1/log(1+rank).
    topn: An integer as the cutoff of examples in the sorted list.
  Returns:
    A `Tensor` with shape [batch_size, 1].
  """
  ideal_sorted_labels, = sort_by_scores(labels, [labels], topn=topn)
  rank = math_ops.range(array_ops.shape(ideal_sorted_labels)[1]) + 1
  discounted_gain = gain_fn(ideal_sorted_labels) * rank_discount_fn(
      math_ops.to_float(rank))
  discounted_gain = math_ops.reduce_sum(discounted_gain, 1, keepdims=True)
  return array_ops.where(math_ops.greater(discounted_gain, 0.),
                         1. / discounted_gain,
                         array_ops.zeros_like(discounted_gain))

def get_batch_idx_size(logits, labels, rank_id, name_prefix):
  batch_size = tf.shape(logits)[0]
  rank_key, rank_idx, count = tf.unique_with_counts(rank_id)
  max_count = tf.reduce_max(count)
  unique_rank_id_num = tf.shape(count)[0]
  rank_idx_tile = tf.tile(tf.expand_dims(rank_idx, 0), [unique_rank_id_num, 1])
  range_count_tile = tf.tile(tf.expand_dims(tf.range(unique_rank_id_num), 1),
                             [1, batch_size])
  list_id_mask = tf.cast(tf.equal(rank_idx_tile, range_count_tile), tf.int32)
  cum_mask = tf.cumsum(list_id_mask, axis=1, exclusive=True)
  masked_list_id = list_id_mask * cum_mask
  col_cor = tf.reduce_sum(masked_list_id, axis=0)
  row_cor = rank_idx
  batch_idx = tf.concat(
      [tf.expand_dims(row_cor, 1),
       tf.expand_dims(col_cor, 1)], axis=1)
  output_shape = [tf.shape(count)[0], max_count]

  logits_idx = tf.scatter_nd(updates=logits,
                             indices=batch_idx,
                             shape=output_shape,
                             name=name_prefix + "logits_idx")
  label_idx = tf.scatter_nd(updates=labels,
                            indices=batch_idx,
                            shape=output_shape,
                            name=name_prefix + "label_idx") - 1e-6
  mask_idx = tf.scatter_nd(updates=tf.ones_like(logits),
                           indices=batch_idx,
                           shape=[batch_size, batch_size],
                           name=name_prefix + 'mask_idx')
  unique_idx = tf.argmax(list_id_mask, axis=1)
  return logits_idx, label_idx, mask_idx, unique_idx

class RankingLossKey(object):
  """Ranking loss key strings."""
  # Names for the ranking based loss functions.
  PAIRWISE_HINGE_LOSS = 'pairwise_hinge_loss'
  PAIRWISE_LOGISTIC_LOSS = 'pairwise_logistic_loss'
  PAIRWISE_SOFT_ZERO_ONE_LOSS = 'pairwise_soft_zero_one_loss'
  SOFTMAX_LOSS = 'softmax_loss'
  SIGMOID_CROSS_ENTROPY_LOSS = 'sigmoid_cross_entropy_loss'
  MEAN_SQUARED_LOSS = 'mean_squared_loss'
  LIST_MLE_LOSS = 'list_mle_loss'
  APPROX_NDCG_LOSS = 'approx_ndcg_loss'

def make_loss_fn(loss_keys,
                 loss_weights=None,
                 weights_feature_name=None,
                 lambda_weight=None,
                 reduction=core_losses.Reduction.SUM_BY_NONZERO_WEIGHTS,
                 name=None,
                 seed=None,
                 extra_args=None):
  """Makes a loss function using a single loss or multiple losses.

  Args:
    loss_keys: A string or list of strings representing loss keys defined in
      `RankingLossKey`. Listed loss functions will be combined in a weighted
      manner, with weights specified by `loss_weights`. If `loss_weights` is
      None, default weight of 1 will be used.
    loss_weights: List of weights, same length as `loss_keys`. Used when merging
      losses to calculate the weighted sum of losses. If `None`, all losses are
      weighted equally with weight being 1.
    weights_feature_name: A string specifying the name of the weights feature in
      `features` dict.
    lambda_weight: A `_LambdaWeight` object created by factory methods like
      `create_ndcg_lambda_weight()`.
    reduction: One of `tf.losses.Reduction` except `NONE`. Describes how to
      reduce training loss over batch.
    name: A string used as the name for this loss.
    seed: A randomization seed used in computation of some loss functions such
      as ListMLE and pListMLE.
    extra_args: A string-keyed dictionary that contains any other loss-specific
      arguments.

  Returns:
    A function _loss_fn(). See `_loss_fn()` for its signature.

  Raises:
    ValueError: If `reduction` is invalid.
    ValueError: If `loss_keys` is None or empty.
    ValueError: If `loss_keys` and `loss_weights` have different sizes.
  """
  if (reduction not in core_losses.Reduction.all() or
      reduction == core_losses.Reduction.NONE):
    raise ValueError('Invalid reduction: {}'.format(reduction))

  if not loss_keys:
    raise ValueError('loss_keys cannot be None or empty.')

  if loss_weights:
    if len(loss_keys) != len(loss_weights):
      raise ValueError('loss_keys and loss_weights must have the same size.')

  if not isinstance(loss_keys, list):
    loss_keys = [loss_keys]

  def _loss_fn(labels, logits, features):
    """Computes a single loss or weighted combination of losses.

    Args:
      labels: A `Tensor` of the same shape as `logits` representing relevance.
      logits: A `Tensor` with shape [batch_size, list_size]. Each value is the
        ranking score of the corresponding item.
      features: Dict of Tensors of shape [batch_size, list_size, ...] for
        per-example features and shape [batch_size, ...] for non-example context
        features.

    Returns:
      An op for a single loss or weighted combination of multiple losses.

    Raises:
      ValueError: If `loss_keys` is invalid.
    """
    weights = features[weights_feature_name] if weights_feature_name else None
    loss_kwargs = {
        'labels': labels,
        'logits': logits,
        'weights': weights,
        'reduction': reduction,
        'name': name,
    }
    if extra_args is not None:
      loss_kwargs.update(extra_args)

    loss_kwargs_with_lambda_weight = loss_kwargs.copy()
    loss_kwargs_with_lambda_weight['lambda_weight'] = lambda_weight

    loss_kwargs_with_lambda_weight_and_seed = (
        loss_kwargs_with_lambda_weight.copy())
    loss_kwargs_with_lambda_weight_and_seed['seed'] = seed

    key_to_fn = {
        RankingLossKey.PAIRWISE_HINGE_LOSS:
            (_pairwise_hinge_loss, loss_kwargs_with_lambda_weight),
        RankingLossKey.PAIRWISE_LOGISTIC_LOSS:
            (_pairwise_logistic_loss, loss_kwargs_with_lambda_weight),
        RankingLossKey.PAIRWISE_SOFT_ZERO_ONE_LOSS:
            (_pairwise_soft_zero_one_loss, loss_kwargs_with_lambda_weight),
        RankingLossKey.SOFTMAX_LOSS:
            (_softmax_loss, loss_kwargs_with_lambda_weight),
        RankingLossKey.SIGMOID_CROSS_ENTROPY_LOSS:
            (_sigmoid_cross_entropy_loss, loss_kwargs),
        RankingLossKey.MEAN_SQUARED_LOSS: (_mean_squared_loss, loss_kwargs),
        RankingLossKey.LIST_MLE_LOSS:
            (_list_mle_loss, loss_kwargs_with_lambda_weight_and_seed),
        RankingLossKey.APPROX_NDCG_LOSS: (_approx_ndcg_loss, loss_kwargs),
    }

    # Obtain the list of loss ops.
    loss_ops = []
    for loss_key in loss_keys:
      if loss_key not in key_to_fn:
        raise ValueError('Invalid loss_key: {}.'.format(loss_key))
      loss_fn, kwargs = key_to_fn[loss_key]
      loss_ops.append(loss_fn(**kwargs))

    # Compute weighted combination of losses.
    if loss_weights:
      weighted_losses = []
      for loss_op, loss_weight in zip(loss_ops, loss_weights):
        weighted_losses.append(math_ops.multiply(loss_op, loss_weight))
    else:
      weighted_losses = loss_ops

    return math_ops.add_n(weighted_losses)

  return _loss_fn

def create_ndcg_lambda_weight(topn=None, smooth_fraction=0.):
  """Creates _LambdaWeight for NDCG metric."""
  return DCGLambdaWeight(
      topn,
      gain_fn=lambda labels: math_ops.pow(2.0, labels) - 1.,
      rank_discount_fn=lambda rank: 1. / math_ops.log1p(rank),
      normalized=True,
      smooth_fraction=smooth_fraction)

def create_reciprocal_rank_lambda_weight(topn=None, smooth_fraction=0.):
  """Creates _LambdaWeight for MRR-like metric."""
  return DCGLambdaWeight(topn,
                         gain_fn=lambda labels: labels,
                         rank_discount_fn=lambda rank: 1. / rank,
                         normalized=True,
                         smooth_fraction=smooth_fraction)

def create_p_list_mle_lambda_weight(list_size):
  """Creates _LambdaWeight based on Position-Aware ListMLE paper.

  Produces a weight based on the formulation presented in the
  "Position-Aware ListMLE" paper (Lan et al.) and available using
  create_p_list_mle_lambda_weight() factory function above.

  Args:
    list_size: Size of the input list.

  Returns:
    A _LambdaWeight for Position-Aware ListMLE.
  """
  return ListMLELambdaWeight(
      rank_discount_fn=lambda rank: math_ops.pow(2., list_size - rank) - 1.)

class _LambdaWeight(object):
  """Interface for ranking metric optimization.

  This class wraps weights used in the LambdaLoss framework for ranking metric
  optimization (https://ai.google/research/pubs/pub47258). Such an interface is
  to be instantiated by concrete lambda weight models. The instance is used
  together with standard loss such as logistic loss and softmax loss.
  """

  __metaclass__ = abc.ABCMeta

  def _get_valid_pairs_and_clean_labels(self, sorted_labels):
    """Returns a boolean Tensor for valid pairs and cleaned labels."""
    sorted_labels = ops.convert_to_tensor(sorted_labels)
    sorted_labels.get_shape().assert_has_rank(2)
    is_label_valid = label_valid_fn(sorted_labels)
    valid_pairs = math_ops.logical_and(array_ops.expand_dims(is_label_valid, 2),
                                       array_ops.expand_dims(is_label_valid, 1))
    sorted_labels = array_ops.where(is_label_valid, sorted_labels,
                                    array_ops.zeros_like(sorted_labels))
    return valid_pairs, sorted_labels

  @abc.abstractmethod
  def pair_weights(self, sorted_labels):
    """Returns the weight adjustment `Tensor` for example pairs.

    Args:
      sorted_labels: A dense `Tensor` of labels with shape [batch_size,
        list_size] that are sorted by logits.

    Returns:
      A `Tensor` that can weight example pairs.
    """
    raise NotImplementedError('Calling an abstract method.')

  def individual_weights(self, sorted_labels):
    """Returns the weight `Tensor` for individual examples.

    Args:
      sorted_labels: A dense `Tensor` of labels with shape [batch_size,
        list_size] that are sorted by logits.

    Returns:
      A `Tensor` that can weight individual examples.
    """
    return sorted_labels

class IdentityLambdaWeight(_LambdaWeight):

  def __init__(self,):
    pass

  def pair_weights(self, sorted_labels):
    return 1.0

class DCGLambdaWeight(_LambdaWeight):
  """LambdaWeight for Discounted Cumulative Gain metric."""

  def __init__(self,
               topn=None,
               gain_fn=lambda label: label,
               rank_discount_fn=lambda rank: 1. / rank,
               normalized=False,
               smooth_fraction=0.):
    """Constructor.

    Ranks are 1-based, not 0-based. Given rank i and j, there are two types of
    pair weights:
      u = |rank_discount_fn(|i-j|) - rank_discount_fn(|i-j| + 1)|
      v = |rank_discount_fn(i) - rank_discount_fn(j)|
    where u is the newly introduced one in LambdaLoss paper
    (https://ai.google/research/pubs/pub47258) and v is the original one in the
    LambdaMART paper "From RankNet to LambdaRank to LambdaMART: An Overview".
    The final pair weight contribution of ranks is
      (1-smooth_fraction) * u + smooth_fraction * v.

    Args:
      topn: (int) The topn for the DCG metric.
      gain_fn: (function) Tranforms labels.
      rank_discount_fn: (function) The rank discount function.
      normalized: (bool) If True, normalize weight by the max DCG.
      smooth_fraction: (float) parameter to control the contribution from
        LambdaMART.
    """
    self._topn = topn
    self._gain_fn = gain_fn
    self._rank_discount_fn = rank_discount_fn
    self._normalized = normalized
    assert 0. <= smooth_fraction and smooth_fraction <= 1., (
        'smooth_fraction %s should be in range [0, 1].' % smooth_fraction)
    self._smooth_fraction = smooth_fraction

  def pair_weights(self, sorted_labels):
    """See `_LambdaWeight`."""
    with ops.name_scope(None, 'dcg_lambda_weight', (sorted_labels,)):
      valid_pair, sorted_labels = self._get_valid_pairs_and_clean_labels(
          sorted_labels)
      gain = self._gain_fn(sorted_labels)
      if self._normalized:
        gain *= inverse_max_dcg(sorted_labels,
                                gain_fn=self._gain_fn,
                                rank_discount_fn=self._rank_discount_fn,
                                topn=self._topn)
      pair_gain = array_ops.expand_dims(gain, 2) - array_ops.expand_dims(
          gain, 1)
      pair_gain *= math_ops.to_float(valid_pair)

      list_size = array_ops.shape(sorted_labels)[1]
      topn = self._topn or list_size
      rank = math_ops.range(list_size) + 1

      def _discount_for_relative_rank_diff():
        """Rank-based discount in the LambdaLoss paper."""
        # The LambdaLoss is not well defined when topn is active and topn <
        # list_size. We cap the rank of examples to topn + 1 so that the rank
        # differene is capped to topn. This is just a convenient upperbound
        # when topn is active. We need to revisit this.
        capped_rank = array_ops.where(math_ops.greater(rank, topn),
                                      array_ops.ones_like(rank) * (topn + 1),
                                      rank)
        rank_diff = math_ops.to_float(
            math_ops.abs(
                array_ops.expand_dims(capped_rank, 1) -
                array_ops.expand_dims(capped_rank, 0)))
        pair_discount = array_ops.where(
            math_ops.greater(rank_diff, 0),
            math_ops.abs(
                self._rank_discount_fn(rank_diff) -
                self._rank_discount_fn(rank_diff + 1)),
            array_ops.zeros_like(rank_diff))
        return pair_discount

      def _discount_for_absolute_rank():
        """Standard discount in the LambdaMART paper."""
        # When the rank discount is (1 / rank) for example, the discount is
        # |1 / r_i - 1 / r_j|. When i or j > topn, the discount becomes 0.
        rank_discount = array_ops.where(
            math_ops.greater(rank, topn),
            array_ops.zeros_like(math_ops.to_float(rank)),
            self._rank_discount_fn(math_ops.to_float(rank)))
        pair_discount = math_ops.abs(
            array_ops.expand_dims(rank_discount, 1) -
            array_ops.expand_dims(rank_discount, 0))
        return pair_discount

      u = _discount_for_relative_rank_diff()
      v = _discount_for_absolute_rank()
      pair_discount = (1. -
                       self._smooth_fraction) * u + self._smooth_fraction * v
      pair_weight = math_ops.abs(pair_gain) * pair_discount
      if self._topn is None:
        return pair_weight
      pair_mask = math_ops.logical_or(
          array_ops.expand_dims(math_ops.less_equal(rank, self._topn), 1),
          array_ops.expand_dims(math_ops.less_equal(rank, self._topn), 0))
      return pair_weight * math_ops.to_float(pair_mask)

  def individual_weights(self, sorted_labels):
    """See `_LambdaWeight`."""
    with ops.name_scope(None, 'dcg_lambda_weight', (sorted_labels,)):
      sorted_labels = ops.convert_to_tensor(sorted_labels)
      sorted_labels = array_ops.where(label_valid_fn(sorted_labels),
                                      sorted_labels,
                                      array_ops.zeros_like(sorted_labels))
      gain = self._gain_fn(sorted_labels)
      if self._normalized:
        gain *= inverse_max_dcg(sorted_labels,
                                gain_fn=self._gain_fn,
                                rank_discount_fn=self._rank_discount_fn,
                                topn=self._topn)
      rank_discount = self._rank_discount_fn(
          math_ops.to_float(
              math_ops.range(array_ops.shape(sorted_labels)[1]) + 1))
      return gain * rank_discount

class PrecisionLambdaWeight(_LambdaWeight):
  """LambdaWeight for Precision metric."""

  def __init__(self,
               topn,
               positive_fn=lambda label: math_ops.greater_equal(label, 1.0)):
    """Constructor.

    Args:
      topn: (int) The K in Precision@K metric.
      positive_fn: (function): A function on `Tensor` that output boolean True
        for positive examples. The rest are negative examples.
    """
    self._topn = topn
    self._positive_fn = positive_fn

  def pair_weights(self, sorted_labels):
    """See `_LambdaWeight`.

    The current implementation here is that for any pairs of documents i and j,
    we set the weight to be 1 if
      - i and j have different labels.
      - i <= topn and j > topn or i > topn and j <= topn.
    This is exactly the same as the original LambdaRank method. The weight is
    the gain of swapping a pair of documents.

    Args:
      sorted_labels: A dense `Tensor` of labels with shape [batch_size,
        list_size] that are sorted by logits.

    Returns:
      A `Tensor` that can weight example pairs.
    """
    with ops.name_scope(None, 'precision_lambda_weight', (sorted_labels,)):
      valid_pair, sorted_labels = self._get_valid_pairs_and_clean_labels(
          sorted_labels)
      binary_labels = math_ops.to_float(self._positive_fn(sorted_labels))
      label_diff = math_ops.abs(
          array_ops.expand_dims(binary_labels, 2) -
          array_ops.expand_dims(binary_labels, 1))
      label_diff *= math_ops.to_float(valid_pair)
      # i <= topn and j > topn or i > topn and j <= topn, i.e., xor(i <= topn, j
      # <= topn).
      list_size = array_ops.shape(sorted_labels)[1]
      rank = math_ops.range(list_size) + 1
      rank_mask = math_ops.logical_xor(
          array_ops.expand_dims(math_ops.less_equal(rank, self._topn), 1),
          array_ops.expand_dims(math_ops.less_equal(rank, self._topn), 0))
      return label_diff * math_ops.to_float(rank_mask)

class ListMLELambdaWeight(_LambdaWeight):
  """LambdaWeight for ListMLE cost function."""

  def __init__(self, rank_discount_fn):
    """Constructor.

    Ranks are 1-based, not 0-based.

    Args:
      rank_discount_fn: (function) The rank discount function.
    """
    self._rank_discount_fn = rank_discount_fn

  def pair_weights(self, sorted_labels):
    """See `_LambdaWeight`."""
    return sorted_labels

  def individual_weights(self, sorted_labels):
    """See `_LambdaWeight`."""
    with ops.name_scope(None, 'p_list_mle_lambda_weight', (sorted_labels,)):
      sorted_labels = ops.convert_to_tensor(sorted_labels)
      rank_discount = self._rank_discount_fn(
          math_ops.to_float(
              math_ops.range(array_ops.shape(sorted_labels)[1]) + 1))
      return array_ops.ones_like(sorted_labels) * rank_discount

def _sort_and_normalize(labels, logits, weights=None):
  """Sorts `labels` and `logits` and normalize `weights`.

  Args:
    labels: A `Tensor` of the same shape as `logits` representing graded
      relevance.
    logits: A `Tensor` with shape [batch_size, list_size]. Each value is the
      ranking score of the corresponding item.
    weights: A scalar, a `Tensor` with shape [batch_size, 1], or a `Tensor` with
      the same shape as `labels`.

  Returns:
    A tuple of (sorted_labels, sorted_logits, sorted_weights).
  """
  labels = ops.convert_to_tensor(labels)
  logits = ops.convert_to_tensor(logits)
  logits.get_shape().assert_has_rank(2)
  logits.get_shape().assert_is_compatible_with(labels.get_shape())
  weights = 1.0 if weights is None else ops.convert_to_tensor(weights)
  weights = array_ops.ones_like(labels) * weights
  _, topn = array_ops.unstack(array_ops.shape(logits))

  # Only sort entries with valid labels that are >= 0.
  scores = array_ops.where(
      math_ops.greater_equal(labels,
                             0.), logits, -1e-6 * array_ops.ones_like(logits) +
      math_ops.reduce_min(logits, axis=1, keepdims=True))
  sorted_labels, sorted_logits, sorted_weights = sort_by_scores(
      scores, [labels, logits, weights], topn=topn)
  return sorted_labels, sorted_logits, sorted_weights

def _pairwise_comparison(sorted_labels,
                         sorted_logits,
                         sorted_weights,
                         lambda_weight=None):
  r"""Returns pairwise comparison `Tensor`s.

  Given a list of n items, the labels of graded relevance l_i and the logits
  s_i, we sort the items in a list based on s_i and obtain ranks r_i. We form
  n^2 pairs of items. For each pair, we have the following:

                        /
                        | 1   if l_i > l_j
  * `pairwise_labels` = |
                        | 0   if l_i <= l_j
                        \
  * `pairwise_logits` = s_i - s_j
                         /
                         | 0              if l_i <= l_j,
  * `pairwise_weights` = | |l_i - l_j|    if lambda_weight is None,
                         | lambda_weight  otherwise.
                         \

  The `sorted_weights` is item-wise and is applied non-symmetrically to update
  pairwise_weights as
    pairwise_weights(i, j) = w_i * pairwise_weights(i, j).
  This effectively applies to all pairs with l_i > l_j. Note that it is actually
  symmetric when `sorted_weights` are constant per list, i.e., listwise weights.

  Args:
    sorted_labels: A `Tensor` with shape [batch_size, list_size] of labels
      sorted.
    sorted_logits: A `Tensor` with shape [batch_size, list_size] of logits
      sorted.
    sorted_weights: A `Tensor` with shape [batch_size, list_size] of item-wise
      weights sorted.
    lambda_weight: A `_LambdaWeight` object.

  Returns:
    A tuple of (pairwise_labels, pairwise_logits, pairwise_weights) with each
    having the shape [batch_size, list_size, list_size].
  """
  # Compute the difference for all pairs in a list. The output is a Tensor with
  # shape [batch_size, list_size, list_size] where the entry [-1, i, j] stores
  # the information for pair (i, j).
  pairwise_label_diff = array_ops.expand_dims(
      sorted_labels, 2) - array_ops.expand_dims(sorted_labels, 1)
  pairwise_logits = array_ops.expand_dims(
      sorted_logits, 2) - array_ops.expand_dims(sorted_logits, 1)
  pairwise_labels = math_ops.to_float(math_ops.greater(pairwise_label_diff, 0))
  is_label_valid = label_valid_fn(sorted_labels)
  valid_pair = math_ops.logical_and(array_ops.expand_dims(is_label_valid, 2),
                                    array_ops.expand_dims(is_label_valid, 1))
  # Only keep the case when l_i > l_j.
  pairwise_weights = pairwise_labels * math_ops.to_float(valid_pair)
  # Apply the item-wise weights along l_i.
  pairwise_weights *= tf.cast(array_ops.expand_dims(sorted_weights, 2),
                              tf.float32)
  if lambda_weight is not None:
    pairwise_weights *= lambda_weight.pair_weights(sorted_labels)
  else:
    pairwise_weights *= math_ops.abs(pairwise_label_diff)
  pairwise_weights = array_ops.stop_gradient(pairwise_weights,
                                             name='weights_stop_gradient')
  return pairwise_labels, pairwise_logits, pairwise_weights

def _pairwise_loss(loss_fn,
                   labels,
                   logits,
                   weights=None,
                   lambda_weight=None,
                   lambda_scale=True,
                   reduction=core_losses.Reduction.SUM_BY_NONZERO_WEIGHTS):
  """Template to compute pairwise loss.

  Args:
    loss_fn: A function that computes loss from the pairwise logits with l_i >
      l_j.
    labels: A `Tensor` of the same shape as `logits` representing graded
      relevance.
    logits: A `Tensor` with shape [batch_size, list_size]. Each value is the
      ranking score of the corresponding item.
    weights: A scalar, a `Tensor` with shape [batch_size, 1] for list-wise
      weights, or a `Tensor` with shape [batch_size, list_size] for item-wise
      weights.
    lambda_weight: A `_LambdaWeight` object.
    reduction: One of `tf.losses.Reduction` except `NONE`. Describes how to
      reduce training loss over batch.

  Returns:
    An op for the pairwise loss.
  """
  sorted_labels, sorted_logits, sorted_weights = _sort_and_normalize(
      labels, logits, weights)
  _, pairwise_logits, pairwise_weights = _pairwise_comparison(
      sorted_labels, sorted_logits, sorted_weights, lambda_weight)

  if lambda_weight is not None and lambda_scale:
    # For LambdaLoss with relative rank difference, the scale of loss becomes
    # much smaller when applying LambdaWeight. This affects the training can
    # make the optimal learning rate become much larger. We use a heuristic to
    # scale it up to the same magnitude as standard pairwise loss.
    pairwise_weights *= math_ops.to_float(array_ops.shape(sorted_labels)[1])
  return core_losses.compute_weighted_loss(loss_fn(pairwise_logits),
                                           weights=pairwise_weights,
                                           reduction=reduction)

def _pairwise_hinge_loss(labels,
                         logits,
                         weights=None,
                         lambda_weight=None,
                         lambda_scale=True,
                         reduction=core_losses.Reduction.SUM_BY_NONZERO_WEIGHTS,
                         name=None):
  """Computes the pairwise hinge loss for a list.

  The hinge loss is defined as Hinge(l_i > l_j) = max(0, 1 - (s_i - s_j)). So a
  correctly ordered pair has 0 loss if (s_i - s_j >= 1). Otherwise the loss
  increases linearly with s_i - s_j. When the list_size is 2, this reduces to
  the standard hinge loss.

  Args:
    labels: A `Tensor` of the same shape as `logits` representing graded
      relevance.
    logits: A `Tensor` with shape [batch_size, list_size]. Each value is the
      ranking score of the corresponding item.
    weights: A scalar, a `Tensor` with shape [batch_size, 1] for list-wise
      weights, or a `Tensor` with shape [batch_size, list_size] for item-wise
      weights.
    lambda_weight: A `_LambdaWeight` object.
    reduction: One of `tf.losses.Reduction` except `NONE`. Describes how to
      reduce training loss over batch.
    name: A string used as the name for this loss.

  Returns:
    An op for the pairwise hinge loss.
  """

  def _loss(logits):
    """The loss of pairwise logits with l_i > l_j."""
    # TODO(xuanhui, pointer-team): Consider pass params object into the loss and
    # put a margin here.
    return nn_ops.relu(1. - logits)

  with ops.name_scope(name, 'pairwise_hinge_loss', (labels, logits, weights)):
    return _pairwise_loss(_loss,
                          labels,
                          logits,
                          weights,
                          lambda_weight,
                          lambda_scale,
                          reduction=reduction)

def _pairwise_logistic_loss(
    labels,
    logits,
    weights=None,
    lambda_weight=None,
    lambda_scale=True,
    reduction=core_losses.Reduction.SUM_BY_NONZERO_WEIGHTS,
    name=None):
  """Computes the pairwise logistic loss for a list.

  The preference probability of each pair is computed as the sigmoid function:
  P(l_i > l_j) = 1 / (1 + exp(s_j - s_i)) and the logistic loss is log(P(l_i >
  l_j)) if l_i > l_j and 0 otherwise.

  Args:
    labels: A `Tensor` of the same shape as `logits` representing graded
      relevance.
    logits: A `Tensor` with shape [batch_size, list_size]. Each value is the
      ranking score of the corresponding item.
    weights: A scalar, a `Tensor` with shape [batch_size, 1] for list-wise
      weights, or a `Tensor` with shape [batch_size, list_size] for item-wise
      weights.
    lambda_weight: A `_LambdaWeight` object.
    reduction: One of `tf.losses.Reduction` except `NONE`. Describes how to
      reduce training loss over batch.
    name: A string used as the name for this loss.

  Returns:
    An op for the pairwise logistic loss.
  """

  def _loss(logits):
    """The loss of pairwise logits with l_i > l_j."""
    # The following is the same as log(1 + exp(-pairwise_logits)).
    return nn_ops.relu(-logits) + math_ops.log1p(
        math_ops.exp(-math_ops.abs(logits)))

  with ops.name_scope(name, 'pairwise_logistic_loss',
                      (labels, logits, weights)):
    return _pairwise_loss(_loss,
                          labels,
                          logits,
                          weights,
                          lambda_weight,
                          lambda_scale,
                          reduction=reduction)

def _pairwise_soft_zero_one_loss(
    labels,
    logits,
    weights=None,
    lambda_weight=None,
    reduction=core_losses.Reduction.SUM_BY_NONZERO_WEIGHTS,
    name=None):
  """Computes the pairwise soft zero-one loss.

  Note this is different from sigmoid cross entropy in that soft zero-one loss
  is a smooth but non-convex approximation of zero-one loss. The preference
  probability of each pair is computed as the sigmoid function: P(l_i > l_j) = 1
  / (1 + exp(s_j - s_i)). Then 1 - P(l_i > l_j) is directly used as the loss.
  So a correctly ordered pair has a loss close to 0, while an incorrectly
  ordered pair has a loss bounded by 1.

  Args:
    labels: A `Tensor` of the same shape as `logits` representing graded
      relevance.
    logits: A `Tensor` with shape [batch_size, list_size]. Each value is the
      ranking score of the corresponding item.
    weights: A scalar, a `Tensor` with shape [batch_size, 1] for list-wise
      weights, or a `Tensor` with shape [batch_size, list_size] for item-wise
      weights.
    lambda_weight: A `_LambdaWeight` object.
    reduction: One of `tf.losses.Reduction` except `NONE`. Describes how to
      reduce training loss over batch.
    name: A string used as the name for this loss.

  Returns:
    An op for the pairwise soft zero one loss.
  """

  def _loss(logits):
    """The loss of pairwise logits with l_i > l_j."""
    return array_ops.where(math_ops.greater(logits, 0),
                           1. - math_ops.sigmoid(logits),
                           math_ops.sigmoid(-logits))

  with ops.name_scope(name, 'pairwise_soft_zero_one_loss',
                      (labels, logits, weights)):
    return _pairwise_loss(_loss,
                          labels,
                          logits,
                          weights,
                          lambda_weight,
                          reduction=reduction)

def _softmax_loss(labels,
                  logits,
                  weights=None,
                  lambda_weight=None,
                  reduction=core_losses.Reduction.SUM_BY_NONZERO_WEIGHTS,
                  name=None):
  """Computes the softmax cross entropy for a list.

  Given the labels l_i and the logits s_i, we sort the examples and obtain ranks
  r_i. The standard softmax loss doesn't need r_i and is defined as
      -sum_i l_i * log(exp(s_i) / (exp(s_1) + ... + exp(s_n))).
  The `lambda_weight` re-weight examples based on l_i and r_i.
      -sum_i w(l_i, r_i) * log(exp(s_i) / (exp(s_1) + ... + exp(s_n))).abc
  See 'individual_weights' in 'DCGLambdaWeight' for how w(l_i, r_i) is computed.

  Args:
    labels: A `Tensor` of the same shape as `logits` representing graded
      relevance.
    logits: A `Tensor` with shape [batch_size, list_size]. Each value is the
      ranking score of the corresponding item.
    weights: A scalar, a `Tensor` with shape [batch_size, 1] for list-wise
      weights, or a `Tensor` with shape [batch_size, list_size] for item-wise
      weights.
    lambda_weight: A `DCGLambdaWeight` instance.
    reduction: One of `tf.losses.Reduction` except `NONE`. Describes how to
      reduce training loss over batch.
    name: A string used as the name for this loss.

  Returns:
    An op for the softmax cross entropy as a loss.
  """
  with ops.name_scope(name, 'softmax_loss', (labels, logits, weights)):
    sorted_labels, sorted_logits, sorted_weights = _sort_and_normalize(
        labels, logits, weights)
    is_label_valid = label_valid_fn(sorted_labels)
    # Reset the invalid labels to 0 and reset the invalid logits to a logit with
    # ~= 0 contribution in softmax.
    sorted_labels = array_ops.where(is_label_valid, sorted_labels,
                                    array_ops.zeros_like(sorted_labels))
    sorted_logits = array_ops.where(
        is_label_valid, sorted_logits,
        math_ops.log(_EPSILON) * array_ops.ones_like(sorted_logits))
    if lambda_weight is not None and isinstance(lambda_weight, DCGLambdaWeight):
      sorted_labels = lambda_weight.individual_weights(sorted_labels)
    sorted_labels *= sorted_weights
    label_sum = math_ops.reduce_sum(sorted_labels, 1, keepdims=True)
    nonzero_mask = math_ops.greater(array_ops.reshape(label_sum, [-1]), 0.0)
    label_sum, sorted_labels, sorted_logits = [
        array_ops.boolean_mask(x, nonzero_mask)
        for x in [label_sum, sorted_labels, sorted_logits]
    ]
    return core_losses.softmax_cross_entropy(sorted_labels / label_sum,
                                             sorted_logits,
                                             weights=array_ops.reshape(
                                                 label_sum, [-1]),
                                             reduction=reduction)

def _sigmoid_cross_entropy_loss(
    labels,
    logits,
    weights=None,
    reduction=core_losses.Reduction.SUM_BY_NONZERO_WEIGHTS,
    name=None):
  """Computes the sigmoid_cross_entropy loss for a list.

  Given the labels of graded relevance l_i and the logits s_i, we calculate
  the sigmoid cross entropy for each ith position and aggregate the per position
  losses.

  Args:
    labels: A `Tensor` of the same shape as `logits` representing graded
      relevance.
    logits: A `Tensor` with shape [batch_size, list_size]. Each value is the
      ranking score of the corresponding item.
    weights: A scalar, a `Tensor` with shape [batch_size, 1] for list-wise
      weights, or a `Tensor` with shape [batch_size, list_size] for item-wise
      weights.
    reduction: One of `tf.losses.Reduction` except `NONE`. Describes how to
      reduce training loss over batch.
    name: A string used as the name for this loss.

  Returns:
    An op for the sigmoid cross entropy as a loss.
  """
  with ops.name_scope(name, 'sigmoid_cross_entropy_loss',
                      (labels, logits, weights)):
    is_label_valid = array_ops.reshape(label_valid_fn(labels), [-1])
    weights = 1.0 if weights is None else ops.convert_to_tensor(weights)
    weights = array_ops.ones_like(labels) * weights
    label_vector, logit_vector, weight_vector = [
        array_ops.boolean_mask(array_ops.reshape(x, [-1]), is_label_valid)
        for x in [labels, logits, weights]
    ]
    return core_losses.sigmoid_cross_entropy(label_vector,
                                             logit_vector,
                                             weights=weight_vector,
                                             reduction=reduction)

def _mean_squared_loss(labels,
                       logits,
                       weights=None,
                       reduction=core_losses.Reduction.SUM_BY_NONZERO_WEIGHTS,
                       name=None):
  """Computes the mean squared loss for a list.

  Given the labels of graded relevance l_i and the logits s_i, we calculate
  the squared error for each ith position and aggregate the per position
  losses.

  Args:
    labels: A `Tensor` of the same shape as `logits` representing graded
      relevance.
    logits: A `Tensor` with shape [batch_size, list_size]. Each value is the
      ranking score of the corresponding item.
    weights: A scalar, a `Tensor` with shape [batch_size, 1] for list-wise
      weights, or a `Tensor` with shape [batch_size, list_size] for item-wise
      weights.
    reduction: One of `tf.losses.Reduction` except `NONE`. Describes how to
      reduce training loss over batch.
    name: A string used as the name for this loss.

  Returns:
    An op for the mean squared error as a loss.
  """
  with ops.name_scope(name, 'mean_squared_loss', (labels, logits, weights)):
    is_label_valid = array_ops.reshape(label_valid_fn(labels), [-1])
    weights = 1.0 if weights is None else ops.convert_to_tensor(weights)
    weights = array_ops.ones_like(labels) * weights
    label_vector, logit_vector, weight_vector = [
        array_ops.boolean_mask(array_ops.reshape(x, [-1]), is_label_valid)
        for x in [labels, logits, weights]
    ]
    return core_losses.mean_squared_error(label_vector,
                                          logit_vector,
                                          weights=weight_vector,
                                          reduction=reduction)

def _list_mle_loss(labels,
                   logits,
                   weights=None,
                   lambda_weight=None,
                   reduction=core_losses.Reduction.SUM_BY_NONZERO_WEIGHTS,
                   name=None,
                   seed=None):
  """Computes the ListMLE loss [Xia et al. 2008] for a list.

  Given the labels of graded relevance l_i and the logits s_i, we calculate
  the ListMLE loss for the given list.

  The `lambda_weight` re-weights examples based on l_i and r_i.
  The recommended weighting scheme is the formulation presented in the
  "Position-Aware ListMLE" paper (Lan et al.) and available using
  create_p_list_mle_lambda_weight() factory function above.

  Args:
    labels: A `Tensor` of the same shape as `logits` representing graded
      relevance.
    logits: A `Tensor` with shape [batch_size, list_size]. Each value is the
      ranking score of the corresponding item.
    weights: A scalar, a `Tensor` with shape [batch_size, 1] for list-wise
      weights, or a `Tensor` with shape [batch_size, list_size] for item-wise
      weights.
    lambda_weight: A `DCGLambdaWeight` instance.
    reduction: One of `tf.losses.Reduction` except `NONE`. Describes how to
      reduce training loss over batch.
    name: A string used as the name for this loss.
    seed: A randomization seed used when shuffling ground truth permutations.

  Returns:
    An op for the ListMLE loss.
  """
  with ops.name_scope(name, 'list_mle_loss', (labels, logits, weights)):
    is_label_valid = label_valid_fn(labels)
    # Reset the invalid labels to 0 and reset the invalid logits to a logit with
    # ~= 0 contribution.
    labels = array_ops.where(is_label_valid, labels,
                             array_ops.zeros_like(labels))
    logits = array_ops.where(
        is_label_valid, logits,
        math_ops.log(_EPSILON) * array_ops.ones_like(logits))
    weights = 1.0 if weights is None else ops.convert_to_tensor(weights)
    weights = array_ops.squeeze(weights)

    # Shuffle labels and logits to add randomness to sort.
    shuffled_indices = shuffle_valid_indices(is_label_valid, seed)
    shuffled_labels = array_ops.gather_nd(labels, shuffled_indices)
    shuffled_logits = array_ops.gather_nd(logits, shuffled_indices)

    sorted_labels, sorted_logits = sort_by_scores(
        shuffled_labels, [shuffled_labels, shuffled_logits])

    raw_max = math_ops.reduce_max(sorted_logits, axis=1, keepdims=True)
    sorted_logits = sorted_logits - raw_max
    sums = math_ops.cumsum(math_ops.exp(sorted_logits), axis=1, reverse=True)
    sums = math_ops.log(sums) - sorted_logits

    if lambda_weight is not None and isinstance(lambda_weight,
                                                ListMLELambdaWeight):
      sums *= lambda_weight.individual_weights(sorted_labels)

    negative_log_likelihood = math_ops.reduce_sum(sums, 1)

    return core_losses.compute_weighted_loss(negative_log_likelihood,
                                             weights=weights,
                                             reduction=reduction)

def _approx_ndcg_loss(labels,
                      logits,
                      weights=None,
                      reduction=core_losses.Reduction.SUM,
                      name=None,
                      alpha=10.):
  """Computes ApproxNDCG loss.

  ApproxNDCG ["A general approximation framework for direct optimization of
  information retrieval measures" by Qin et al.] is a smooth approximation
  to NDCG.

  Args:
    labels: A `Tensor` of the same shape as `logits` representing graded
      relevance.
    logits: A `Tensor` with shape [batch_size, list_size]. Each value is the
      ranking score of the corresponding item.
    weights: A scalar, a `Tensor` with shape [batch_size, 1] for list-wise
      weights, or a `Tensor` with shape [batch_size, list_size] for item-wise
      weights. If None, the weight of a list in the mini-batch is set to
      the sum of the labels of the items in that list.
    reduction: One of `tf.losses.Reduction` except `NONE`. Describes how to
      reduce training loss over batch.
    name: A string used as the name for this loss.
    alpha: The exponent in the generalized sigmoid function.

  Returns:
    An op for the ApproxNDCG loss.
  """
  with ops.name_scope(name, 'approx_ndcg_loss', (labels, logits, weights)):
    is_label_valid = label_valid_fn(labels)
    labels = array_ops.where(is_label_valid, labels,
                             array_ops.zeros_like(labels))
    logits = array_ops.where(
        is_label_valid, logits, -1e3 * array_ops.ones_like(logits) +
        math_ops.reduce_min(logits, axis=-1, keepdims=True))

    label_sum = math_ops.reduce_sum(labels, 1, keepdims=True)
    if weights is None:
      weights = array_ops.ones_like(label_sum)
    weights = array_ops.squeeze(weights)

    nonzero_mask = math_ops.greater(array_ops.reshape(label_sum, [-1]), 0.0)
    labels, logits, weights = [
        array_ops.boolean_mask(x, nonzero_mask)
        for x in [labels, logits, weights]
    ]

    gains = math_ops.pow(2., math_ops.to_float(labels)) - 1.
    ranks = approx_ranks(logits, alpha=alpha)
    discounts = 1. / math_ops.log1p(ranks)
    dcg = math_ops.reduce_sum(gains * discounts, -1)
    cost = -dcg * array_ops.squeeze(inverse_max_dcg(labels))

    return core_losses.compute_weighted_loss(cost,
                                             weights=weights,
                                             reduction=reduction)


#!flask/bin/python
from app import db
from app import User
from app import Item
from app import Order
from app import Orderrow

db.create_all()

#prep
u = User(username='john.doe@email.com', name='John Doe', address='homeaddress', postalCode='00100', city='Helsinki')
i1 = Item(productId='1001', productName='Shovel', productPrice='1.1')
i2 = Item(productId='2002', productName='Bucket', productPrice='2.5')
o = Order()
or1 = Orderrow(count='2')
or2 = Orderrow(count='3')

#append
i1.productRow.append(or1)
i2.productRow.append(or2)
o.orderrow.append(or1)
o.orderrow.append(or2)
u.order.append(o)

#add
db.session.add(u)
db.session.add(i1)
db.session.add(i2)
db.session.add(o)
db.session.add(or1)
db.session.add(or2)

#commit
db.session.commit()


#!/usr/bin/env python3

import os, re, sys

def main():
    raw_re = re.compile('RAW \[(\S+)\]')

    if os.isatty(sys.stdout.fileno()):
        print("Refusing to write binary data to a tty")
        sys.exit(1)

    while True:
        line = sys.stdin.readline()
        if line == "":
            break

        m = raw_re.search(line)
        if m != None:
            hex_array = m.group(1).split(",")
            int_array = list(map(lambda x: int(x, 16), hex_array))
            sys.stdout.buffer.write(bytes(int_array))

if __name__ == "__main__":
    main()


import matplotlib.pyplot as plt
import csv
import numpy as np
from configuration_path import *

def read_from_csv(file_path):
    volumes = []
    distances = []

    with open(file_path, 'r') as csvfile:
        reader = csv.reader(csvfile)
        for row in reader:
            if len(row) >= 2:  # Ensure there are at least two columns in the row
                volume = float(row[0])
                distance = float(row[1])
                volumes.append(volume)
                distances.append(distance)

    return volumes, distances

def plot_histogram(values):
    plt.hist(values, bins=100, edgecolor='black')
    mean_vol = np.mean(values)
    plt.vlines(mean_vol, 0, 50, linestyles ="dashed", colors ="g")
    plt.xlabel('Values')
    plt.ylabel('Frequency')
    plt.title('Histogram of Values, mean:' +  str(int(mean_vol)) + " STD:" + str(int(np.std(values))))
    plt.show()

print("reading volumes...")
# Read values from the CSV file
volumes,distances = read_from_csv(csv_file_path)

# Plot histogram from the values read from the CSV file
plot_histogram(volumes)

# Create a scatter plot
plt.scatter(distances, volumes, label='Volume vs. Distance', color='blue', marker='o')

# Add labels and a title
plt.xlabel('Distances')
plt.ylabel('Volumes')
plt.title('Volume vs. Distance')

# Add a legend
plt.legend()

# Show the plot
plt.grid(True)
plt.show()


from __future__ import print_function

import getpass
import json
import os
import time

from chromite.cbuildbot import buildbucket_lib
from chromite.cbuildbot import config_lib
from chromite.cbuildbot import constants
from chromite.cbuildbot import repository
from chromite.cbuildbot import manifest_version
from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
from chromite.lib import cache
from chromite.lib import git

site_config = config_lib.GetConfig()

class ChromiteUpgradeNeeded(Exception):
  """Exception thrown when it's detected that we need to upgrade chromite."""

  def __init__(self, version=None):
    Exception.__init__(self)
    self.version = version
    self.args = (version,)

  def __str__(self):
    version_str = ''
    if self.version:
      version_str = "  Need format version %r support." % (self.version,)
    return (
        "Your version of cbuildbot is too old; please resync it, "
        "and then retry your submission.%s" % (version_str,))

class ValidationError(Exception):
  """Thrown when tryjob validation fails."""

class RemoteTryJob(object):
  """Remote Tryjob that is submitted through a Git repo."""
  EXTERNAL_URL = os.path.join(site_config.params.EXTERNAL_GOB_URL,
                              'chromiumos/tryjobs')
  INTERNAL_URL = os.path.join(site_config.params.INTERNAL_GOB_URL,
                              'chromeos/tryjobs')

  # In version 3, remote patches have an extra field.
  # In version 4, cherry-picking is the norm, thus multiple patches are
  # generated.
  TRYJOB_FORMAT_VERSION = 4
  TRYJOB_FORMAT_FILE = '.tryjob_minimal_format_version'

  # Constants for controlling the length of JSON fields sent to buildbot.
  # - The trybot description is shown when the run starts, and helps users
  #   distinguish between their various runs. If no trybot description is
  #   specified, the list of patches is used as the description. The buildbot
  #   database limits this field to MAX_DESCRIPTION_LENGTH characters.
  # - When checking the trybot description length, we also add some PADDING
  #   to give buildbot room to add extra formatting around the fields used in
  #   the description.
  # - We limit the number of patches listed in the description to
  #   MAX_PATCHES_IN_DESCRIPTION. This is for readability only.
  # - Every individual field that is stored in a buildset is limited to
  #   MAX_PROPERTY_LENGTH. We use this to ensure that our serialized list of
  #   arguments fits within that limit.
  MAX_DESCRIPTION_LENGTH = 256
  MAX_PATCHES_IN_DESCRIPTION = 10
  MAX_PROPERTY_LENGTH = 1023
  PADDING = 50

  # Buildbucket_put response must contain 'buildbucket_bucket:bucket]',
  # '[config:config_name] and '[buildbucket_id:id]'.
  BUILDBUCKET_PUT_RESP_FORMAT = ('Successfully sent PUT request to '
                                 '[buildbucket_bucket:%s] '
                                 'with [config:%s] [buildbucket_id:%s].')

  def __init__(self, options, bots, local_patches):
    """Construct the object.

    Args:
      options: The parsed options passed into cbuildbot.
      bots: A list of configs to run tryjobs for.
      local_patches: A list of LocalPatch objects.
    """
    self.options = options
    self.use_buildbucket = options.use_buildbucket
    self.user = getpass.getuser()
    self.repo_cache = cache.DiskCache(self.options.cache_dir)
    cwd = os.path.dirname(os.path.realpath(__file__))
    self.user_email = git.GetProjectUserEmail(cwd)
    logging.info('Using email:%s', self.user_email)
    # Name of the job that appears on the waterfall.
    patch_list = options.gerrit_patches + options.local_patches
    self.name = options.remote_description
    if self.name is None:
      self.name = ''
      if options.branch != 'master':
        self.name = '[%s] ' % options.branch

      self.name += ','.join(patch_list[:self.MAX_PATCHES_IN_DESCRIPTION])
      if len(patch_list) > self.MAX_PATCHES_IN_DESCRIPTION:
        remaining_patches = len(patch_list) - self.MAX_PATCHES_IN_DESCRIPTION
        self.name += '... (%d more CLs)' % (remaining_patches,)

    self.bots = bots[:]
    self.slaves_request = options.slaves
    self.description = ('name: %s\n patches: %s\nbots: %s' %
                        (self.name, patch_list, self.bots))
    self.extra_args = options.pass_through_args
    if '--buildbot' not in self.extra_args:
      self.extra_args.append('--remote-trybot')

    self.extra_args.append('--remote-version=%s'
                           % (self.TRYJOB_FORMAT_VERSION,))
    self.local_patches = local_patches
    self.repo_url = self.EXTERNAL_URL
    self.cache_key = ('trybot',)
    self.manifest = None
    if repository.IsARepoRoot(options.sourceroot):
      self.manifest = git.ManifestCheckout.Cached(options.sourceroot)
      if repository.IsInternalRepoCheckout(options.sourceroot):
        self.repo_url = self.INTERNAL_URL
        self.cache_key = ('trybot-internal',)

  @property
  def values(self):
    return {
        'bot' : self.bots,
        'email' : [self.user_email],
        'extra_args' : self.extra_args,
        'name' : self.name,
        'slaves_request' : self.slaves_request,
        'user' : self.user,
        'version' : self.TRYJOB_FORMAT_VERSION,
        }

  def _VerifyForBuildbot(self):
    """Early validation, to ensure the job can be processed by buildbot."""

    # Buildbot stores the trybot description in a property with a 256
    # character limit. Validate that our description is well under the limit.
    if (len(self.user) + len(self.name) + self.PADDING >
        self.MAX_DESCRIPTION_LENGTH):
      logging.warning('remote tryjob description is too long, truncating it')
      self.name = self.name[:self.MAX_DESCRIPTION_LENGTH - self.PADDING] + '...'

    # Buildbot will set extra_args as a buildset 'property'.  It will store
    # the property in its database in JSON form.  The limit of the database
    # field is 1023 characters.
    if len(json.dumps(self.extra_args)) > self.MAX_PROPERTY_LENGTH:
      raise ValidationError(
          'The number of extra arguments passed to cbuildbot has exceeded the '
          'limit.  If you have a lot of local patches, upload them and use the '
          '-g flag instead.')

  def _Submit(self, workdir, testjob, dryrun):
    """Internal submission function.  See Submit() for arg description."""
    # TODO(rcui): convert to shallow clone when that's available.
    current_time = str(int(time.time()))

    ref_base = os.path.join('refs/tryjobs', self.user_email, current_time)
    for patch in self.local_patches:
      # Isolate the name; if it's a tag or a remote, let through.
      # Else if it's a branch, get the full branch name minus refs/heads.
      local_branch = git.StripRefsHeads(patch.ref, False)
      ref_final = os.path.join(ref_base, local_branch, patch.sha1)

      checkout = patch.GetCheckout(self.manifest)
      checkout.AssertPushable()
      print('Uploading patch %s' % patch)
      patch.Upload(checkout['push_url'], ref_final, dryrun=dryrun)

      # TODO(rcui): Pass in the remote instead of tag. http://crosbug.com/33937.
      tag = constants.EXTERNAL_PATCH_TAG
      if checkout['remote'] == site_config.params.INTERNAL_REMOTE:
        tag = constants.INTERNAL_PATCH_TAG

      self.extra_args.append('--remote-patches=%s:%s:%s:%s:%s'
                             % (patch.project, local_branch, ref_final,
                                patch.tracking_branch, tag))

    self._VerifyForBuildbot()
    repository.UpdateGitRepo(workdir, self.repo_url)
    version_path = os.path.join(workdir,
                                self.TRYJOB_FORMAT_FILE)
    with open(version_path, 'r') as f:
      try:
        val = int(f.read().strip())
      except ValueError:
        raise ChromiteUpgradeNeeded()
      if val > self.TRYJOB_FORMAT_VERSION:
        raise ChromiteUpgradeNeeded(val)

    if self.use_buildbucket:
      self._PostConfigsToBuildBucket(testjob, dryrun)
    else:
      self._PushConfig(workdir, testjob, dryrun, current_time)

  def _GetBuilder(self, bot):
    """Find and return the builder for bot."""

    # Default to etc builder.
    builder = 'etc'
    if site_config[bot] and site_config[bot]['_template']:
      builder = site_config[bot]['_template']

    return builder

  def _GetProperties(self, bot):
    """Construct and return properties for buildbucket request."""
    properties = dict(self.values)
    properties.update({
        'cbb_config': bot,
        'cbb_extra_args': self.extra_args,
        'owners': [self.user_email]
    })
    return properties

  def _PutConfigToBuildBucket(self, bot, http, testjob, dryrun):
    """Put the tryjob request to buildbucket.

    Args:
      bot: The bot config to put.
      http: An authorized http instance.
      testjob: Whether to use the test instance of the buildbucket server.
      dryrun: Whether a dryrun.
    """

    body = json.dumps({
        'bucket': constants.TRYSERVER_BUILDBUCKET_BUCKET,
        'parameters_json': json.dumps({
            'builder_name': self._GetBuilder(bot),
            'properties': self._GetProperties(bot),
        }),
    })
    content = buildbucket_lib.PutBuildBucket(
        body, http, testjob, dryrun)
    buildbucket_id = buildbucket_lib.GetBuildId(content)

    if buildbucket_id is not None:
      print(self.BUILDBUCKET_PUT_RESP_FORMAT %
            (constants.TRYSERVER_BUILDBUCKET_BUCKET, bot, buildbucket_id))

  def _PostConfigsToBuildBucket(self, testjob=False, dryrun=False):
    """Posts the tryjob configs to buildbucket.

    Args:
      dryrun: Whether to skip the request to buildbucket.
      testjob: Whether to use the test instance of the buildbucket server.
    """
    http = buildbucket_lib.BuildBucketAuth(
        service_account=buildbucket_lib.GetServiceAccount(
            constants.CHROMEOS_SERVICE_ACCOUNT))

    for bot in self.bots:
      self._PutConfigToBuildBucket(bot, http, testjob, dryrun)

  def _PushConfig(self, workdir, testjob, dryrun, current_time):
    """Pushes the tryjob config to Git as a file.

    Args:
      workdir: see Submit()
      testjob: see Submit()
      dryrun: see Submit()
      current_time: the current time as a string represention of the time since
        unix epoch.
    """
    push_branch = manifest_version.PUSH_BRANCH

    remote_branch = None
    if testjob:
      remote_branch = git.RemoteRef('origin', 'refs/remotes/origin/test')
    git.CreatePushBranch(push_branch, workdir, sync=False,
                         remote_push_branch=remote_branch)
    file_name = '%s.%s' % (self.user, current_time)
    user_dir = os.path.join(workdir, self.user)
    if not os.path.isdir(user_dir):
      os.mkdir(user_dir)

    fullpath = os.path.join(user_dir, file_name)
    with open(fullpath, 'w+') as job_desc_file:
      json.dump(self.values, job_desc_file)

    git.RunGit(workdir, ['add', fullpath])
    extra_env = {
        # The committer field makes sure the creds match what the remote
        # gerrit instance expects while the author field allows lookup
        # on the console to work.  http://crosbug.com/27939
        'GIT_COMMITTER_EMAIL' : self.user_email,
        'GIT_AUTHOR_EMAIL'    : self.user_email,
    }
    git.RunGit(workdir, ['commit', '-m', self.description],
               extra_env=extra_env)

    try:
      git.PushWithRetry(push_branch, workdir, retries=3, dryrun=dryrun)
    except cros_build_lib.RunCommandError:
      logging.error(
          'Failed to submit tryjob.  This could be due to too many '
          'submission requests by users.  Please try again.')
      raise

  def Submit(self, workdir=None, testjob=False, dryrun=False):
    """Submit the tryjob through Git.

    Args:
      workdir: The directory to clone tryjob repo into.  If you pass this
               in, you are responsible for deleting the directory.  Used for
               testing.
      testjob: Submit job to the test branch of the tryjob repo.  The tryjob
               will be ignored by production master.
      dryrun: Setting to true will run everything except the final submit step.
    """
    if workdir is None:
      with self.repo_cache.Lookup(self.cache_key) as ref:
        self._Submit(ref.path, testjob, dryrun)
    else:
      self._Submit(workdir, testjob, dryrun)

  def GetTrybotWaterfallLink(self):
    """Get link to the waterfall for the user."""
    # The builders on the trybot waterfall are named after the templates.
    builders = set(site_config[bot]['_template'] for bot in self.bots)

    # Note that this will only show the jobs submitted by the user in the last
    # 24 hours.
    return '%s/waterfall?committer=%s&%s' % (
        constants.TRYBOT_DASHBOARD, self.user_email,
        '&'.join('builder=%s' % b for b in sorted(builders)))


from .alu_instructions import *
from .memory_instructions import *
from .control_instructions import *

OPCODES = {
    # * Means a general purpose register
    # @ means a registers to write to
    # $ Means an constant operand
    # MEM[] Means a location in memory
    # {} Are special registers: PC,
    # TYPE R ~ |OPC| * | */$ | */$ |
    # TYPE I ~ |OPC| */$   |  */$  |
    # TYPE J ~ |OPC|      */$      |

    # Arithmetic, comparisons and logic
    "ADD" : ADD,  # | ADD  | @dest | *src1 | *src2 |  ->  @dest = *src1 + *src2
    "ADDI": ADDI, # | ADDI | @dest | *src1 | $valu |  ->  @dest = *src1 + $valu
    "SUB" : SUB,  # | SUB  | @dest | *src1 | *src2 |  ->  @dest = *src1 - *src2
    "SUBI": SUBI, # | SUBI | @dest | *src1 | $valu |  ->  @dest = *src1 - $valu
    "MUL" : MUL,  # | MUL  | @dest | *src1 | *src2 |  ->  @dest = *src1 * *src2
    "DIV" : DIV,  # | DIV  | @dest | *src1 | *src2 |  ->  @dest = *src1 / *src2
    "IDIV": IDIV, # | IDIV | @dest | *src1 | *src2 |  ->  @dest = *src1 // *src2
    "CMP" : CMP,  # | CMP  | @dest | *src1 | *src2 |  ->  @dest = 1, 0, -1 IF *src1 >/=/< *src2

    # Memory access
    "LD"  : LD,   # | LD   | @dest | *addr | *offs |  ->  @dest = MEM[*addr + *offs]
    "LDC" : LDC,  # | LDC  | @dest | $valu |          ->  @dest = $valu
    "MOV" : MOV,  # | MOV  | @dest | *src  |          ->  @dest = *src
    "ST"  : ST,   # | ST   | *addr | *src  | *offs |  ->  MEM[*addr + *offs] = *src
    "STC" : STC,  # | STC  | *addr | $valu |          ->  MEM[*addr] = $valu

    # Branches and Jumps
    "J"   : J,    # | J    | *addr |                  ->  {PC} = *addr
    "BEQZ": BEQZ, # | BEQZ | *addr | *src  |          ->  {PC} = *addr IF *src = 0
    "BNEZ": BNEZ, # | BNEZ | *addr | *src  |          ->  {PC} = *addr IF *src != 0
    "BLTZ": BLTZ, # | BLTZ | *addr | *src  |          ->  {PC} = *addr IF *src < 0
    "BGEZ": BGEZ  # | BGEZ | *addr | *src  |          ->  {PC} = *addr IF *src >= 0
}


from scNodes.core.node import *

def create():
    return MathNode()

class MathNode(Node):
    description = "Perform various mathematical operations on the input images' pixel data, such as taking the logarithm."
    title = "Math"
    colour = (143 / 255, 143 / 255, 143 / 255, 1.0)
    group = "Converters"
    sortid = 2000

    OPERATIONS = ["Power", "Log", "Invert", "Threshold", "Add constant", "Multiply", "Absolute"]

    def __init__(self):
        super().__init__()

        self.connectable_attributes["dataset_in"] = ConnectableAttribute(ConnectableAttribute.TYPE_MULTI, ConnectableAttribute.INPUT, self, [ConnectableAttribute.TYPE_DATASET, ConnectableAttribute.TYPE_IMAGE])
        self.connectable_attributes["dataset_in"].colour = ConnectableAttribute.COLOUR[ConnectableAttribute.TYPE_DATASET]
        self.connectable_attributes["dataset_out"] = ConnectableAttribute(ConnectableAttribute.TYPE_MULTI, ConnectableAttribute.OUTPUT, self, [ConnectableAttribute.TYPE_DATASET, ConnectableAttribute.TYPE_IMAGE])
        self.connectable_attributes["dataset_out"].colour = ConnectableAttribute.COLOUR[ConnectableAttribute.TYPE_DATASET]

        self.params["operation"] = 0

        # operation specific vars
        self.params["power"] = 2.0
        self.params["threshold"] = 100
        self.params["threshold_keep_high"] = True
        self.params["threshold_fill_value"] = 0
        self.params["threshold_make_mask"] = True
        self.params["constant"] = 100
        self.params["factor"] = 2.0

        self.params["last_image_min"] = 0
        self.params["last_image_max"] = 256

        self.params["replace_nan_by"] = 0
        self.params["replace_inf_by"] = 0
        self.params["replace_inf"] = False

    def render(self):
        if super().render_start():
            self.connectable_attributes["dataset_in"].render_start()
            self.connectable_attributes["dataset_out"].render_start()
            self.connectable_attributes["dataset_in"].render_end()
            self.connectable_attributes["dataset_out"].render_end()

            imgui.spacing()
            imgui.separator()
            imgui.spacing()

            imgui.text("Operation")
            imgui.set_next_item_width(170)
            _c, self.params["operation"] = imgui.combo("##operation", self.params["operation"], MathNode.OPERATIONS)
            self.any_change = self.any_change or _c
            imgui.push_item_width(120)
            if self.params["operation"] == 0:
                _c, self.params["power"] = imgui.slider_float("power", self.params["power"], -2.0, 2.0, format = "%.1f")
                self.any_change = self.any_change or _c
            elif self.params["operation"] == 3:
                _c, self.params["threshold"] = imgui.slider_float("threshold", self.params["threshold"], self.params["last_image_min"], self.params["last_image_max"], format = "%.0f")
                self.any_change = self.any_change or _c
                _c, self.params["threshold_make_mask"] = imgui.checkbox("output mask", self.params["threshold_make_mask"])
                self.any_change = self.any_change or _c
                if not self.params["threshold_make_mask"]:
                    _c, self.params["threshold_fill_value"] = imgui.slider_float("fill", self.params["threshold_fill_value"], self.params["last_image_min"], self.params["last_image_max"], format = "%.0f")
                    self.any_change = self.any_change or _c
                _c, self.params["threshold_keep_high"] = imgui.checkbox("keep low", self.params["threshold_keep_high"])
                Node.tooltip("By default, values lower than the threshold value are replaced/masked.\n"
                             "Check this box to invert the behaviour: values higher than the treshold\n"
                             "are replaced/masked instead.")
                self.any_change = self.any_change or _c
            elif self.params["operation"] == 4:
                _c, self.params["constant"] = imgui.slider_float("value", self.params["constant"], -1000, 1000, format = "%.0f")
                self.any_change = self.any_change or _c
            elif self.params["operation"] == 5:
                _c, self.params["factor"] = imgui.drag_float("factor", self.params["factor"], 20.0, 0.0, 0.0, format = "%.2f")
                self.any_change = self.any_change or _c
            Node.tooltip("Use ctrl + click on a slider to override it and manually enter a value.")

            imgui.spacing()
            header_expanded, _ = imgui.collapsing_header("Advanced", None)
            if header_expanded:
                self.render_advanced()
            super().render_end()

    def render_advanced(self):
        imgui.push_item_width(40)
        imgui.text("Replace NaN by:")
        _c, self.params["replace_nan_by"] = imgui.input_float("##nan", self.params["replace_nan_by"], format = "%.0f")
        self.any_change = self.any_change or _c
        imgui.text("Replace inf by:")
        _c, self.params["replace_inf_by"] = imgui.input_float("##inf", self.params["replace_inf_by"], format = "%.0f")
        self.any_change = self.any_change or _c
        imgui.same_line()
        _c, self.params["replace_inf"] = imgui.checkbox("replace inf", self.params["replace_inf"])
        self.any_change = self.any_change or _c
        imgui.pop_item_width()

    def get_image_impl(self, idx):
        datasource = self.connectable_attributes["dataset_in"].get_incoming_node()
        if datasource:
            img_in = datasource.get_image(idx)
            pxd = img_in.load()

            if self.params["operation"] == 0:
                pxd = pxd ** self.params["power"]
            elif self.params["operation"] == 1:
                pxd = np.log(pxd)
            elif self.params["operation"] == 2:
                pxd = - pxd
            elif self.params["operation"] == 3:
                if self.FRAME_REQUESTED_BY_IMAGE_VIEWER:
                    self.params["last_image_min"] = np.amin(pxd)
                    self.params["last_image_max"] = np.amax(pxd)
                if self.params["threshold_keep_high"]:
                    if self.params["threshold_make_mask"]:
                        pxd = pxd < self.params["threshold"]
                    else:
                        pxd[pxd < self.params["threshold"]] = self.params["threshold_fill_value"]
                else:
                    if self.params["threshold_make_mask"]:
                        pxd = pxd >= self.params["threshold"]
                    else:
                        pxd[pxd >= self.params["threshold"]] = self.params["threshold_fill_value"]
            elif self.params["operation"] == 4:
                pxd += self.params["constant"]
            elif self.params["operation"] == 5:
                pxd *= self.params["factor"]
            elif self.params["operation"] == 6:
                pxd = np.abs(pxd)

            # replace nan and inf
            _inf_val = self.params["replace_inf_by"] if self.params["replace_inf"] else None
            np.nan_to_num(pxd, copy=False, nan=self.params["replace_nan_by"], posinf = _inf_val, neginf = _inf_val)
            img_out = img_in.clone()
            img_out.data = pxd
            return img_out


from pprint import pprint
from pymongo import MongoClient
from pymongo import ReturnDocument
from bson.objectid import ObjectId

class MongoDBManagement:
    def __init__(self):
        self.client = MongoClient('localhost', 27017)
        self.db = self.client['mongoDatabaseDemo']
        self.collection_patient = self.db['patients']

    def insert_one_patient(self):
        self.collection_patient.insert_one({
            "NAMA": input("MASUKKAN NAMA PASIEN: "),
            "UMUR" :input("MASUKKAN UMUR: "),
            "WALI": input("MASUKKAN NAMA WALI PASIEN: "),
            "ALAMAT": input("MASUKKAN ALAMAT: "),
        })

    # def delete_one_patient(self):
    #     self.collection_patient.delete_one({"NAMA": input("HAPUS PASIEN: ")})
if __name__ == "__main__":
    mongoDB = MongoDBManagement()
    mongoDB.insert_one_patient()
    # mongoDB.delete_one_patient()


import board
import busio
import digitalio
import gc

from laser_egismos import Laser, LaserCommandFailedError

def initialize_uart():
    global uart
    uart = busio.UART(board.TX, board.RX, baudrate=9600)
    return uart

uart = initialize_uart()

laser_power = digitalio.DigitalInOut(board.D3)
laser_power.switch_to_output(True)

laser = Laser(uart)

def laser_on():
    try:
        laser.laser_on()
    except Exception:
        uart.reset_input_buffer()
        print("Laser on failed")
        return "Laser on failed"

def laser_off():
    try:
        laser.laser_off()
    except Exception:
        uart.reset_input_buffer()
        print("Laser off failed")
        return "Laser off failed"

def buzzer_on():
    try:
        laser.buzzer_on()
    except Exception:
        uart.reset_input_buffer()
        print("Buzzer on failed")
        return "Buzzer on failed"

def buzzer_off():
    try:
        laser.buzzer_off()
    except Exception:
        uart.reset_input_buffer()
        print("Buzzer off failed")
        return "Buzzer off failed"

def take_measurment():
    try:
        distance = laser.distance
        distance = distance/100
        distance = round(distance,2)
        distance = f"{distance}m"
        return(distance)
    except Exception as e:
        print(e)
        uart.reset_input_buffer()
        return "Measurement Failed"


from PyQt5 import QtCore, QtGui, QtWidgets

from .QuizUI import QuizUI

class QuestionWidget(QtWidgets.QWidget):
    def __init__(self,test,accManager,rdm,user):
        self.test=test

        self.accManager=accManager

        self.rdm=rdm
        self.user=user

        super().__init__()
        self.setGeometry(QtCore.QRect(90, 10, 531, 41))
        self.setObjectName("QuestionWidget")
        self.horizontalLayout = QtWidgets.QHBoxLayout(self)
        self.horizontalLayout.setContentsMargins(0, 0, 0, 0)
        self.horizontalLayout.setObjectName("horizontalLayout")
        self.label = QtWidgets.QLabel(self)
        self.label.setObjectName("label")
        self.horizontalLayout.addWidget(self.label)
        self.label_2 = QtWidgets.QLabel(self)
        self.label_2.setObjectName("label_2")
        self.horizontalLayout.addWidget(self.label_2)
        self.label_3 = QtWidgets.QLabel(self)
        self.label_3.setObjectName("label_3")
        self.horizontalLayout.addWidget(self.label_3)
        self.pushButton = QtWidgets.QPushButton(self)
        self.pushButton.setObjectName("pushButton")
        self.horizontalLayout.addWidget(self.pushButton)

        self.retranslateUi()
        QtCore.QMetaObject.connectSlotsByName(self)
        self.pushButton.clicked.connect(self.openQuiz)

    def retranslateUi(self):
        _translate = QtCore.QCoreApplication.translate
        self.setWindowTitle(_translate("Form", "Form"))
        self.label.setText(_translate("Form", self.test.testName))
        self.label_2.setText(_translate("Form", self.test.date_created))
        self.label_3.setText(_translate("Form", self.accManager.IDtoName(self.test.authorID)))
        self.pushButton.setText(_translate("Form", "Solve"))

    def openQuiz(self):
        self.quiz = QuizUI(self.rdm,self.user)
        self.quiz.setGeometry(100,100,600,600)
        self.quiz.setupQuiz(self.test)

        self.quiz.show()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# 25.11.2018 10:56:13 MSK

import os
import re

names = ['bashgazet', 'bashdram', 'gsrb', 'jeshlek', 'kiskeufa', 'kulturarb', 'president-rb', 'tabin']

def main():
    wordcount = 0
    for folder in names:
        pth = 'texts-{}/'.format(folder)
        lst = os.listdir(pth)
        for fl in lst:
            with open(pth + fl) as f:
                c = f.read()
                words = c.split()
                for word in words:
                    if re.search('[а-яҙңҫүһғәҡөА-ЯҘҢҪҮҺҒӘҠӨ]', c):
                        wordcount += 1
    print(wordcount)
    return 0

if __name__ == '__main__':
    main()


from __future__ import print_function
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable

# class Inversion_feature(nn.Module):
#     def __init__(self, nc, ngf, nz, truncation, c):
#         super(Inversion_feature, self).__init__()
#
#         self.nc = nc #3 channel
#         self.ngf = ngf #media variable
#         self.nz = nz #input feature
#         self.truncation = truncation #truncation is the number of remain input must has
#         self.c = c #50.
#
#         # self.decoder = nn.Sequential(
#         #     # input is Z
#         #     nn.ConvTranspose2d(nz, ngf * 8, 4, 1, 0),
#         #     nn.BatchNorm2d(ngf * 8),
#         #     nn.Tanh(),
#         #     # state size. (ngf*8) x 4 x 4
#         #     nn.ConvTranspose2d(ngf * 8, ngf * 4, 4, 2, 1),
#         #     nn.BatchNorm2d(ngf * 4),
#         #     nn.Tanh(),
#         #     # state size. (ngf*4) x 8 x 8
#         #     nn.ConvTranspose2d(ngf * 4, ngf * 2, 4, 2, 1),
#         #     nn.BatchNorm2d(ngf * 2),
#         #     nn.Tanh(),
#         #     # state size. (ngf*2) x 16 x 16
#         #     nn.ConvTranspose2d(ngf * 2, ngf, 4, 2, 1),#in_channels,out_channels,kernel_size,stride,padding
#         #     nn.BatchNorm2d(ngf),
#         #     nn.Tanh(),
#         #     # # state size. (ngf) x 32 x 32
#         #     nn.ConvTranspose2d(ngf, nc, 4, 2, 1),
#         #     nn.Sigmoid()
#         #     # state size. (nc) x 64 x 64
#         # )
#         self.decoder = nn.Sequential(
#             # input is Z
#             nn.ConvTranspose2d(nz, ngf * 4, 4, 1, 0),
#             nn.BatchNorm2d(ngf * 4),
#             nn.Tanh(),
#             # state size. (ngf*8) x 4 x 4
#             nn.ConvTranspose2d(ngf * 4, ngf * 2, 4, 2, 1),
#             nn.BatchNorm2d(ngf * 2),
#             nn.Tanh(),
#             # state size. (ngf*4) x 8 x 8
#             nn.ConvTranspose2d(ngf * 2, ngf, 4, 2, 1),
#             nn.BatchNorm2d(ngf),
#             nn.Tanh(),
#             # state size. (ngf*2) x 16 x 16
#             # nn.ConvTranspose2d(ngf * 2, ngf, 4, 2, 1),  # in_channels,out_channels,kernel_size,stride,padding
#             # nn.BatchNorm2d(ngf),
#             # nn.Tanh(),
#             # # state size. (ngf) x 32 x 32
#             nn.ConvTranspose2d(ngf, nc, 4, 2, 1),
#             nn.Sigmoid()
#         )
#
#     def forward(self, x):
#         topk, indices = torch.topk(x, self.truncation)
#         topk = torch.clamp(torch.log(topk), min=-1000) + self.c
#         topk_min = topk.min(1, keepdim=True)[0]
#         topk = topk + F.relu(-topk_min)
#         x = torch.zeros(len(x), self.nz).cuda().scatter_(1, indices, topk)
#
#         x = x.view(-1, self.nz, 1, 1)# 10, 1, 1
#         # print("x0", x.shape)
#         x = self.decoder(x)
#         # print("x1", x.shape)
#         x = x.view(-1, 3, 32, 32)#-1, 1, 64, 64
#         return x

class InversionNet(nn.Module):

    def __init__(self):
        super(InversionNet, self).__init__()

        self.linear = torch.nn.Linear(100, 1024 * 2 * 2)#1024 * 4 * 4

        self.conv1 = nn.Sequential(
            nn.ConvTranspose2d(
                in_channels=1024, out_channels=512, kernel_size=4,
                stride=2, padding=1, bias=False
            ),
            nn.BatchNorm2d(512),
            nn.ReLU(inplace=True)
        )
        self.conv2 = nn.Sequential(
            nn.ConvTranspose2d(
                in_channels=512, out_channels=256, kernel_size=4,
                stride=2, padding=1, bias=False
            ),
            nn.BatchNorm2d(256),
            nn.ReLU(inplace=True)
        )
        self.conv3 = nn.Sequential(
            nn.ConvTranspose2d(
                in_channels=256, out_channels=128, kernel_size=4,
                stride=2, padding=1, bias=False
            ),
            nn.BatchNorm2d(128),
            nn.ReLU(inplace=True)
        )
        self.conv4 = nn.Sequential(
            nn.ConvTranspose2d(
                in_channels=128, out_channels=3, kernel_size=4,
                stride=2, padding=1, bias=False
            )
        )
        self.out = torch.nn.Tanh()

    def forward(self, x):
        # Project and reshape
        x = self.linear(x)
        x = x.view(x.shape[0], 1024, 2, 2)
        # Convolutional layers
        x = self.conv1(x)
        x = self.conv2(x)
        x = self.conv3(x)
        x = self.conv4(x)
        # Apply Tanh
        return self.out(x)

class DiscriminativeNet(nn.Module):

    def __init__(self):
        super(DiscriminativeNet, self).__init__()

        self.conv1 = nn.Sequential(
            nn.Conv2d(
                in_channels=3, out_channels=128, kernel_size=4,
                stride=2, padding=1, bias=False
            ),
            nn.LeakyReLU(0.2, inplace=True)
        )
        self.conv2 = nn.Sequential(
            nn.Conv2d(
                in_channels=128, out_channels=256, kernel_size=4,
                stride=2, padding=1, bias=False
            ),
            nn.BatchNorm2d(256),
            nn.LeakyReLU(0.2, inplace=True)
        )
        self.conv3 = nn.Sequential(
            nn.Conv2d(
                in_channels=256, out_channels=512, kernel_size=4,
                stride=2, padding=1, bias=False
            ),
            nn.BatchNorm2d(512),
            nn.LeakyReLU(0.2, inplace=True)
        )
        self.conv4 = nn.Sequential(
            nn.Conv2d(
                in_channels=512, out_channels=1024, kernel_size=4,
                stride=2, padding=1, bias=False
            ),
            nn.BatchNorm2d(1024),
            nn.LeakyReLU(0.2, inplace=True)
        )
        self.out = nn.Sequential(
            nn.Linear(1024 * 2 * 2, 1),#1024 * 4 * 4
            nn.Sigmoid(),
        )

    def forward(self, x):
        # Convolutional layers
        x = self.conv1(x)
        x = self.conv2(x)
        x = self.conv3(x)
        x = self.conv4(x)
        # Flatten and apply sigmoid
        x = x.view(-1, 1024 * 2 * 2)#1024 * 4 * 4
        x = self.out(x)
        return x

# Noise
def noise(size):
    n = Variable(torch.randn(size, 100)) #10 we don't need much,just try
    if torch.cuda.is_available(): return n.cuda()
    return n


"""
### Calculate distances on a raster. ###

Module to calculate the distance from a pixel value to other pixels.
"""

# Standard library
import sys; sys.path.append("../../")
from typing import Union, List, Optional

# External
import numpy as np
from osgeo import gdal

# Internal
from buteo.utils import (
    utils_base,
    utils_io,
    utils_path,
)
from buteo.raster import core_raster, core_raster_io
from buteo.array.distance import convolve_distance

def _raster_get_proximity(
    raster: Union[str, gdal.Dataset],
    target_value: Union[int, float] = 1,
    unit: str = "geo",
    out_path: Optional[str] = None,
    max_dist: Union[int, float] = 1000,
    add_border: bool = False,
    border_value: Union[int, float] = 0,
    inverted: bool = False,
    overwrite: bool = True,
) -> str:
    """
    Internal. Calculate the proximity of a raster to values.
    """
    assert isinstance(raster, (str, gdal.Dataset)), f"Invalid raster. {raster}"
    assert isinstance(target_value, (int, float)), f"Invalid target_value. {target_value}"
    assert isinstance(unit, str), f"Invalid unit. {unit}"

    if out_path is None:
        out_path = utils_path._get_temp_filepath("proximity_raster.tif")

    array = core_raster_io.raster_to_array(raster, filled=True)

    if unit.lower() == "geo":
        metadata = core_raster._get_basic_metadata_raster(raster)
        pixel_height = metadata["pixel_height"]
        pixel_width = metadata["pixel_width"]
    else:
        pixel_height = 1
        pixel_width = 1

    if add_border:
        padded = np.full((array.shape[0] + 2, array.shape[1] + 2, array.shape[2]), border_value, dtype=array.dtype)
        padded[1:-1, 1:-1] = array
        array = padded

    array = convolve_distance(
        array,
        target=target_value,
        maximum_distance=max_dist,
        pixel_width=pixel_width,
        pixel_height=pixel_height,
    )

    if inverted:
        array = array.max() - array

    if add_border:
        array = array[1:-1, 1:-1]

    out_path = core_raster_io.array_to_raster(
        array,
        reference=raster,
        out_path=out_path,
        overwrite=overwrite,
    )

    return out_path

def raster_get_proximity(
    raster: Union[str, gdal.Dataset, List[Union[str, gdal.Dataset]]],
    target_value: Union[int, float] = 1,
    unit: str = "geo",
    out_path: Optional[Union[str, List[str]]] = None,
    max_dist: Union[int, float] = 1000,
    add_border: bool = False,
    border_value: Union[int, float] = 0,
    inverted: bool = False,
    overwrite: bool = True,
    prefix: str = "",
    suffix: str = "",
    add_uuid: bool = False,
    add_timestamp: bool = False,
) -> Union[str, List[str]]:
    """
    Calculate the proximity of input_raster to values.

    Parameters
    ----------
    raster : Union[str, gdal.Dataset, List]
        The raster(s) to use as input.

    target_value : Union[int, float], optional
        The value to use as target, default: 1.

    unit : str, optional
        The unit to use for the distance, GEO or PIXEL, default: "GEO".

    out_path : Union[str, None, List], optional
        The output path, default: None.

    max_dist : Union[int, float], optional
        The maximum distance to use, default: 1000.

    add_border : bool, optional
        If True, a border will be added to the raster, default: False.

    border_value : Union[int, float], optional
        The value to use for the border, default: 0.

    inverted : bool, optional
        If True, the target will be inversed, default: False.

    overwrite : bool, optional
        If the output path exists already, should it be overwritten?, default: True.

    prefix : str, optional
        Prefix to add to the output, default: "".

    suffix : str, optional
        Suffix to add to the output, default: "_proximity".

    add_uuid : bool, optional
        Should a uuid be added to the output path?, default: False.

    add_timestamp : bool, optional
        Should a timestamp be added to the output path?, default: False.

    Returns
    -------
    str
        A path to a raster with the calculated proximity.
    """
    utils_base._type_check(raster, [str, gdal.Dataset, [str, gdal.Dataset]], "raster")
    utils_base._type_check(target_value, [int, float], "target_value")
    utils_base._type_check(unit, [str], "unit")
    utils_base._type_check(out_path, [str, [str], None], "out_path")
    utils_base._type_check(max_dist, [int, float], "max_dist")
    utils_base._type_check(add_border, [bool], "add_border")
    utils_base._type_check(border_value, [int, float], "border_value")
    utils_base._type_check(inverted, [bool], "inverted")
    utils_base._type_check(overwrite, [bool], "overwrite")
    utils_base._type_check(prefix, [str], "prefix")
    utils_base._type_check(suffix, [str], "suffix")
    utils_base._type_check(add_uuid, [bool], "add_uuid")
    utils_base._type_check(add_timestamp, [bool], "add_timestamp")

    assert unit.lower() in ["geo", "pixel"], f"Invalid unit. {unit}"

    input_is_list = isinstance(raster, list)
    input_rasters = utils_io._get_input_paths(raster, "raster")
    output_rasters = utils_io._get_output_paths(
        input_rasters,
        out_path,
        in_place=False,
        prefix=prefix,
        suffix=suffix,
        add_uuid=add_uuid,
        add_timestamp=add_timestamp,
        change_ext="tif"
    )

    utils_path._delete_if_required_list(output_rasters, overwrite)

    output = []
    for idx, in_raster in enumerate(input_rasters):
        output.append(_raster_get_proximity(
            in_raster,
            target_value=target_value,
            unit=unit.lower(),
            out_path=output_rasters[idx],
            max_dist=max_dist,
            add_border=add_border,
            border_value=border_value,
            inverted=inverted,
            overwrite=overwrite,
        ))

    if input_is_list:
        return output

    return output[0]


messages = [
		# Message 0
		(0x100, 'A Test Message',
			'This is a test message.\nWith some lines.\nAnd more lines.\nAnd a really long line to see if nw4r::lyt can handle wrapping or not...\nFinal line.'
			),
		]

import struct, sys, os.path, codecs

if len(sys.argv) > 1:
	target = sys.argv[1]
elif os.path.exists('/home/me/Games/Newer'):
	target = '/home/me/Games/Newer/ISO/files/NewerRes/Messages.bin'
else:
	target = 'Messages.bin'

messageCount = len(messages)

stringOffset = (messageCount * 0xC) + 4
infoStruct = struct.Struct('>III')

headerData = bytearray(struct.pack('>I', messageCount))
stringData = bytearray()

for msgID, title, msg in messages:
	titleOffset = stringOffset + len(stringData)
	stringData += codecs.utf_16_be_encode(title)[0] + '\0\0'
	msgOffset = stringOffset + len(stringData)
	stringData += codecs.utf_16_be_encode(msg)[0] + '\0\0'

	headerData += infoStruct.pack(msgID, titleOffset, msgOffset)

with open(target, 'wb') as out:
	out.write(headerData)
	out.write(stringData)


class Solution(object):
    def threeSum(self, nums):
        """
        :type nums: List[int]
        :rtype: List[List[int]]
        """
        # Init results array to return
        results = []

        # Sort array to handle duplicates
        nums.sort()

        for i, num in enumerate(nums):
            # Ignore duplicates for first number
            if i > 0 and num == nums[i - 1]:
                continue

            # Two pointers at start and end of subarray next to i
            left = i + 1
            right = len(nums) - 1

            # While pointers do not intersect
            while left < right:
                # Temp sum
                sum = num + nums[left] + nums[right]

                if sum == 0:
                    # Triplet found, add to results
                    results.append([num, nums[left], nums[right]])

                    # Only have to update one pointer, other conditions will update pointers later on
                    left += 1
                    while nums[left] == nums[left - 1] and left < right:
                        left += 1

                # If total is greater than 0, move right pointer down to decrease sum
                elif sum > 0:
                    right -= 1

                # If total is less than 0, move left pointer up to increase sum
                else:
                    left += 1

        return results

ob = Solution()
nums = [-1,0,1,2,-1,-4]

print(ob.threeSum(nums))