from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tempfile

from tensorflow.contrib import framework as contrib_framework
from tensorflow.contrib import layers
from tensorflow.contrib.framework.python.ops import variables
from tensorflow.contrib.layers.python.layers import initializers
from tensorflow.contrib.layers.python.layers import optimizers
from tensorflow.contrib.learn.python.learn import evaluable
from tensorflow.contrib.learn.python.learn import metric_spec
from tensorflow.contrib.learn.python.learn import trainable
from tensorflow.contrib.learn.python.learn.estimators import estimator
from tensorflow.contrib.metrics.python.ops import metric_ops
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import control_flow_ops
from tensorflow.python.ops import init_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import nn
from tensorflow.python.ops import partitioned_variables
from tensorflow.python.ops import standard_ops
from tensorflow.python.ops import variable_scope
from tensorflow.python.training import training as train

_CLASSES = "classes"
_TOP_K = "top_k"
_PROBABILITIES = "probabilities"
_DEFAULT_LEARNING_RATE = 0.01

def _as_iterable(preds, output):
  for pred in preds:
    yield pred[output]

def _get_optimizer(optimizer):
  if callable(optimizer):
    return optimizer()
  else:
    return optimizer

def _get_default_optimizer():
  """Default optimizer for DNN models."""
  return train.AdagradOptimizer(_DEFAULT_LEARNING_RATE)

def _get_feature_dict(features):
  if isinstance(features, dict):
    return features
  return {"": features}

def _dnn_sampled_softmax_classifier_model_fn(features, targets, mode, params):
  """model_fn that uses candidate sampling.

  Args:
    features: Single Tensor or dict of Tensor (depends on data passed to `fit`)
    targets: A single Tensor of shape [batch_size, n_labels] containing
      the target indices.
    mode: Represents if this training, evaluation or prediction. See `ModeKeys`.
    params: A dict of hyperparameters that are listed below.
      hidden_units- List of hidden units per layer. All layers are fully
        connected. Ex. `[64, 32]` means first layer has 64 nodes and second one
        has 32.
      feature_columns- An iterable containing all the feature columns used by
        the model. All items in the set should be instances of classes derived
        from `FeatureColumn`.
      n_classes- number of target classes. It must be greater than 2.
      n_samples- number of sample target classes. Needs to be tuned - A good
        starting point could be 2% of n_classes.
      n_labels- number of labels in each example.
      top_k- The number of classes to predict.
      optimizer- An instance of `tf.Optimizer` used to train the model. If
        `None`, will use an Adagrad optimizer.
      dropout- When not `None`, the probability we will drop out a given
        coordinate.
      gradient_clip_norm- A float > 0. If provided, gradients are
        clipped to their global norm with this clipping ratio. See
        tf.clip_by_global_norm for more details.
      num_ps_replicas- The number of parameter server replicas.

  Returns:
    predictions: A single Tensor or a dict of Tensors.
    loss: A scalar containing the loss of the step.
    train_op: The op for training.
  """

  hidden_units = params["hidden_units"]
  feature_columns = params["feature_columns"]
  n_classes = params["n_classes"]
  n_samples = params["n_samples"]
  n_labels = params["n_labels"]
  top_k = params["top_k"]
  optimizer = params["optimizer"]
  dropout = params["dropout"]
  gradient_clip_norm = params["gradient_clip_norm"]
  num_ps_replicas = params["num_ps_replicas"]

  parent_scope = "dnn_ss"

  features = _get_feature_dict(features)
  targets = _reshape_targets(targets)

  # Setup the input layer partitioner.
  input_layer_partitioner = (
      partitioned_variables.min_max_variable_partitioner(
          max_partitions=num_ps_replicas,
          min_slice_size=64 << 20))

  # Create the input layer.
  with variable_scope.variable_scope(
      parent_scope + "/input_from_feature_columns",
      features.values(),
      partitioner=input_layer_partitioner) as scope:
    net = layers.input_from_feature_columns(
        features,
        feature_columns,
        weight_collections=[parent_scope],
        scope=scope)

  # Setup the hidden layer partitioner.
  hidden_layer_partitioner = (
      partitioned_variables.min_max_variable_partitioner(
          max_partitions=num_ps_replicas))

  final_hidden_layer_dim = None
  # Create hidden layers using fully_connected.
  for layer_id, num_hidden_units in enumerate(hidden_units):
    with variable_scope.variable_scope(
        parent_scope + "/hiddenlayer_%d" % layer_id, [net],
        partitioner=hidden_layer_partitioner) as scope:
      net = layers.fully_connected(net,
                                   num_hidden_units,
                                   variables_collections=[parent_scope],
                                   scope=scope)
      final_hidden_layer_dim = num_hidden_units
      # Add dropout if it is enabled.
      if dropout is not None and mode == estimator.ModeKeys.TRAIN:
        net = layers.dropout(net, keep_prob=(1.0 - dropout))

  # Create the weights and biases for the logit layer.
  with variable_scope.variable_scope(
      parent_scope + "/logits", [net],
      partitioner=hidden_layer_partitioner) as scope:
    dtype = net.dtype.base_dtype
    weights_shape = [n_classes, final_hidden_layer_dim]
    weights = variables.model_variable(
        "weights",
        shape=weights_shape,
        dtype=dtype,
        initializer=initializers.xavier_initializer(),
        trainable=True,
        collections=[parent_scope])
    biases = variables.model_variable(
        "biases",
        shape=[n_classes,],
        dtype=dtype,
        initializer=init_ops.zeros_initializer,
        trainable=True,
        collections=[parent_scope])

  if mode == estimator.ModeKeys.TRAIN:
    # Call the candidate sampling APIs and calculate the loss.
    sampled_values = nn.learned_unigram_candidate_sampler(
        true_classes=math_ops.to_int64(targets),
        num_true=n_labels,
        num_sampled=n_samples,
        unique=True,
        range_max=n_classes)

    sampled_softmax_loss = nn.sampled_softmax_loss(
        weights=weights,
        biases=biases,
        inputs=net,
        labels=math_ops.to_int64(targets),
        num_sampled=n_samples,
        num_classes=n_classes,
        num_true=n_labels,
        sampled_values=sampled_values)

    loss = math_ops.reduce_mean(sampled_softmax_loss, name="loss")

    train_op = optimizers.optimize_loss(
        loss=loss, global_step=contrib_framework.get_global_step(),
        learning_rate=_DEFAULT_LEARNING_RATE,
        optimizer=_get_optimizer(optimizer), clip_gradients=gradient_clip_norm,
        name=parent_scope)
    return None, loss, train_op

  elif mode == estimator.ModeKeys.EVAL:
    logits = nn.bias_add(standard_ops.matmul(net, array_ops.transpose(weights)),
                         biases)
    predictions = {}
    predictions[_PROBABILITIES] = nn.softmax(logits)
    predictions[_CLASSES] = math_ops.argmax(logits, 1)
    _, predictions[_TOP_K] = nn.top_k(logits, top_k)

    # Since the targets have multiple labels, setup the target probabilities
    # as 1.0/n_labels for each of the labels.
    target_one_hot = array_ops.one_hot(
        indices=targets, depth=n_classes, on_value=1.0 / n_labels)
    target_one_hot = math_ops.reduce_sum(
        input_tensor=target_one_hot,
        reduction_indices=[1])

    loss = math_ops.reduce_mean(
        nn.softmax_cross_entropy_with_logits(logits, target_one_hot))

    return predictions, loss, None

  elif mode == estimator.ModeKeys.INFER:
    logits = nn.bias_add(standard_ops.matmul(net, array_ops.transpose(weights)),
                         biases)
    predictions = {}
    predictions[_PROBABILITIES] = nn.softmax(logits)
    predictions[_CLASSES] = math_ops.argmax(logits, 1)
    _, predictions[_TOP_K] = nn.top_k(logits, top_k)

    return predictions, None, None

def _reshape_targets(targets):
  if targets is None:
    return None
  check_shape_op = control_flow_ops.Assert(
      math_ops.less_equal(array_ops.rank(targets), 2),
      ["target's should be either [batch_size, n_labels] or [batch_size]"])
  with ops.control_dependencies([check_shape_op]):
    targets = array_ops.reshape(
        targets, shape=[array_ops.shape(targets)[0], -1])
  return targets

def _top_k_fn_wrapper(metric_fn, k):

  def wrap_func(predictions, labels):
    return metric_fn(predictions, _reshape_targets(labels), k=k)

  wrap_func.__name__ = metric_fn.__name__
  return wrap_func

class _DNNSampledSoftmaxClassifier(trainable.Trainable, evaluable.Evaluable):
  """A classifier for TensorFlow DNN models.

  Example:

  ```python
  legos = sparse_column_with_hash_bucket(column_name="legos",
                                         hash_bucket_size=1000)
  watched_videos = sparse_column_with_hash_bucket(
                     column_name="watched_videos",
                     hash_bucket_size=20000)

  legos_emb = embedding_column(sparse_id_column=legos, dimension=16,
                               combiner="sum")
  watched_videos_emb = embedding_column(sparse_id_column=watched_videos,
                                        dimension=256,
                                        combiner="sum")

  estimator = DNNSampledSoftmaxClassifier(
      n_classes=500000, n_samples=10000, n_labels=5,
      feature_columns=[legos_emb, watched_videos_emb],
      hidden_units=[1024, 512, 256])

  # Or estimator using the Adam optimizer with dropout.
  estimator = DNNSampledSoftmaxClassifier(
      feature_columns=[education_emb, occupation_emb],
      hidden_units=[1024, 512, 256],
      optimizer=tf.train.ProximalAdagradOptimizer(
        learning_rate=0.1),
      dropout=0.1)

  # Input builders
  def input_fn_train: # returns x, Y
    pass
  estimator.fit(input_fn=input_fn_train)

  def input_fn_eval: # returns x, Y
    pass
  estimator.evaluate(input_fn=input_fn_eval)
  estimator.predict(x=x)
  ```

  Input of `fit` and `evaluate` should have following features,
    otherwise there will be a `KeyError`:

  * for each `column` in `feature_columns`:
    - if `column` is a `SparseColumn`, a feature with `key=column.name`
      whose `value` is a `SparseTensor`.
    - if `column` is a `EmbeddingColumn`, a feature with `key=column.name`
      whose `value` is a `SparseTensor`.
    - if `column` is a `WeightedSparseColumn`, two features: the first with
      `key` the id column name, the second with `key` the weight column name.
      Both features' `value` must be a `SparseTensor`.
    - if `column` is a `RealValuedColumn`, a feature with `key=column.name`
      whose `value` is a `Tensor`.
  """

  def __init__(self,
               hidden_units,
               feature_columns,
               n_classes,
               n_samples,
               n_labels=1,
               top_k=1,
               model_dir=None,
               optimizer=None,
               dropout=None,
               gradient_clip_norm=None,
               config=None,
               feature_engineering_fn=None):
    """Initializes a DNNSampledSoftmaxClassifier instance.

    Args:
      hidden_units: List of hidden units per layer. All layers are fully
        connected. Ex. `[64, 32]` means first layer has 64 nodes and second one
        has 32.
      feature_columns: An iterable containing all the feature columns used by
        the model. All items in the set should be instances of classes derived
        from `FeatureColumn`.
      n_classes: number of target classes. It must be greater than 2.
      n_samples: number of sample target classes. Needs to be tuned - A good
        starting point could be 2% of n_classes.
      n_labels: number of labels in each example.
      top_k: The number of classes to predict.
      model_dir: Directory to save model parameters, graph and etc. This can
        also be used to load checkpoints from the directory into a estimator to
        continue training a previously saved model.
      optimizer: An instance of `tf.Optimizer` used to train the model. If
        `None`, will use an Adagrad optimizer.
      dropout: When not `None`, the probability we will drop out a given
        coordinate.
      gradient_clip_norm: A float > 0. If provided, gradients are
        clipped to their global norm with this clipping ratio. See
        tf.clip_by_global_norm for more details.
      config: `RunConfig` object to configure the runtime settings.
      feature_engineering_fn: Feature engineering function. Takes features and
                        targets which are the output of `input_fn` and
                        returns features and targets which will be fed
                        into the model.

    Returns:
      A `DNNSampledSoftmaxClassifier` estimator.

    Raises:
      ValueError: If n_classes <= 2.
      ValueError: If n_classes < n_samples.
      ValueError: If n_classes < n_labels.
    """
    # Validate all the inputs.
    if n_classes <= 2:
      raise ValueError("n_classes should be greater than 2. For n_classes <= 2,"
                       " use DNNClassifier.")
    if n_classes < n_samples:
      raise ValueError("n_classes (%d) should be greater than n_samples (%d)." %
                       (n_classes, n_samples))
    if n_classes < n_labels:
      raise ValueError("n_classes (%d) should be greater than n_labels"
                       " (%d)." % (n_classes, n_labels))

    self._top_k = top_k
    self._feature_columns = feature_columns
    assert self._feature_columns
    self._model_dir = model_dir or tempfile.mkdtemp()

    # Build the estimator with _dnn_sampled_softmax_classifier_model_fn.
    self._estimator = estimator.Estimator(
        model_fn=_dnn_sampled_softmax_classifier_model_fn,
        model_dir=self._model_dir,
        config=config,
        params={
            "hidden_units": hidden_units,
            "feature_columns": feature_columns,
            "n_classes": n_classes,
            "n_samples": n_samples,
            "n_labels": n_labels,
            "top_k": top_k,
            "optimizer": optimizer or _get_default_optimizer(),
            "dropout": dropout,
            "gradient_clip_norm": gradient_clip_norm,
            "num_ps_replicas": config.num_ps_replicas if config else 0
        },
        feature_engineering_fn=feature_engineering_fn)

  def get_estimator(self):
    return self._estimator

  def fit(self, x=None, y=None, input_fn=None, steps=None, batch_size=None,
          monitors=None, max_steps=None):
    """See trainable.Trainable."""
    return self._estimator.fit(x=x, y=y, input_fn=input_fn, steps=steps,
                               batch_size=batch_size, monitors=monitors,
                               max_steps=max_steps)

  def evaluate(self, x=None, y=None, input_fn=None, feed_fn=None,
               batch_size=None, steps=None, metrics=None, name=None,
               range_k=None):
    # pylint: disable=g-doc-args,g-doc-return-or-yield
    """See evaluable.Evaluable for a description of the Args.

    Calculates the following metrics by default:
      loss
      average_precision@top_k: see
        https://en.wikipedia.org/wiki/Information_retrieval#Average_precision
      for k in range_k:
        precision@k and recall@k

    range_k: A list of numbers where precision and recall have to be obtained.
      For eg. range_k=[1,5] will calculate precision@1, precision@5,
      recall@1 and recall@5. If None, defaults to [1, top_k].
    """
    if not metrics:
      metrics = {}
    metrics.update({
        "average_precision_at_%d" % self._top_k: metric_spec.MetricSpec(
            metric_fn=_top_k_fn_wrapper(
                metric_ops.streaming_sparse_average_precision_at_k,
                k=self._top_k),
            prediction_key=_PROBABILITIES)
    })
    if range_k is None:
      if self._top_k > 1:
        range_k = [1, self._top_k]
      else:
        range_k = [1]
    for k in range_k:
      metrics.update({
          "precision_at_%d" % k: metric_spec.MetricSpec(
              metric_fn=_top_k_fn_wrapper(
                  metric_ops.streaming_sparse_precision_at_k, k=k),
              prediction_key=_PROBABILITIES,)
      })
      metrics.update({
          "recall_at_%d" % k: metric_spec.MetricSpec(
              metric_fn=_top_k_fn_wrapper(
                  metric_ops.streaming_sparse_recall_at_k, k=k),
              prediction_key=_PROBABILITIES,)
      })

    return self._estimator.evaluate(x=x, y=y, input_fn=input_fn,
                                    feed_fn=feed_fn, batch_size=batch_size,
                                    steps=steps, metrics=metrics, name=name)

  def predict(self, x=None, input_fn=None, batch_size=None, as_iterable=False,
              get_top_k=False):
    """Returns predicted classes for given features.

    Args:
      x: features.
      input_fn: Input function. If set, x must be None.
      batch_size: Override default batch size.
      as_iterable: If True, return an iterable which keeps yielding predictions
        for each example until inputs are exhausted. Note: The inputs must
        terminate if you want the iterable to terminate (e.g. be sure to pass
        num_epochs=1 if you are using something like read_batch_features).
      get_top_k : if set to true returns the top k classes otherwise returns
        the top class.

    Returns:
      Numpy array of predicted classes (or an iterable of predicted classes if
      as_iterable is True).
    """
    if get_top_k:
      key = _TOP_K
    else:
      key = _CLASSES
    preds = self._estimator.predict(x=x, input_fn=input_fn,
                                    batch_size=batch_size, outputs=[key],
                                    as_iterable=as_iterable)
    if as_iterable:
      return _as_iterable(preds, output=key)
    return preds[key]

  def predict_proba(self, x=None, input_fn=None, batch_size=None,
                    as_iterable=False):
    """Returns prediction probabilities for given features.

    Args:
      x: features.
      input_fn: Input function. If set, x and y must be None.
      batch_size: Override default batch size.
      as_iterable: If True, return an iterable which keeps yielding predictions
        for each example until inputs are exhausted. Note: The inputs must
        terminate if you want the iterable to terminate (e.g. be sure to pass
        num_epochs=1 if you are using something like read_batch_features).

    Returns:
      Numpy array of predicted probabilities (or an iterable of predicted
      probabilities if as_iterable is True).
    """
    preds = self._estimator.predict(x=x, input_fn=input_fn,
                                    batch_size=batch_size,
                                    outputs=[_PROBABILITIES],
                                    as_iterable=as_iterable)
    if as_iterable:
      return _as_iterable(preds, output=_PROBABILITIES)
    return preds[_PROBABILITIES]

  def export(self, export_dir, signature_fn=None,
             input_fn=None, default_batch_size=1,
             exports_to_keep=None):
    """Exports inference graph into given dir.

    Args:
      export_dir: A string containing a directory to write the exported graph
        and checkpoints.
      signature_fn: Function that returns a default signature and a named
        signature map, given `Tensor` of `Example` strings, `dict` of `Tensor`s
        for features and `Tensor` or `dict` of `Tensor`s for predictions.
      input_fn: If `use_deprecated_input_fn` is true, then a function that given
        `Tensor` of `Example` strings, parses it into features that are then
        passed to the model. Otherwise, a function that takes no argument and
        returns a tuple of (features, targets), where features is a dict of
        string key to `Tensor` and targets is a `Tensor` that's currently not
        used (and so can be `None`).
      default_batch_size: Default batch size of the `Example` placeholder.
      exports_to_keep: Number of exports to keep.

    Returns:
      The string path to the exported directory. NB: this functionality was
      added ca. 2016/09/25; clients that depend on the return value may need
      to handle the case where this function returns None because subclasses
      are not returning a value.
    """
    def default_input_fn(unused_estimator, examples):
      return layers.parse_feature_columns_from_examples(
          examples, self._feature_columns)
    return self._estimator.export(export_dir=export_dir,
                                  signature_fn=signature_fn,
                                  input_fn=input_fn or default_input_fn,
                                  default_batch_size=default_batch_size,
                                  exports_to_keep=exports_to_keep)

  def get_variable_names(self):
    return self._estimator.get_variable_names()

  @property
  def model_dir(self):
    return self._model_dir


from flask import Flask, jsonify, request

app = Flask(__name__)

# Sample initial data
books = [
    {"id": 1, "title": "River and The Source", "author": "Margaret Ogola"},
    {"id": 2, "title": "DevOps Handbook", "author": "Gene Kim"},
    {"id": 3, "title": "Phoenix Project", "author": "Micheal Hatterman"}
]

next_id = 4

@app.route('/books', methods=['GET'])
def get_books():
    return jsonify(books)

@app.route('/books', methods=['POST'])
def add_book():
    global next_id
    book = {
        "id": next_id,
        "title": request.json.get('title'),
        "author": request.json.get('author')
    }
    books.append(book)
    next_id += 1
    return jsonify(book), 201

@app.route('/books/<int:book_id>', methods=['DELETE'])
def delete_book(book_id):
    global books
    books = [book for book in books if book['id'] != book_id]
    return '', 204

if __name__ == '__main__':
    app.run()


#프레임을 통한 캐릭터의 속도 조절
import pygame

pygame.init() # 초기화 (반드시 필요)

#화면 크기 설정
screen_width = 480 # 가로 크기
screen_height = 640 # 세로 크기
screen = pygame.display.set_mode((screen_width,screen_height)) # 화면

#화면 타이틀(제목) 설정
pygame.display.set_caption("Nado Game") #게임 이름

# FPS
clock = pygame.time.Clock()
FPS = 60

# 배경 이미지 불러오기
background = pygame.image.load("C:\\Users\\donha\\OneDrive\\바탕 화면\\python\\PythonGame\\NaDo_PythonGameMake\\배경.png")

# 캐릭터(스프라이트) 불러오기
character = pygame.image.load("C:\\Users\\donha\\OneDrive\\바탕 화면\\python\\PythonGame\\NaDo_PythonGameMake\\캐릭터.png")
character_size = character.get_rect().size # 캐릭터의 크기 구함
character_width = character_size[0] #캐릭터의 가로크기
character_height = character_size[1] #캐릭터의 세로크기
character_x_pos = (screen_width / 2) - (character_width / 2) # 화면 가로의 절반 크기에 해당하는 곳에 위치
character_y_pos = screen_height - character_height # 화면 세로의 가장 아래에 위치

# 이동할 좌표
to_x = 0
to_y = 0

# 캐릭터 이동 속도
# ex 캐릭터가 100 만큼 이동을 해야함
# 10 fps : 1초 동안에 10번 동작 -> 1번에 10만큼 이동
# 20 fps : 1초 동안에 20번 동작 -> 1번에 5만큼 이동

character_speed = 0.6

#이벤트 루프
running = True # 게임이 진행중인가 확인하는 변수
while running:
    dt = clock.tick(FPS) # 게임화면의 초당 프레임 수

    print("fps: " + str(clock.get_fps()))
    for event in pygame.event.get(): # 파이게임을 위해 무조건 필요한 코드, 사용자의 동작을 체크하는 코드
        if event.type == pygame.QUIT: # 창의 x버튼을 눌렀을 때 이벤트 발생
            running = False # 게임이 진행중이 아니다라는 값으로 바꿈

        if event.type == pygame.KEYDOWN: #키가 눌러졌는지 확인
            if event.key == pygame.K_LEFT: # 캐릭터를 왼쪽으로
                to_x -= character_speed
            elif event.key == pygame.K_RIGHT: # 캐릭터를 오른쪽으로
                to_x += character_speed
            elif event.key == pygame.K_UP: # 캐릭터를 위로
                to_y -= character_speed
            elif event.key == pygame.K_DOWN: # 캐릭터를 아래로
                to_y += character_speed

        if event.type == pygame.KEYUP: # 키가 때진걸 확인
            if event.key == pygame.K_LEFT or event.key == pygame.K_RIGHT: # 오른쪽과 왼쪽을 땔때
                to_x = 0 # 움직이지 않음
            elif event.key == pygame.K_DOWN or event.key == pygame.K_UP: # 위와 아래키를 땔때
                to_y = 0 # 움직이지 않음

    character_x_pos += to_x * dt
    character_y_pos += to_y * dt

    if character_x_pos < 0:
        character_x_pos = 0
    elif character_x_pos > (screen_width - character_width):
        character_x_pos = screen_width - character_width

    if character_y_pos < 0:
        character_y_pos = 0
    elif character_y_pos > (screen_height - character_height):
        character_y_pos = screen_height - character_height

    screen.blit(background, (0, 0)) #배경 그리기

    screen.blit(character, (character_x_pos, character_y_pos))

    pygame.display.update() # 게임화면을 다시 그리기, 이게 있어야 배경화면이 그려짐

# pygame 종료
pygame.quit()


from django.conf import settings as dj_settings
from django.core.validators import RegexValidator

from rest_framework import serializers
from rest_framework.fields import empty

import pydoc
from copy import deepcopy

TIME_ONLY_FORMAT = "HH:mm:ss"
DATE_ONLY_FORMAT = "YYYY-MM-DD"
DATETIME_FORMAT = "YYYY-MM-DD HH:mm:ss"
UUID_REGEX = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"

FIELD_AND_VUE_FORM_GENERATOR_MAP = {

    # boolean
    serializers.BooleanField:     {"type": "switch"},
    serializers.NullBooleanField: {"type": "switch"},

    # string
    serializers.CharField:  {"type": "input"},
    serializers.EmailField: {"type": "input", "validator": ["email"]},
    serializers.RegexField: {"type": "input", "validator": ["regexp"]},
    serializers.SlugField:  {"type": "input", "validator": ["regexp"]},
    serializers.URLField:   {"type": "input", "validator": ["url"]},
    serializers.UUIDField:  {
        "type": "input", "validator": ["regexp"], "pattern": UUID_REGEX
    },

    serializers.FilePathField:  {"type": "input"},
    serializers.IPAddressField: {"type": "input", "validator": ["string"]},

    # number
    serializers.IntegerField: {
        "type": "input", "inputType": "number", "validator": ["number"]
    },
    serializers.FloatField:   {"type": "input", "validator": ["number"]},
    serializers.DecimalField: {"type": "input", "validator": ["number"]},

    # date
    serializers.DateTimeField: {
        "type": "dateTimePicker",
        "validator": ["date"],
        "dateTimePickerOptions": {"format": DATETIME_FORMAT},
    },
    serializers.DateField: {
        "type": "dateTimePicker",
        "validator": ["date"],
        "dateTimePickerOptions": {"format": DATE_ONLY_FORMAT},
    },
    serializers.TimeField: {
        "type": "dateTimePicker",
        "validator": ["date"],
        "format": TIME_ONLY_FORMAT,
        "dateTimePickerOptions": {"format": TIME_ONLY_FORMAT},
    },
    serializers.DurationField: {"type": "input"},

    # Choices
    serializers.ChoiceField: {
        "type": "vueMultiSelect",
        "selectOptions": {"key": "name", "label": "name",}
    },

    serializers.MultipleChoiceField: {
        "type": "vueMultiSelect",
        "selectOptions": {
            "multiple": True,
            "trackBy": "name",
            "key": "name",
            "label": "name",
            "hideSelected": True,
        }
    },

    # Hidden Field
    serializers.HiddenField: {"type": "input", "inputType": "hidden",},
}

external_field_and_vue_form_generator_map = getattr(
    dj_settings, "VUE_FORM_GENERATOR_SETTINGS", {}
).get(
    "EXTERNAL_FIELD_AND_VUE_FORM_GENERATOR_MAP", {}
)

for class_path, value in external_field_and_vue_form_generator_map.items():
    FIELD_AND_VUE_FORM_GENERATOR_MAP.update(
        {pydoc.locate(class_path): value}
    )

DEFAULT_FIELD_AND_VUE_FORM_GENERATOR_MAP = {"type": "input"}

class VueFormGeneratorEncoder:

    def __init__(self, serializer):
        self._serializer = serializer

    def _prepare_validatorand_insert_required(self, field, data):
        if not "validator" in data:
            data["validator"] = list()

        data["validator"] = list()
        # if field.required:
        #     data["validator"].append("required")

    def _set_input_max_length(self, field, data):
        if data["type"] == "input":
            max_length = getattr(field, "max_length", None)

            if max_length:
                data["maxlength"] = max_length

    def _set_number_min_and_max(self, field, data):
        if data.get("inputType", None) == "number":
            data["max"] = field.max_value
            data["min"] = field.min_value

    def _set_regex_expression_if_using_regex_validator(self, field, data):
        if "regexp" not in data["validator"]:
            return

        pattern = None
        for validator in field.validators:
            if isinstance(validator, RegexValidator):
                pattern = validator.regex.pattern
                break

        if pattern:
            pattern = pattern.encode('unicode-escape').decode("utf-8")
            data["pattern"] = pattern

    def _set_select_values(self, field, data):

        if data["type"] in ("radios", "checklist",):
            data["styleClasses"] = ["vfg-radio-class"]
            data["values"] = [
                {"name": label, "value": value}
                    for value, label in field.choices.items()
            ]

        elif data["type"] in ("vueMultiSelect",):
            data["values"] = []
            for value, label in field.choices.items():
                if not value:
                    data["selectOptions"]["noneSelectedText"] = label
                else:
                    data["values"].append({"name": label, "id": value})

            # set ID to select input box
            data["selectOptions"]["id"] = field.field_name

    def _set_default(self, field, data):
        if not field.initial:
            return

        def _default_value(value):
            return {"name": field.choices.get(value, ""), "id": value}

        if data["type"] in ("vueMultiSelect",):
            if data.get("selectOptions", {}).get("multiple", False):
                data["default"] = list()
                for value in field.initial:
                    data["default"].append(_default_value(value))
            else:
                data["default"] = _default_value(field.initial)

        elif data["type"] in ("dateTimePicker",):
            data["default"] = field.initial.isoformat()

        else:
            data["default"] = field.initial

    def set_field_label_and_hint(self, field, data):
        label = ""
        hint = ""

        if not isinstance(field, serializers.HiddenField):
            label = field.label
            hint = field.help_text

        data["label"] = label
        data["hint"] = hint

    def _build_schema(self, field):
        data = deepcopy(
            FIELD_AND_VUE_FORM_GENERATOR_MAP.get(
                field.__class__, DEFAULT_FIELD_AND_VUE_FORM_GENERATOR_MAP)
        )

        self.set_field_label_and_hint(field, data)
        self._prepare_validatorand_insert_required(field, data)
        self._set_input_max_length(field, data)
        self._set_number_min_and_max(field, data)
        self._set_regex_expression_if_using_regex_validator(field, data)
        self._set_select_values(field, data)
        self._set_default(field, data)

        return data

    def get_vue_form_generator_schema(self):
        result = {"fields": []}
        for field_name, field in self._serializer.fields.items():
            field_data = {
                "id": field_name,
                "model": field_name,
                "required": field.required,
                "readonly": field.read_only,
            }
            field_data.update(self._build_schema(field))
            result["fields"].append(field_data)

        return result


# Hello-world
#第一个库
#西建大图书馆预约脚本
import requests
import json
import time
import datetime
import os
import sys
url = "http://seat.lib.xauat.edu.cn/reservations/futureReserve"   # 预约明天座位POST的网址（当日预约POST的网址不是这个）
urlone = input('''Hi！我是追风蟹，生活愉快
我可以定时帮您预约我们学校图书馆的座位哦
请在这里复制粘贴您的座位预约网址：''')
#定义基本变量
a = input('''请输入您要预约的日期
例如 02-07  ： ''')
b = input('''
雁塔三楼围廊E区：1
雁塔一楼大厅A区：2
雁塔二楼文库B区：3
雁塔三楼上网D区：4
雁塔三楼自修F区：5
雁塔四楼自习G区：6
雁塔四楼上网H区：8
请输入您要预约区域的代号：'''
              )

hour = input('请输入开始预约时间的小时数：')
minute = input('请输入开始预约时间的分钟数：')

timing = input('请问是否需要预约结束后自动关机（关机扣1敲回车，不关机扣0敲回车）：')

print('好的，已经开始工作，时间到了我会告诉您预约结果，请别关闭电脑、程序或者断网...')
time.sleep(2)
print('请等待...')
try:
    while (True):
        now = datetime.datetime.now()  # 获取当前系统时间
        if (now.hour == eval(hour) and now.minute == eval(minute)):
            break
    time.sleep(0.5)
    v = time.perf_counter()     #开始计时
#获取cookie
    m = requests.session()
    m = m.get(urlone)
    cookies = requests.utils.dict_from_cookiejar(m.cookies)     #将cookie的cookiejar格式转换为dict格式
#提取不同键cookie的值
    g = cookies['XSRF-TOKEN']
    h = cookies['laravel_session']
    p = g.strip('%3D')                 #本cookie的URL编码最后部分可能会有=号，在获取时会被编码成%3D，所以要消除掉
    l = h.strip('%3D')
#修改请求头
    kv = {'Host': 'seat.lib.xauat.edu.cn',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:63.0) Gecko/20100101 Firefox/63.0',
        'Accept': 'application/json, text/plain, */*',
        'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
        'Accept-Encoding': 'gzip, deflate',
        'Referer': 'http://seat.lib.xauat.edu.cn/',
        'Content-Type': 'application/json;charset=utf-8',
        'X-XSRF-TOKEN': p,
        'Content-Length': '43',
        'Connection': 'keep-alive',
        'Cookie':'XSRF-TOKEN='+p+'; laravel_session='+l
          }

    for x in range(21):                                #高峰期可能会请求失败，所以循环提交21次请求
#预约上午座位
        kt = {'date': '2019-' + a,
              'period': 0,
              'roomId': eval(b)
              }
        kt1 = json.dumps(kt)
        requests.post(url, headers=kv, data=kt1)
#预约下午座位
        kt = {'date': '2019-' + a,
              'period': 1,
              'roomId': eval(b)
              }
        kt1 = json.dumps(kt)
        requests.post(url, headers=kv, data=kt1)

        if (x == 0):
            print('恭喜啦！成功预约到全天座位')
            o = time.perf_counter() - v
            print('预约用时：{:.3f}秒'.format(o))
        else:
            print('保险重复预约{0}次'.format(x))
    time.sleep(30)
    if (eval(timing) == 1):          #如果开始输入的关机指令为1，计算机到时间自动关闭
        os.system('shutdown -s -f -t 1')
    else:
        pass
except:
    print("爬取失败")
    time.sleep(30)
    if (eval(timing) == 1):
        os.system('shutdown -s -f -t 1')
    else:
        pass


class ModelToDict(object):
    """
    Serialize a model sql alchemy object into a dict.
    """
    def to_dict(self, exclude=None):
        """
        Args:
            exclude (list[str]): list of attributes to exclude in the dict.

        Returns:
            dict: serialized sql alchemy object into a dict.
        """
        seralized_model = {}
        exclude = set(exclude) if exclude else set()

        for attr in self.__mapper__.c.keys():
            if attr not in exclude:
                seralized_model[attr] = getattr(self, attr)
        return seralized_model


import PyQt5.QtCore as qtc
import PyQt5.QtGui as qtg
import PyQt5.QtWidgets as qtw

from application_gui.common_gui_functions import CLabel, CHorizontalSeparator, CLabelledLineEdit
from application_gui.settings_scale.functions import SetScaleFunctions

##-\-\-\-\-\-\-\-\-\-\-\-\-\-\
## WINDOW FOR READING METADATA
##-/-/-/-/-/-/-/-/-/-/-/-/-/-/

class SetScaleWindow(qtw.QMainWindow, SetScaleFunctions):
    def __init__(self, parent, image_class=None):
        super(SetScaleWindow, self).__init__(parent)

        # Initialise the subwindow
        self.parent = parent
        self.image_class = image_class
        #self.setWindowModality(qtc.Qt.ApplicationModal)

        # Generate the window
        self.mainWidget = qtw.QWidget()
        self.mainLayout = qtw.QVBoxLayout(self.mainWidget)
        self.setWindowTitle("Set Scale")

        # Populate the panel
        self.createScaleSettings(self.mainLayout)
        #self.mainLayout.addWidget( CHorizontalSeparator() )
        self.createUserActions(self.mainLayout)

        # Display the panel
        self.mainWidget.setLayout(self.mainLayout)
        self.setCentralWidget(self.mainWidget)
        self.show()
        self.setFixedSize(self.size())

    # ---------------------------------------------------
    # Reinitialise the display when the window is closed
    def closeEvent(self, event=None):
        event.accept()
        self.parent.subWindows['set_scale'] = None

    ##-\-\-\-\-\-\-\-\-\-\
    ## GENERATE THE DISPLAY
    ##-/-/-/-/-/-/-/-/-/-/

    # ------------------------------------------
    # Generate the display for the spatial scale
    def createScaleSettings(self, parentWidget):

        # Generate the widget
        self.scaleSettingsWidget = qtw.QWidget()
        self.scaleSettingsLayout = qtw.QGridLayout(self.scaleSettingsWidget)

        # Label for space scale
        current_row = 0
        self.scaleSettingsLayout.addWidget( CLabel("Space Scale:"), current_row, 0, 1, 2 )

        # Entry for the distance in pixel
        current_row += 1
        self.scaleSettingsLayout.addWidget( qtw.QLabel("Distance in pixels"), current_row, 0 )
        self.pixelDistanceEntry = qtw.QLineEdit()
        self.pixelDistanceEntry.setText( str(self.image_class.scale.space_scale) )
        self.scaleSettingsLayout.addWidget( self.pixelDistanceEntry, current_row, 1 )

        # Entry for the real distance
        current_row += 1
        self.scaleSettingsLayout.addWidget( qtw.QLabel("Known distance"), current_row, 0 )
        self.knownDistanceEntry = qtw.QLineEdit()
        self.knownDistanceEntry.setText( "1" )
        self.scaleSettingsLayout.addWidget( self.knownDistanceEntry, current_row, 1 )

        # Entry for the length unit
        current_row += 1
        self.scaleSettingsLayout.addWidget( qtw.QLabel("Unit of length"), current_row, 0 )
        self.lengthUnitEntry = qtw.QLineEdit()
        self.lengthUnitEntry.setText( self.image_class.scale.space_unit )
        self.scaleSettingsLayout.addWidget( self.lengthUnitEntry, current_row, 1 )

        current_row += 1
        self.scaleSettingsLayout.addWidget( CHorizontalSeparator(), current_row, 0, 1, 2 )

        # Label for time scale
        current_row += 1
        self.scaleSettingsLayout.addWidget( CLabel("Time Scale:"), current_row, 0, 1, 2 )

        # Entry for the length unit
        current_row += 1
        self.scaleSettingsLayout.addWidget( qtw.QLabel("Frame rate"), current_row, 0 )
        self.frameRateEntry = qtw.QLineEdit()
        self.frameRateEntry.setText( str(self.image_class.scale.frame_rate) )
        self.scaleSettingsLayout.addWidget( self.frameRateEntry, current_row, 1 )

        # Display the widget
        self.scaleSettingsWidget.setLayout(self.scaleSettingsLayout)
        parentWidget.addWidget(self.scaleSettingsWidget)

    # ----------------------------------
    # Generate the controls for the user
    def createUserActions(self, parentWidget):

        # Generate the widget
        self.userActionWidget = qtw.QWidget()
        self.userActionLayout = qtw.QVBoxLayout(self.userActionWidget)

        # Add the global checkbox
        self.globalScaleCheckBox = qtw.QCheckBox("Global")
        self.userActionLayout.addWidget(self.globalScaleCheckBox)

        # Generate the widget
        self.userButtonWidget = qtw.QWidget()
        self.userButtonLayout = qtw.QHBoxLayout(self.userButtonWidget)

        # Add the button to open a new file
        self.applyButton = qtw.QPushButton("Apply")
        self.applyButton.clicked.connect(self.applyScale)
        self.applyButton.setStatusTip("Apply the scale to the stack.")
        self.applyButton.setFixedWidth(125)
        self.userButtonLayout.addWidget(self.applyButton, alignment=qtc.Qt.AlignLeft)

        # Add the button to close
        self.closeButton = qtw.QPushButton("Cancel")
        self.closeButton.clicked.connect(self.close)
        self.closeButton.setStatusTip("Close the current window.")
        self.closeButton.setFixedWidth(125)
        self.userButtonLayout.addWidget(self.closeButton, alignment=qtc.Qt.AlignRight)

        # Display the widget
        self.userButtonWidget.setLayout(self.userButtonLayout)
        self.userActionLayout.addWidget(self.userButtonWidget)

        # Display the widget
        self.userActionWidget.setLayout(self.userActionLayout)
        parentWidget.addWidget(self.userActionWidget)


class Solution:
    def compareVersion(self, version1: str, version2: str) -> int:
        v1, v2 = self.helper(version1), self.helper(version2)

        return 1 if v1 > v2 else (-1 if v1 < v2 else 0)

    def helper(self, v):
        v = map(int, v.split("."))
        i = len(v) - 1

        while i >= 0 and v[i] == 0:
            i -= 1

        return v[:i+1]

Leetcode/165. Compare Version Numbers.py


#!/usr/bin/python3
import math
text = input('Type here: ')
shift = int(input ('Shift? : '))
text_List = list(text)

# # Encrytion method
def Encrypt (t, s) :
        encrypt = list()
        n = shift % 26
        for i in range(0, len(text)):
                char = text[i]
                # if character is uppercase letter
                if(char.isupper()):
                        encrypt += chr((ord(char)-65+n)%26+65)
                # if character is in lower case
                else :
                        encrypt += chr((ord(char)-97+n)%26+97)

        return encrypt

# Decrytion method
def Decrypt (t, s) :
        decrypt = list()
        n = shift % 26
        for i in range(0, len(text)):
                char = text[i]
                # if character is uppercase letter
                if(char.isupper()):
                        decrypt += chr((ord(char)-65-n)%26+65)
                # if character is in lower case
                else :
                        decrypt += chr((ord(char)-97-n)%26+97)
        return decrypt

# Collecting User choice to select Encrytion or Cryption
choose = int(input ( 'Press \n 1 to Encrypt text \n 2 to Decrypt text \n => '))
if (choose == 1) :
        print ('Encrypting : ')
        result = Encrypt(text,shift)

elif (choose == 2) :
        print ('Decrypting : ')
        result = Decrypt(text,shift)

else :
        print ('Unknown Input')
print(result)

# Columnar Transposition

key_length = int(input('Key length: '))

# Columnar Decryption
x= math.ceil(len(text)/key_length)
def columnar (t, K) :
        for i in range(0, len(t), K):
                yield t[i:i+K]
list_using_key = list(columnar(result, x))

print(*(list_using_key), sep = "\n")
Columnar_decrypted_list = list()
for j in range (0, len(list_using_key[0])) :
        for i in list_using_key :
                 Columnar_decrypted_list += i[j]

        j+=1
print(*Columnar_decrypted_list)

l = list(columnar(Columnar_decrypted_list, key_length))
print(*(l), sep='\n')

# print(*list(columnar(result, x)), sep = "\n")


# -*- coding: utf-8 -*-
"""
Created on Fri Dec  4 08:48:26 2020

@author: Adam-22-26
"""

import pafy
import os
os.add_dll_directory(r'C:\Program Files\VideoLAN\VLC')
import threading
# import _thread
from PyQt5 import QtCore
from PyQt5.QtCore import  pyqtSignal
from PyQt5.QtWidgets import  (QWidget,
                             QPushButton, QHBoxLayout,
                            QVBoxLayout,QLabel, QLineEdit, QProgressBar,
                              QMainWindow)

###################################################################

class DownloadWindow(QMainWindow):

    progressChanged = pyqtSignal(int)
    def __init__(self, url,parent=None):
        super(DownloadWindow, self).__init__(parent)

        QMainWindow.__init__(self)
        self.urlD = url
        self.videoD = pafy.new(self.urlD)

        self.audioD = pafy.new(self.urlD)

        self.bestD = self.videoD.streams[0]
        self.bestD = self.videoD.getbest()

        self.audiobest = self.audioD.streams[0]
        self.audiobest = self.audioD.getbestaudio(preftype ="m4a")

        self.widgetD = QWidget(self)
        self.setCentralWidget(self.widgetD)

        self.video_size = QLabel(f" MP4 SIZE: {self.bestD.get_filesize()} Bytes\n MP3 SIZE: {self.audiobest.get_filesize()}\n {self.audiobest.title}")
        self.video_size.setStyleSheet("font: bold 13px")

        self.setWindowTitle("Choose Format")

        self.exitD = QPushButton("Exit")
        self.exitD.setStyleSheet("max-width: 50px;")
        self.exitD.clicked.connect(self.exitd)

        self.mp3 = QPushButton("MP3")
        self.mp3.setStyleSheet("max-width: 50px;")
        self.mp3.clicked.connect(self.mp3F)

        self.mp4 = QPushButton("MP4")
        self.mp4.setStyleSheet("max-width: 50px;")
        self.mp4.clicked.connect(self.mp4F)

        self.layoutFormat = QHBoxLayout()
        self.layoutFormat.addStretch(50)
        self.VdownloadLayout = QVBoxLayout()
        self.layoutFormat.addWidget(self.mp3)
        self.layoutFormat.addWidget(self.mp4)
        self.layoutFormat.addWidget(self.exitD)

        self.pbar = QProgressBar(maximum=100)
        self.pbar.setGeometry(30, 40, 200, 25)
        self.progressChanged.connect(self.pbar.setValue)

        path = QtCore.QStandardPaths.writableLocation(QtCore.QStandardPaths.DownloadLocation)
        self.le_output = QLineEdit(path)

        self.VdownloadLayout.addWidget(self.video_size)
        self.VdownloadLayout.addLayout(self.layoutFormat)

        self.VdownloadLayout.addWidget(self.le_output)

        self.widgetD.setLayout(self.VdownloadLayout)
        self.show()

    def download_info(self, total, recvd, ratio, rate, eta): # pafy callback for pafy.download
        self.setWindowTitle(f"downloading {self.videoD.title}")
        val = int(ratio * 100)
        self.progressChanged.emit(val)
        if val == 100:
            self.close()

            # add condition if it's not receiving or inter connection timeout

    def mp3F(self):
        video_save = self.le_output.text()
        self.audiobest = self.audioD.streams[0]
        self.audiobest = self.audioD.getbestaudio(preftype ="m4a")
        threading.Thread(target=self.audiobest.download, kwargs={'filepath' : video_save, 'callback': self.download_info}, daemon=True).start()
        self.VdownloadLayout.addWidget(self.pbar)

    def mp4F(self):

        video_save = self.le_output.text()
        self.bestD = self.videoD.streams[0]
        self.bestD = self.videoD.getbest(preftype ="mp4")
        threading.Thread(target=self.bestD.download, kwargs={'filepath' : video_save, 'callback': self.download_info}, daemon=True).start()
        self.VdownloadLayout.addWidget(self.pbar)

    def exitd(self):
        self.close()


'''
Created on Jun 20, 2018

@author: Ljubica
'''
from prvi.putanje import autoFajl, dzipFajl, kvadFajl
from prvi.pocetak import Automobil, Dzip, Kvad
from prvi.vrednosti import dzipovi, kvadovi
from prvi import vrednosti

def ucitavanje_automobila():
    automobili=[]
    with open(autoFajl) as f:
        for line in f:
            fields = line.split("|")
            oznaka = fields[0]
            opis = fields[1]
            duzina = float(fields[2])
            sirina = float(fields[3])
            visina = float(fields[4])
            maksimalna_brzina = int(fields[5])
            godina_proizvodnje  = int(fields[6])
            broj_sedista = int(fields[7])
            broj_vrata = int(fields[8])
            tip_menjaca = fields[9]
            automobili.append(Automobil(oznaka,opis,duzina,sirina,visina,maksimalna_brzina,godina_proizvodnje,broj_sedista,broj_vrata,tip_menjaca))
    return automobili

def ucitavanje_dzipova():
    dzipovi=[]
    with open(dzipFajl) as f:
        for line in f:
            fields = line.split("|")
            oznaka = fields[0]
            opis = fields[1]
            duzina = float(fields[2])
            sirina = float(fields[3])
            visina = float(fields[4])
            konjskih_snaga = int(fields[5])
            broj_vrata = int(fields[6])
            spustajuca_zadnja_klupa = fields[7]
            dzipovi.append(Dzip(oznaka,opis,duzina,sirina,visina,konjskih_snaga,broj_vrata,spustajuca_zadnja_klupa))
    return dzipovi

def ucitavanje_kvadova():
    kvadovi=[]
    with open(kvadFajl) as f:
        for line in f:
            fields = line.split("|")
            oznaka = fields[0]
            opis = fields[1]
            duzina = float(fields[2])
            sirina = float(fields[3])
            visina = float(fields[4])
            prostor_za_stvari = float[5]
            kvadovi.append(Kvad(oznaka,opis,duzina,sirina,visina,prostor_za_stvari))
    return kvadovi

def prikaz_vrednosti(lista):
    for i,el in enumerate(lista):
        print("{} {}".format(i, el))
if __name__ == '__main__':
    vrednosti.automobili=ucitavanje_automobila()
    vrednosti.dzipovi=ucitavanje_dzipova()
    vrednosti.kvadovi=ucitavanje_kvadova()

    print("Automobili")
    prikaz_vrednosti(vrednosti.automobili)
    print("Dzipovi")
    prikaz_vrednosti(vrednosti.dzipovi)
    print("Kvadovi")
    prikaz_vrednosti(vrednosti.kvadovi)


import numpy as np
from matplotlib import colors
import matplotlib.pyplot as plt
from matplotlib.ticker import LogLocator
from datetime import datetime, timedelta
import matplotlib.dates as dts
import pandas as pd
import os
import locale
import warnings
import yaml
import re
import sys
from dateutil.parser import parse
from tinydb import TinyDB, Query
from tinydb.operations import add
import time
import json

# Fixed diameter and mobility bins
dp_ion = np.array([7.949610066873187275e-01,9.181737924552214603e-01,
1.060513600503926179e+00,1.224959679823698799e+00,1.414958699738506631e+00,
1.634499249798819331e+00,1.888198514085806856e+00,2.181403433339687226e+00,
2.520308747865528165e+00,2.912095102815642989e+00,3.365090891236600878e+00,
3.888962384293289887e+00,4.494937535166431353e+00,5.196070414640996837e+00,
6.007554438162747701e+00,6.947095098447752193e+00,8.035355151375323857e+00,
9.296489193192451594e+00,1.075878902024538242e+01,1.245546773082500103e+01,
1.442561898219513949e+01,1.671539984850161886e+01,1.937950186998520152e+01,
2.248299804137784363e+01,2.610368545677439300e+01,3.033508982931992648e+01,
3.529036394466827886e+01,4.110740875515996606e+01])

dp_par = np.array([7.498942093324539870e-01,8.659643233600640144e-01,
9.999999999999980016e-01,1.154781984689456031e+00,1.333521432163321974e+00,
1.539926526059490097e+00,1.778279410038920094e+00,2.053525026457140079e+00,
2.371373705661659947e+00,2.738419634264360081e+00,3.162277660168379967e+00,
3.651741272548380213e+00,4.216965034285819591e+00,4.869675251658620141e+00,
5.623413251903479626e+00,6.493816315762099833e+00,7.498942093324560076e+00,
8.659643233600640144e+00,1.000000000000000000e+01,1.154781984689457985e+01,
1.333521432163323972e+01,1.539926526059490008e+01,1.778279410038922137e+01,
2.053525026457139901e+01,2.371373705661660125e+01,2.738419634264360170e+01,
3.162277660168379967e+01,3.651741272548380124e+01,4.216965034285819769e+01])

mob_ion = np.array([3.162277660168379937e-04,2.371373705661659990e-04,
1.778279410038920258e-04,1.333521432163320159e-04,1.000000000000000048e-04,
7.498942093324559917e-05,5.623413251903490022e-05,4.216965034285820205e-05,
3.162277660168380208e-05,2.371373705661660125e-05,1.778279410038919852e-05,
1.333521432163319990e-05,1.000000000000000082e-05,7.498942093324561442e-06,
5.623413251903490361e-06,4.216965034285830030e-06,3.162277660168380038e-06,
2.371373705661659871e-06,1.778279410038920148e-06,1.333521432163330027e-06,
1.000000000000000167e-06,7.498942093324570124e-07,5.623413251903499890e-07,
4.216965034285829924e-07,3.162277660168379721e-07,2.371373705661660136e-07,
1.778279410038920042e-07,1.333521432163329868e-07])*1e4

# Some other values calculated from the fixed bins
mob_ion_geomeans=np.array([2.73841963e-04, 2.05352503e-04, 1.53992653e-04, 1.15478198e-04,
       8.65964323e-05, 6.49381632e-05, 4.86967525e-05, 3.65174127e-05,
       2.73841963e-05, 2.05352503e-05, 1.53992653e-05, 1.15478198e-05,
       8.65964323e-06, 6.49381632e-06, 4.86967525e-06, 3.65174127e-06,
       2.73841963e-06, 2.05352503e-06, 1.53992653e-06, 1.15478198e-06,
       8.65964323e-07, 6.49381632e-07, 4.86967525e-07, 3.65174127e-07,
       2.73841963e-07, 2.05352503e-07, 1.53992653e-07])*1e4

dp_par_geomeans=np.array([ 0.80584219,  0.93057204,  1.07460783,  1.24093776,  1.43301257,
        1.6548171 ,  1.91095297,  2.20673407,  2.54829675,  2.94272718,
        3.39820833,  3.92418976,  4.53158364,  5.23299115,  6.0429639 ,
        6.97830585,  8.05842188,  9.30572041, 10.74607828, 12.40937761,
       14.3301257 , 16.548171  , 19.10952975, 22.06734069, 25.48296748,
       29.42727176, 33.98208329, 39.24189758])

dlogmob_ion=np.array([0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125,
       0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125,
       0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125,
       0.125])

dlogdp_ion=np.array([0.06257907, 0.06258521, 0.06259845, 0.06261376, 0.06263147,
       0.06265194, 0.06267563, 0.06270305, 0.06273478, 0.06277153,
       0.06281409, 0.06286343, 0.06292064, 0.06298703, 0.06306411,
       0.06315368, 0.06325786, 0.06337916, 0.06352054, 0.06368553,
       0.06387836, 0.06410408, 0.06436873, 0.06467961, 0.06504553,
       0.06547715, 0.06598741, 0.06626396])

dlogdp_par=np.array([0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625,
       0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625,
       0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625,
       0.0625, 0.0625, 0.0625, 0.0625, 0.0625])

# Names and naming formats encountered
filename_formats = [
['%Y-%m-%d.ions.nds','%Y-%m-%d.particles.nds','%Y-%m-%d.log'],
['%Y%m%d-block-ions.spectra','%Y%m%d-block-particles.spectra','%Y%m%d-block.records'],
['%Y%m%d-block-ions.spectra','%Y%m%d-block-particles.spectra','%Y%m%d-block.diagnostics']]

# Possible names for diagnostic parameters
possible_sampleflow_names = [
'pos_sampleflow.mean',
'neg_sampleflow.mean',
'pos_sampleflow',
'neg_sampleflow',
'sampleflow',
'Flowaer']

possible_temperature_names = [
'temperature.mean',
'temperature',
'temp']

possible_pressure_names = [
'baro.mean',
'baro']

# Define standard conditions
temp_ref = 273.15 # K, 0C
pres_ref = 101325.0 # Pa, 1atm

def make_config():
    """ Make a configuration file for processing NAIS data """

    # Collect data from the user
    print()
    print("Enter name of configuration file.")
    print("E.g. ./configs/campaign.yml")
    while True:
        config_file = input("> ")
        if len(config_file)>0:
            if not config_file.lower().endswith(".yml"):
                config_file = config_file + ".yml"
            break
        else:
            continue

    print()
    print("Give path(s) to raw data. If multiple paths give them as comma separated list.")
    print("E.g. /data/nais/2021,/data/nais/2022")
    while True:
        user_input = input("> ")
        if len(user_input)>0:
            load_path=user_input.split(",")
            break
        else:
            continue

    print()
    print("Path to where processed data is saved.")
    print("E.g. ./data/campaign/processed")
    while True:
        save_path = input("> ")
        if len(save_path)>0:
            break
        else:
            continue

    print()
    print("Path to where figures are saved. Leave empty if no figures.")
    print("E.g. ./data/campaign/figures")
    fig_path = input("> ")

    print()
    print("Start of measurement (YYYY-MM-DD)")
    while True:
        start_date = input("> ")
        try:
            start_dt = pd.to_datetime(start_date)
            break
        except:
            continue

    print()
    print("End of measurement (YYYY-MM-DD)")
    print("If empty processor assumes current day, use for continuous measurements.")
    while True:
        end_date = input("> ")
        if len(end_date)==0:
            break
        try:
            end_dt = pd.to_datetime(end_date)
            break
        except:
            continue

    print()
    print("Enter name of database file")
    print("E.g. ./logs/campaign.json")
    while True:
        database_file = input("> ")
        if len(database_file)>0:
            if not database_file.lower().endswith(".json"):
                database_file = database_file + ".json"
            break
        else:
            continue

    print()
    print("Measurement location")
    print("E.g. Helsinki, Kumpula")
    location = input("> ")

    print()
    print("Apply corrections to data? (True/False)")
    print("Requires a NAIS with temperature and pressure sensors.")
    while True:
        apply_corrections = input("> ")
        if ((apply_corrections=='True') or (apply_corrections=='False')):
            if apply_corrections=='True':
                apply_corrections=True
            else:
                apply_corrections=False
                sealevel_correction=False
                inlet_length = 0.0
            break
        else:
            continue

    if apply_corrections:
        print()
        print("Length of the inlet in meters")
        while True:
            inlet_length = input("> ")
            try:
                inlet_length = float(inlet_length)
                break
            except:
                continue

        print()
        print("Correct concentrations to sealevel conditions? (True/False)")
        while True:
            sealevel_correction = input("> ")
            if ((sealevel_correction=='True') or (sealevel_correction=='False')):
                if sealevel_correction=='True':
                    sealevel_correction=True
                else:
                    sealevel_correction=False
                break
            else:
                continue

    print()
    print("Configuration saved to: %s"%config_file)
    print()

    # Make a dictionary out of the user input
    config_info = {
        "data_folder": load_path,
        "processed_folder": save_path,
        "figure_folder": fig_path,
        "start_date": start_date,
        "end_date": end_date,
        "database_file": database_file,
        "location": location,
        "inlet_length": inlet_length,
        "apply_corrections":apply_corrections,
        "sealevel_correction": sealevel_correction
    }

    # Save the config file
    with open(config_file,"w") as cf:
        yaml.dump(config_info,cf)

# FUNCTIONS TO CALCULATE LOSSES IN THE INLET
def visc(temp):
    """ Calculate viscosity of air """

    nyy_ref=18.203e-6
    S=110.4
    temp_ref=293.15
    nyy=nyy_ref*((temp_ref+S)/(temp+S))*((temp/temp_ref)**(3./2.))
    return nyy

def rlambda(temp,press):
    """ Calculate mean-free path """

    kaasuv=8.3143
    dm=3.7e-10
    avoc=6.022e23
    return kaasuv*temp/(np.sqrt(2.)*avoc*press*np.pi*dm*dm)

def cunn(Dp,temp,press):
    """ Calculate Cunningham correction """

    lambd = rlambda(temp,press)
    return 1. + 2.*lambd/Dp * (1.165 + 0.483 * np.exp(-0.997*Dp/lambd))

def diffuus(dpp,temp,press):
    """ Calculate diffusion coefficient """

    K=1.38e-23
    return (K*temp*cunn(dpp,temp,press))/(3.*np.pi*visc(temp)*dpp)

def tubeloss(dpp,pflow,plength,temp,press):
    """ Laminar diffusion losses in circular straight conduit """

    DPP,TEMP = np.meshgrid(dpp,temp)
    DPP,PRESS = np.meshgrid(dpp,press)
    DPP,PFLOW = np.meshgrid(dpp,pflow)

    rmuu = np.pi*diffuus(DPP,TEMP,PRESS)*plength/PFLOW;
    pene = np.zeros(rmuu.shape)

    cond1=rmuu<0.02
    cond2=rmuu>=0.02
    pene[cond1]=1. - 2.56*rmuu[cond1]**(2./3.) + 1.2*rmuu[cond1]+0.177*rmuu[cond1]**(4./3.)
    pene[cond2]=1. - 2.56*rmuu[cond2]**(2./3.) + 1.2*rmuu[cond2]+0.177*rmuu[cond2]**(4./3.)

    return pene

# CONVERT TO MATLAB DATENUM
def datetime2datenum(dt):
    """ Convert from python datetime to matlab datenum """

    mdn = dt + timedelta(days = 366)
    frac = (dt-datetime(dt.year,dt.month,dt.day,0,0,0,tzinfo=dt.tzinfo)).seconds \
           / (24.0 * 60.0 * 60.0)
    return mdn.toordinal() + frac

# PLOTTING FUNCTION
def plot_sumfile(handle,v,clim=(10,100000)):
    """ Plot UHEL's sum-formatted aerosol number-size distribution """

    time = v[1:,0] # This is datenum
    dp = v[0,2:]
    data = v[1:,2:]
    data[data<=0]=1e-30 # No holes in plots
    mesh_dp, mesh_time = np.meshgrid(dp,time)
    pcolorplot = handle.pcolormesh(mesh_time,mesh_dp,data,
                                   norm=colors.LogNorm(),
                                   linewidth=0,rasterized=True,cmap='jet')
    handle.set_yscale('log')
    pcolorplot.set_clim(clim)
    pcolorplot.set_edgecolor('face')
    handle.autoscale(tight='true')
    #handle.set_xlim((np.floor(time[0]),np.floor(time[0])+1)) # makes 24-h axis

    handle.grid('on',which='both',linestyle='--',color='w',lw=0.5)
    handle.xaxis.set_major_locator(dts.HourLocator(interval=1))
    handle.xaxis.set_major_formatter(dts.DateFormatter('%H'))
    #handle.set_xticks(np.floor(time[0])+np.arange(0,25)/24.0)
    #handle.set_xticklabels(["%2.2i" % x for x in np.arange(0,25)])
    plt.setp(handle.get_xticklabels(), rotation=80)
    handle.set_ylabel('Dp, [m]')
    handle.set_xlabel('UTC'+'%+d'%v[0,0]+', [h]')
    cbar = plt.colorbar(pcolorplot, ax=handle,
                        ticks=LogLocator(subs=range(10)))
    cbar.set_label('dN/dlogDp, [cm-3]')
    return pcolorplot

# PARSE THROUGH THE RAW DATA FILES
def read_file(fn):
    """ Read the data files to a pandas dataframe """
    with open(fn) as f:

        header_found = False
        data_matrix=[]
        line = f.readline()
        while line:

             line = line.rstrip("\n")

             # Skip completely empty line
             if len(line)==0:
                 line = f.readline()
                 continue

             # Skip commented line
             if line[0]=='#':
                 line = f.readline()
                 continue

             # Extract header line
             if header_found==False:
                 delimiter = re.search('(.)opmode',line).group(1)
                 header = line.split(delimiter)
                 number_of_columns = len(header)
                 header_found = True
                 line = f.readline()
                 continue

             data_line = line.split(delimiter)

             # Add data line to data matrix only if correct number of columns
             # and not a header line if header is inserted mid file
             # when the NAIS is restarted
             if ((len(data_line)==number_of_columns) & ("opmode" not in data_line)):
                 data_matrix.append(data_line)

             line = f.readline()

    # Construct data frame
    # In the data convert anything that can be converted to float and the rest is coerced to NaNs
    df = pd.DataFrame(columns = header, data = data_matrix)
    df.iloc[:,3:] = df.iloc[:,3:].apply(pd.to_numeric, errors='coerce').astype(float)

    # Convert infinities to nan also in case any were present for some reason
    df.iloc[:,3:] = df.iloc[:,3:].replace([np.inf,-np.inf],np.nan)

    return df

# FUNCTIONS TO AVERAGE DATA INTO FIXED BINS
def average_mob(y,h):
    data = np.nan*np.ones((y.shape[0],len(mob_ion)))

    for i in range(0,len(mob_ion_geomeans)):
        if i==0:
            y_block = y[:,h>mob_ion_geomeans[i]]
        else:
            y_block = y[:,((h>mob_ion_geomeans[i]) & (h<=mob_ion_geomeans[i-1]))]

        data[:,i] = np.nanmean(y_block,axis=1)

    y_block = y[:,h<=mob_ion_geomeans[i]]
    data[:,i+1] = np.nanmean(y_block,axis=1)

    return data

def average_dp(y,h):
    data = np.nan*np.ones((y.shape[0],len(dp_par)))

    for i in range(0,len(dp_par_geomeans)):
        if i==0:
            y_block = y[:,h<dp_par_geomeans[i]]
        else:
            y_block = y[:,((h<dp_par_geomeans[i]) & (h>=dp_par_geomeans[i-1]))]

        data[:,i] = np.nanmean(y_block,axis=1)

    y_block = y[:,h>=dp_par_geomeans[i]]
    data[:,i+1] = np.nanmean(y_block,axis=1)

    return data

def find_diagnostic_names(diag_params):
    """
    Find the names of important diagnostic
    parameters in the diagnostic file

    """
    sampleflow_name=None
    sampleflow_names=None
    temperature_name=None
    pressure_name=None

    for temp_name in possible_temperature_names:
         if temp_name in diag_params:
             temperature_name = temp_name
             break

    for pres_name in possible_pressure_names:
        if pres_name in diag_params:
            pressure_name = pres_name
            break

    sf_name = []
    for flow_name in possible_sampleflow_names:
        if flow_name in diag_params:
            sf_name.append(flow_name)

    if len(sf_name)==2:
        sampleflow_names = sf_name
    if len(sf_name)==1:
        sampleflow_name = sf_name

    return temperature_name, pressure_name, sampleflow_names, sampleflow_name

def process_data(
    df,
    rec,
    mode,
    apply_corr,
    sealevel_corr,
    pipel):
    """ Create the number-size distribution files and apply corrections """

    try:
        df = df.set_index(df.columns[0])
        df.index = [parse(y) for y in df.index]
        df_columns = df.columns
        df_inverter_reso = int((len(df_columns)-2)/4)

        # get the number densities and interpolate out the nans
        neg_df = df.iloc[:,2:2+df_inverter_reso+1].astype(float)
        neg_df = neg_df.interpolate().values
        pos_df = df.iloc[:,2+2*df_inverter_reso:2+3*df_inverter_reso+1].astype(float)
        pos_df = pos_df.interpolate().values

        if mode=="ions":
            mob_ion_inv = np.array([float(re.findall(r"[-+]?\d*\.\d+|\d+",y)[0])
                                    for y in df_columns[2:2+df_inverter_reso+1]])
            neg_df = average_mob(neg_df,mob_ion_inv)
            pos_df = average_mob(pos_df,mob_ion_inv)

            # get the number size distributions
            neg_df = neg_df * dlogmob_ion / dlogdp_ion
            pos_df = pos_df * dlogmob_ion / dlogdp_ion

        if mode=="particles":
            dp_par_inv = 2.0*np.array([float(re.findall(r"[-+]?\d*\.\d+|\d+",y)[0])
                                       for y in df_columns[2:2+df_inverter_reso+1]])
            neg_df = average_dp(neg_df,dp_par_inv)
            pos_df = average_dp(pos_df,dp_par_inv)

        # If all data is NaNs then skip
        if (np.all(np.isnan(neg_df)) |
            np.all(np.isnan(pos_df)) ):
            return None

        if apply_corr:
            rec = rec.set_index('opmode')

            # If relevant diagnostic data to make corrections does not exist:
            # return nothing
            t_name,p_name,sf_names,sf_name = find_diagnostic_names(list(rec))
            if ((t_name is not None) & (p_name is not None) &
                ((sf_names is not None) | (sf_name is not None))):
                pass
            else:
                return None

            # Then extract the records that match the polarity
            if mode=="ions":
                df_rec = rec.loc['ions'].set_index(rec.columns[0])
            if mode=="particles":
                df_rec = rec.loc['particles'].set_index(rec.columns[0])

            df_rec.index = [parse(y) for y in df_rec.index]

            # Match records to data according to time
            df_rec = df_rec.reindex(index=df.index,method='nearest')

            # Temperature
            t_df = 273.15 + df_rec[t_name].astype(float)
            t_df = t_df.interpolate().values.flatten()

            # Pressure
            p_df = 100.0 * df_rec[p_name].astype(float)
            p_df = p_df.interpolate().values.flatten()

            # Sampleflow
            if sf_names is not None:
                flow_df = df_rec[sf_names].astype(float).sum(axis=1)
                flow_df = flow_df.interpolate().values.flatten()
            if sf_name is not None:
                flow_df = df_rec[sf_name].astype(float)
                flow_df = flow_df.interpolate().values.flatten()

            # If any of the relevant data are all NaNs return nothing
            if (np.all(np.isnan(neg_df)) |
                np.all(np.isnan(pos_df)) |
                np.all(np.isnan(flow_df)) |
                np.all(np.isnan(p_df)) |
                np.all(np.isnan(t_df))):
                return None

            # Test if the sampleflow is in cm3/s (old models) or l/min and possibly convert to l/min
            if flow_df[0]>100:
                flow_df = (flow_df/1000.0) * 60.0
            else:
                pass

            # Correct the number concentrations to standard conditions (optional)
            if (sealevel_corr):
                stp_corr_df = (pres_ref*t_df)/(temp_ref*p_df)
                neg_df = (stp_corr_df*neg_df.T).T
                pos_df = (stp_corr_df*pos_df.T).T

            # Diffusion loss correction
            if mode=="ions":
                throughput_df = tubeloss(dp_ion*1e-9,flow_df*1.667e-5,pipel,t_df,p_df)
            if mode=="particles":
                throughput_df = tubeloss(dp_par*1e-9,flow_df*1.667e-5,pipel,t_df,p_df)
            neg_df = neg_df / throughput_df
            pos_df = pos_df / throughput_df

            # Robert Wagner's calibration (only ions)
            if mode=="ions":
                roberts_corr = 0.713*dp_ion**0.120
                neg_df = neg_df / roberts_corr
                pos_df = pos_df / roberts_corr

        # CREATE FINAL DATA MATRICES

        # Integrate total number concentrations
        if mode=="ions":
            total_neg_df = np.nansum(neg_df*dlogdp_ion,axis=1)[np.newaxis].T
            total_pos_df = np.nansum(pos_df*dlogdp_ion,axis=1)[np.newaxis].T
        if mode=="particles":
            total_neg_df = np.nansum(neg_df*dlogdp_par,axis=1)[np.newaxis].T
            total_pos_df = np.nansum(pos_df*dlogdp_par,axis=1)[np.newaxis].T

        # Get the utc offset
        if df.index[0].utcoffset()==None:
            utc_offset = 0
        else:
            utc_offset = df.index[0].utcoffset().total_seconds()/(60.0*60.0)

        time_df = np.array([datetime2datenum(x) for x in df.index])[np.newaxis].T

        # Construct the headers
        if mode=="ions":
            df_header = np.insert(dp_ion*1e-9,0,(utc_offset,0))[np.newaxis]
        if mode=="particles":
            df_header = np.insert(dp_par*1e-9,0,(utc_offset,0))[np.newaxis]

        # Construct the sum-files
        negdf = np.concatenate((df_header,np.concatenate((time_df,total_neg_df,neg_df),axis=1)))
        posdf = np.concatenate((df_header,np.concatenate((time_df,total_pos_df,pos_df),axis=1)))

        return [negdf,posdf]

    except:
        return None

def nais_processor(config_file):
    """ Function that processes data from the NAIS

    The function creates specially formatted number-size distribution
    files from neutral cluster and air ion spectrometer (NAIS) data files
    and optionally corrects for losses, applies ion mode calibration
    and corrects concentrations to standard conditions.

    A measurement setup specific configuration file is needed in the
    processing. A configuration file can be created using make_config().

    Four types of files are created:

    NAISn[yyyymmdd]nds.sum
        ion number size distribution, negative polarity
    NAISp[yyyymmdd]nds.sum
        ion number size distribution, positive polarity
    NAISn[yyyymmdd]np.sum
        particle number size distribution, negative polarity
    NAISp[yyyymmdd]np.sum
        particle number-size distribution, positive polarity

    The created processed files have the following format
        [0,0]  = UTC offset in hours
        [1:,0] = time (MATLAB datenum)
        [0,2:] = geometric mean diameter of size-channel (m)
        [1:,1] = integrated total number concentration (cm-3)
        [1:,2:] = normalized number concentrations, dN/dlogDp (cm-3)

    Function arguments:
        Name of the configuration file (str)

    Example:
        nais_processor('/home/user/data/config.yml')

    """

    # Find out today
    today_dt = datetime.today()
    today = today_dt.strftime('%Y%m%d')

    # Check that the config file exists
    if os.path.isfile(config_file)==False:
        print('"%s" does not exist' % config_file)
        return
    else:
        # Try to parse the config file
        with open(config_file,'r') as stream:
            try:
                config = yaml.safe_load(stream)

                load_path = config['data_folder']
                save_path = config['processed_folder']
                start_date = config['start_date']
                database = config['database_file']
                location = config['location']
                end_date = config['end_date']
                if len(end_date)==0:
                    end_date = today
                pipelength = config['inlet_length']
                sealevel_correction = config['sealevel_correction']
                apply_corrections = config['apply_corrections']
            except:
                print("Something went wrong with parsing %s",config_file)
                return

    # Then check if you can initialize th database
    try:
      db = TinyDB(database)
      check = Query()
    except:
        print("Could not initialize database")
        return

    # Test if the configuration information is valid
    try:
        float(pipelength)
    except:
        print('"%s" must be a number' % pipelength)
        return

    # Test if start and end dates are valid
    try:
       start_dt = pd.to_datetime(start_date)
       end_dt = pd.to_datetime(end_date)
    except:
       print('bad start_date or end_date')
       return

    # Check if given data folders exist
    for x in load_path:
        if os.path.exists(x):
            continue
        else:
            print("At least one data folder does not exist")
            return

    # Check for save path and create folders if they do not exist.
    if os.path.exists(save_path):
        pass
    else:
        print("Save path does not exist")
        return

    print("Configuration file: %s" % config_file)

    start_date_str = start_dt.strftime("%Y%m%d")
    end_date_str = end_dt.strftime("%Y%m%d")

    # Convert load and save paths to absolute paths
    load_path = [os.path.abspath(x) + '/' for x in load_path]
    save_path = os.path.abspath(save_path) + '/'

    # Make a list of datetimes that the config file is interested in
    list_of_datetimes = pd.date_range(start=start_date_str, end=end_date_str)

    # list existing dates
    list_of_existing_dates = [x['timestamp'] for x in db.search(check.diagnostics.exists())]
    earliest_existing_date = np.min(pd.to_datetime(list_of_existing_dates))

    # Add unprocessed datafiles to the database
    for x in list_of_datetimes:
        if ((x.strftime('%Y%m%d') in list_of_existing_dates) |
            (x < earliest_existing_date)):
            continue
        else:
            files_found=False
            for z in load_path:
                for y in filename_formats:

                    if ( (os.path.exists(z+x.strftime(y[0])) | # ions
                         os.path.exists(z+x.strftime(y[1]))) & # particles
                         os.path.exists(z+x.strftime(y[2])) # diagnostics
                       ):

                        db.insert(
                            {'timestamp':x.strftime('%Y%m%d'),
                            'diagnostics':z+x.strftime(y[2])}
                            )

                        if os.path.exists(z+x.strftime(y[0])):
                            db.update(
                                {'ions':z+x.strftime(y[0])},
                                check.timestamp==x.strftime('%Y%m%d'))

                        if os.path.exists(z+x.strftime(y[1])):
                            db.update(
                                {'particles':z+x.strftime(y[1])},
                                check.timestamp==x.strftime('%Y%m%d'))

                        files_found=True
                        break

                if files_found:
                    break

    # From the database find the last day with processed data
    processed_days = db.search(
        check.processed_neg_ion_file.exists() |
        check.processed_pos_ion_file.exists() |
        check.processed_neg_particle_file.exists() |
        check.processed_pos_particle_file.exists())

    if len(processed_days)!=0:
        last_day=np.max([datetime.strptime(x['timestamp'],'%Y%m%d') for x in processed_days]).strftime('%Y%m%d')
    else:
        last_day=None

    # Iterate through data that can be processed
    # But is still not processed
    for x in iter(db.search(
        ((check.timestamp==last_day) &
         (check.timestamp>=start_date_str) &
         (check.timestamp<=end_date_str)) |
         (check.diagnostics.exists() &
          (check.ions.exists() |
          check.particles.exists()) &
          ~check.processed_neg_ion_file.exists() &
          ~check.processed_pos_ion_file.exists() &
          ~check.processed_neg_particle_file.exists() &
          ~check.processed_pos_particle_file.exists() &
          (check.timestamp>=start_date_str) &
          (check.timestamp<=end_date_str)))):

        print('processing %s' % x['timestamp'])

        # Read the diagnostics
        records = read_file(x['diagnostics'])

        ions_exist=db.search(
            check.ions.exists() &
            (check.timestamp==x['timestamp']))
        particles_exist=db.search(
            check.particles.exists() &
            (check.timestamp==x['timestamp']))

        # ions
        if ions_exist:

            ions = read_file(x['ions'])
            ion_datamatrices = process_data(
                 ions,
                 records,
                 "ions",
                 apply_corrections,
                 sealevel_correction,
                 pipelength)

            if ion_datamatrices is not None:

                # Save the sum matrices using the standard names
                np.savetxt(save_path+'NAISn'+x['timestamp']+'nds.sum',ion_datamatrices[0])
                np.savetxt(save_path+'NAISp'+x['timestamp']+'nds.sum',ion_datamatrices[1])

                # Update the database
                db.update(
                    {'processed_neg_ion_file': save_path+'NAISn'+x['timestamp']+'nds.sum',
                    'processed_pos_ion_file': save_path+'NAISp'+x['timestamp']+'nds.sum'},
                    check.timestamp==x['timestamp'])

        # particles
        if particles_exist:

            # Process particles
            particles = read_file(x['particles'])
            particle_datamatrices = process_data(
                particles,
                records,
                "particles",
                apply_corrections,
                sealevel_correction,
                pipelength)

            if particle_datamatrices is not None:

                # Save the sum matrices using the standard names
                np.savetxt(save_path+'NAISn'+x['timestamp']+'np.sum',particle_datamatrices[0])
                np.savetxt(save_path+'NAISp'+x['timestamp']+'np.sum',particle_datamatrices[1])

                # Update the database
                db.update(
                    {'processed_neg_particle_file': save_path+'NAISn'+x['timestamp']+'np.sum',
                    'processed_pos_particle_file': save_path+'NAISp'+x['timestamp']+'np.sum'},
                    check.timestamp==x['timestamp'])

    print("Done!")

def do_figs(config_file):
    """ Plot the processed NAIS data

    Function arguments:
        Name of the configuration file (str)

    Example:
        do_figs('/home/user/data/config.yml')

    """

    # Find out today
    today_dt = datetime.today()
    today = today_dt.strftime('%Y%m%d')

    # Check that the config file exists
    if os.path.isfile(config_file) == False:
        print('"%s" does not exist' % config_file)
        return
    else:
        # Try to parse the config file
        with open(config_file,'r') as stream:
            try:
                config = yaml.safe_load(stream)

                save_path = config['processed_folder']
                database = config['database_file']
                location = config['location']
                fig_path = config['figure_folder']
                if len(fig_path)==0:
                    fig_path = None
            except:
                print("Something went wrong with parsing %s",config_file)
                return

    # Check if you can initialize the database
    try:
      db = TinyDB(database)
      check = Query()
    except:
        print("Could not initialize database")
        return

    # Check if processed data path exists
    if not os.path.exists(save_path):
        print("Path to processed data is invalid")
        return

    # Check the fig path exists
    if fig_path is not None:
        if not os.path.exists(fig_path):
            print('Figure path does not exist')
            return
    else:
        print("figure path not given")
        return

    fig_path = os.path.abspath(fig_path) + '/'

    # Define some plotting styles
    plt.style.use('dark_background')

    fontsize = 14
    plt.rcParams.update({'font.size': fontsize,
                         'axes.titlesize': fontsize,
                         'axes.labelsize': fontsize,
                         'xtick.labelsize': fontsize,
                         'ytick.labelsize': fontsize,
                         'figure.titlesize': fontsize,
                         'legend.fontsize': fontsize})

    # From the database find the last day with processed data
    processed_days = db.search(
        check.processed_neg_ion_file.exists() |
        check.processed_pos_ion_file.exists() |
        check.processed_neg_particle_file.exists() |
        check.processed_pos_particle_file.exists())

    if len(processed_days)!=0:
        last_day=np.max([datetime.strptime(x['timestamp'],'%Y%m%d') for x in processed_days]).strftime('%Y%m%d')
    else:
        last_day=None

    # Iterate through data that can be plotted, but is not
    for x in iter(db.search(
        (
          (check.processed_neg_ion_file.exists() &
          check.processed_pos_ion_file.exists() &
          (check.timestamp==last_day)) |
          (check.processed_neg_particle_file.exists() &
          check.processed_pos_particle_file.exists() &
          (check.timestamp==last_day)) |
          (check.processed_neg_ion_file.exists() &
          check.processed_pos_ion_file.exists() &
          ~check.ion_figure.exists()) |
          (check.processed_neg_particle_file.exists() &
          check.processed_pos_particle_file.exists() &
          ~check.particle_figure.exists())
        )
      )
    ):

        print('plotting %s' % x['timestamp'])

        ions_exist=db.search(
            check.processed_neg_ion_file.exists() &
            check.processed_pos_ion_file.exists() &
            (check.timestamp==x['timestamp']))
        particles_exist=db.search(
            check.processed_neg_particle_file.exists() &
            check.processed_pos_particle_file.exists() &
            (check.timestamp==x['timestamp']))

        if ions_exist:
            negion=np.loadtxt(x["processed_neg_ion_file"])
            posion=np.loadtxt(x["processed_pos_ion_file"])
            fig,ax = plt.subplots(2,1,figsize=(7,7.5),dpi=100)
            ax = ax.flatten()
            plot_sumfile(ax[0],posion,clim=(10,10000))
            plot_sumfile(ax[1],negion,clim=(10,10000))
            ax[0].set_title('Positive ions')
            ax[1].set_title('Negative ions')
            plt.tight_layout(rect=[0, 0.0, 1, 0.96])
            fig.suptitle('NAIS' + ' ' + x['timestamp'] + '\n' + location, y=1.0)
            plt.savefig(fig_path+'NAIS_ions_'+ x['timestamp'] +'.png',dpi=100,bbox_inches='tight')
            db.update({'ion_figure': fig_path+'NAIS_ions_'+ x['timestamp'] +'.png'}, check.timestamp==x['timestamp'])
            plt.close()

        if particles_exist:
            negpar=np.loadtxt(x["processed_neg_particle_file"])
            pospar=np.loadtxt(x["processed_pos_particle_file"])
            fig,ax = plt.subplots(2,1,figsize=(7,7.5),dpi=100)
            ax = ax.flatten()
            plot_sumfile(ax[0],pospar,clim=(10,100000))
            plot_sumfile(ax[1],negpar,clim=(10,100000))
            ax[0].set_title('Particles (positive polarity)')
            ax[1].set_title('Particles (negative polarity)')
            plt.tight_layout(rect=[0, 0.0, 1, 0.96])
            fig.suptitle('NAIS' + ' ' + x['timestamp'] + '\n' + location, y=1.0)
            plt.savefig(fig_path+'NAIS_particles_'+ x['timestamp'] +'.png',dpi=100,bbox_inches='tight')
            db.update({'particle_figure': fig_path+'NAIS_particles_'+x['timestamp'] +'.png'}, check.timestamp==x['timestamp'])
            plt.close()

    print("Done!")


from DBConnection import DBConnection
from typing import *

class DBFunctions:
    db_instance = DBConnection()
    db_connection = db_instance.return_connection()

    def get_ports(self)->List[Tuple[int, str]]:
        """
        Function that retrieves port list with services from DB
        :return: List of tuples in format (port, service)
        """
        cursor = self.db_connection.cursor()
        cursor.execute('SELECT * FROM PortsServices')
        port_services = []
        row = cursor.fetchone()
        while row:
            port_services.append(row)
            row = cursor.fetchone()
        return port_services

    def get_websites(self)->List[str]:
        cursor = self.db_connection.cursor()
        cursor.execute('Select URL from Websites')
        web_urls = []
        row = cursor.fetchone()
        while row:
            web_urls.append(row[0])
            row = cursor.fetchone()
        return web_urls

    def trunc_error_tables(self):
        conn = self.db_connection
        cursor = conn.cursor()
        cursor.execute('Truncate table HttpsErrors')
        cursor.execute('Truncate table Clickjack_Websites')
        cursor.execute('Truncate table Websites_Status_Codes')
        conn.commit()

    def insert_https_error(self, url):
        conn = self.db_connection
        cursor = conn.cursor()
        cursor.execute("insert into HttpsErrors VALUES (%s)", url)
        conn.commit()

    def insert_clickjack_website(self, url):
        conn = self.db_connection
        cursor = conn.cursor()
        cursor.execute("insert into Clickjack_Websites VALUES (%s)", url)
        conn.commit()

    def insert_web_status(self, url, code):
        conn = self.db_connection
        cursor = conn.cursor()
        cursor.execute("insert into Websites_Status_Codes VALUES (%s, %s)", (url, code))
        conn.commit()


from setuptools import setup

with open('README.md', 'r') as fh:
    long_description = fh.read()

setup(
    name='nnir',
    version='1.0.0',
    description='User-friendly machine learning tool for image classification',
    long_description=long_description,
    author='Nesac128',
    author_email='ml.learner.8359@gmail.com',
    long_description_content_type="text/markdown",
    url="https://github.com/Nesac128/NNIR",
    install_requires=['tensorflow==1.9.0', 'Pillow==5.1.0', 'scikit-learn==0.19.1',
                      'numpy==1.14.5', 'opencv-python==3.4.0.12', 'pandas==0.22.0',
                      'click==6.7', 'scipy'],
    classifiers=(
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: GPL-2.0 License",
        "Operating System :: POSIX",
        "Programming Language :: Python"
    )
)


"""
break se utiliza para finalizar y salir el bucle, por ejemplo, si se cumple alguna condición.

Por su parte, continue salta al siguiente paso de la iteración, ignorando todas las sentencias que le siguen y que forman parte del bucle.

Un ejemplo es la mejor manera de entenderlo.
"""

# Uso de break. Encontrar un elemento en una colección

coleccion = [2, 4, 5, 7, 8, 9, 3, 4]
for e in coleccion:
    if e == 7:
        break
    print(e)

# Uso de continue. Imprimir solo los números pares de una colección

coleccion = [2, 4, 5, 7, 8, 9, 3, 4]
for e in coleccion:
    if e % 2 != 0:
        continue
    print(e)

#En este caso, el código anterior mostrará los números 2, 4, 8 y 4.

j2logo/2 - Estructuras de control de flujo/49.iteracion_for_ break&continue.py