import json
from types import SimpleNamespace
import os.path
import pprint
from absl import logging
import numpy as np
import tensorflow.compat.v1 as tf
import tensorflow_addons as tfa
import kws_streaming.data.input_data as input_data
from kws_streaming.models import models
from kws_streaming.models import utils

import math

from transformers import AdamWeightDecay

def train(flags):
  """Model training."""

  flags.training = True

  # Set the verbosity based on flags (default is INFO, so we see all messages)
  logging.set_verbosity(flags.verbosity)

  # Start a new TensorFlow session.
  tf.reset_default_graph()
  config = tf.ConfigProto()
  config.gpu_options.allow_growth = True
  sess = tf.Session(config=config)
  tf.keras.backend.set_session(sess)

  audio_processor = input_data.AudioProcessor(flags)

  time_shift_samples = int((flags.time_shift_ms * flags.sample_rate) / 1000)

  # Figure out the learning rates for each training phase. Since it's often
  # effective to have high learning rates at the start of training, followed by
  # lower levels towards the end, the number of steps and learning rates can be
  # specified as comma-separated lists to define the rate at each stage. For
  # example --how_many_training_steps=10000,3000 --learning_rate=0.001,0.0001
  # will run 13,000 training loops in total, with a rate of 0.001 for the first
  # 10,000, and 0.0001 for the final 3,000.
  training_steps_list = list(map(int, flags.how_many_training_steps.split(',')))
  learning_rates_list = list(map(float, flags.learning_rate.split(',')))
  if len(training_steps_list) != len(learning_rates_list):
    raise Exception(
        '--how_many_training_steps and --learning_rate must be equal length '
        'lists, but are %d and %d long instead' % (len(training_steps_list),
                                                   len(learning_rates_list)))
  logging.info(flags)

  model = models.MODELS[flags.model_name](flags)
  if flags.distill_teacher_json:
    with open(flags.distill_teacher_json, 'r') as f:
      teacher_flags = json.load(f, object_hook=lambda d: SimpleNamespace(
        **{ k: v for k, v in flags.__dict__.items() if not k in d },
        **d))
    teacher_base = models.MODELS[teacher_flags.model_name](teacher_flags)
    hard_labels = tf.keras.layers.Lambda(lambda logits: tf.one_hot(tf.math.argmax(logits, axis=-1), depth=flags.label_count))
    teacher = tf.keras.models.Sequential([teacher_base, hard_labels])
    teacher_base.trainable = False
    teacher.trainable = False
  else:
    teacher = None
    teacher_flags = None

  base_model = model

  logging.info(model.summary())

  # save model summary
  utils.save_model_summary(model, flags.train_dir)

  # save model and data flags
  with open(os.path.join(flags.train_dir, 'flags.txt'), 'wt') as f:
    pprint.pprint(flags, stream=f)

  loss = tf.keras.losses.CategoricalCrossentropy(from_logits=True, label_smoothing=flags.label_smoothing)
  metrics = ['accuracy']

  if flags.optimizer == 'adam':
    optimizer = tf.keras.optimizers.Adam(epsilon=flags.optimizer_epsilon)
  elif flags.optimizer == 'momentum':
    optimizer = tf.keras.optimizers.SGD(momentum=0.9)
  elif flags.optimizer == 'novograd':
    optimizer = tfa.optimizers.NovoGrad(
        lr=0.05,
        beta_1=flags.novograd_beta_1,
        beta_2=flags.novograd_beta_2,
        weight_decay=flags.novograd_weight_decay,
        grad_averaging=bool(flags.novograd_grad_averaging))
  elif flags.optimizer == 'adamw':
    # Exclude some layers for weight decay
    exclude = ["pos_emb", "class_emb", "layer_normalization", "bias"]
    optimizer = AdamWeightDecay(learning_rate=0.05, weight_decay_rate=flags.l2_weight_decay, exclude_from_weight_decay=exclude)
  else:
    raise ValueError('Unsupported optimizer:%s' % flags.optimizer)

  loss_weights = [ 0.5, 0.5, 0.0 ] if teacher else [ 1. ] # equally weight losses form label and teacher, ignore ensemble output
  model.compile(optimizer=optimizer, loss=loss, loss_weights=loss_weights, metrics=metrics)

  train_writer = tf.summary.FileWriter(flags.summaries_dir + '/train',
                                       sess.graph)
  validation_writer = tf.summary.FileWriter(flags.summaries_dir + '/validation')

  sess.run(tf.global_variables_initializer())

  if flags.start_checkpoint:
    model.load_weights(flags.start_checkpoint).expect_partial()
    logging.info('Weights loaded from %s', flags.start_checkpoint)

  if teacher_flags and teacher_flags.start_checkpoint:
    # Load weights into teacher base as this is the actual model that was saved, teacher includes hard label head
    teacher_base.load_weights(teacher_flags.start_checkpoint).assert_existing_objects_matched()
    logging.info('Distillation teacher weights loaded from %s', teacher_flags.start_checkpoint)

  start_step = 0

  logging.info('Training from step: %d ', start_step)

  # Save graph.pbtxt.
  tf.train.write_graph(sess.graph_def, flags.train_dir, 'graph.pbtxt')

  # Save list of words.
  with tf.io.gfile.GFile(os.path.join(flags.train_dir, 'labels.txt'), 'w') as f:
    f.write('\n'.join(audio_processor.words_list))

  best_accuracy = 0.0

  # prepare parameters for exp learning rate decay
  training_steps_max = np.sum(training_steps_list)
  lr_init = learning_rates_list[0]
  exp_rate = -np.log(learning_rates_list[-1] / lr_init)/training_steps_max
  mode = 'training'

  if flags.lr_schedule == 'cosine':
    # Currently, no restarts are performed, so it is just a cosine decay over the entire
    # training process. I think this is how DeiT does it.
    lr_init = lr_init * flags.batch_size / 512
    num_train = audio_processor.set_size(mode)
    warmup_steps = int((num_train / flags.batch_size) * flags.warmup_epochs)
    first_decay_steps=training_steps_max

  # Training loop.
  for training_step in range(start_step, training_steps_max + 1):
    if training_step > 0:
      offset = (training_step -
                1) * flags.batch_size if flags.pick_deterministically else 0

      # Pull the audio samples we'll use for training.
      train_fingerprints, train_ground_truth = audio_processor.get_data(
          flags.batch_size, offset, flags, flags.background_frequency,
          flags.background_volume, time_shift_samples, mode,
          flags.resample, flags.volume_resample, sess)

      if flags.lr_schedule == 'exp':
        learning_rate_value = lr_init * np.exp(-exp_rate * training_step)
      elif flags.lr_schedule == 'linear':
        # Figure out what the current learning rate is.
        training_steps_sum = 0
        for i in range(len(training_steps_list)):
          training_steps_sum += training_steps_list[i]
          if training_step <= training_steps_sum:
            learning_rate_value = learning_rates_list[i]
            break
      elif flags.lr_schedule == 'cosine':
        learning_rate_value = lr_init * min(1, float(training_step) / max(1, warmup_steps)) * (math.cos(math.pi * training_step / training_steps_max) + 1) / 2.
      else:
        raise ValueError('Wrong lr_schedule: %s' % flags.lr_schedule)

      tf.keras.backend.set_value(model.optimizer.learning_rate, learning_rate_value)

      one_hot_labels = tf.keras.utils.to_categorical(train_ground_truth, num_classes=flags.label_count)

      if teacher:
        teacher_labels = teacher.predict_on_batch(train_fingerprints)
        one_hot_labels = [ one_hot_labels, teacher_labels, one_hot_labels ] # third is for the ensemble output, gradient is unused

      result = model.train_on_batch(train_fingerprints, one_hot_labels)

      if teacher:
        loss_total, loss_label, loss_teacher, loss_average, acc_label, acc_teacher, acc_ensemble = result
        differences = (teacher_labels != one_hot_labels).astype(dtype=int).sum()
        logging.info(
            'Step #%d: rate %f, accuracy %.2f%%, cross entropy %f, teacher acc %.2f%% (%d diff), teacher cross entropy %f, ensemble acc %.2f%%',
            *(training_step, learning_rate_value, acc_label * 100, loss_total, acc_teacher * 100, differences, loss_teacher, acc_ensemble * 100))
        summary = tf.Summary(value=[
            tf.Summary.Value(tag='accuracy', simple_value=acc_label),
            tf.Summary.Value(tag='teacher_accuracy', simple_value=acc_teacher),
            tf.Summary.Value(tag='ensemble_accuracy', simple_value=acc_ensemble),
        ])
      else:
        loss_label, acc_label = result
        logging.info(
            'Step #%d: rate %f, accuracy %.2f%%, cross entropy %f',
            *(training_step, learning_rate_value, acc_label * 100, loss_label))
        summary = tf.Summary(value=[
            tf.Summary.Value(tag='accuracy', simple_value=acc_label),
        ])

      train_writer.add_summary(summary, training_step)

    is_last_step = (training_step == training_steps_max)
    if (training_step % flags.eval_step_interval) == 0 or is_last_step:
      set_size = audio_processor.set_size('validation')
      set_size = int(set_size / flags.batch_size) * flags.batch_size
      total_accuracy = 0.0
      count = 0.0
      for i in range(0, set_size, flags.batch_size):
        validation_fingerprints, validation_ground_truth = audio_processor.get_data(
            flags.batch_size, i, flags, 0.0,
            0.0, 0, 'validation',
            0.0, 0.0, sess)

        one_hot_labels = tf.keras.utils.to_categorical(validation_ground_truth, num_classes=flags.label_count)
        if teacher:
          one_hot_labels = [ one_hot_labels, one_hot_labels, one_hot_labels ]
        # Run a validation step and capture training summaries for TensorBoard
        # with the `merged` op.
        result = model.test_on_batch(validation_fingerprints,
                                     one_hot_labels)

        if teacher:
          loss_total, loss_label, loss_teacher, loss_average, acc_label, acc_teacher, acc_ensemble = result
          summary = tf.Summary(value=[
          tf.Summary.Value(tag='accuracy', simple_value=acc_ensemble),
          tf.Summary.Value(tag='label_head_accuracy', simple_value=acc_label),
          tf.Summary.Value(tag='distill_head_accuracy', simple_value=acc_teacher),
          ])
          accuracy = acc_ensemble
        else:
          loss_label, acc_label = result
          summary = tf.Summary(value=[
              tf.Summary.Value(tag='accuracy', simple_value=acc_label),])
          accuracy = acc_label

        validation_writer.add_summary(summary, training_step)

        total_accuracy += accuracy
        count = count + 1.0

      total_accuracy = total_accuracy / count
      logging.info('Step %d: Validation accuracy = %.2f%% (N=%d)',
                   *(training_step, total_accuracy * 100, set_size))

      # Save the model checkpoint when validation accuracy improves
      if total_accuracy >= best_accuracy:
        best_accuracy = total_accuracy
        # overwrite the best model weights
        model.save_weights(flags.train_dir + 'best_weights')
      logging.info('So far the best validation accuracy is %.2f%%',
                   (best_accuracy * 100))

  tf.keras.backend.set_learning_phase(0)
  set_size = audio_processor.set_size('testing')
  set_size = int(set_size / flags.batch_size) * flags.batch_size
  logging.info('set_size=%d', set_size)
  total_accuracy = 0.0
  count = 0.0

  for i in range(0, set_size, flags.batch_size):
    test_fingerprints, test_ground_truth = audio_processor.get_data(
        flags.batch_size, i, flags, 0.0, 0.0, 0, 'testing', 0.0, 0.0, sess)

    one_hot_labels = tf.keras.utils.to_categorical(test_ground_truth, num_classes=flags.label_count)
    if teacher:
      one_hot_labels = [ one_hot_labels, one_hot_labels, one_hot_labels ]
    result = model.test_on_batch(test_fingerprints, one_hot_labels)

    total_accuracy += result[-1] if teacher else result[1]
    count = count + 1.0
  total_accuracy = total_accuracy / count

  logging.info('Final test accuracy = %.2f%% (N=%d)',
               *(total_accuracy * 100, set_size))
  with open(os.path.join(flags.train_dir, 'accuracy_last.txt'), 'wt') as fd:
    fd.write(str(total_accuracy * 100))
  model.save_weights(flags.train_dir + 'last_weights')


import os
import random
import numpy as np
import torch.nn.functional as F
from yolov5.utils.general import *
from yolov5.utils.boxes import xywh2xyxy
from yolov5.utils.plots import plot_images_boxes_and_masks
from yolov5.data import LoadImagesAndLabelsAndMasks, create_dataloader
import cv2
from yolov5.utils.general import init_seeds

names = ["balloon"]
seed = 2
init_seeds(seed)
colors = [[random.randint(0, 255) for _ in range(3)] for _ in range(len(names))]

with open("data/hyps/hyp.scratch.yaml") as f:
    hyp = yaml.load(f, Loader=yaml.FullLoader)  # load hyps

save = False
save_dir = "/d/projects/yolov5/data/play_phone0115/show_labels"

if save and not os.path.exists(save_dir):
    os.mkdir(save_dir)

dataloader, dataset = create_dataloader(
            # '/d/baidubase/COCO/val_yolo/images/train',
            # 'data/license_plates/images/train/',
            # '/d/projects/research/yolov5/data/coco/train2017.txt',
            '/d/projects/research/yolov5/data/coco/val2017.txt',
            # "/home/laughing/code/yolov5-6/data/seg/coco_02/images/train",
            imgsz=640,
            batch_size=4,
            stride=32,
            augment=False,
            shuffle=True,
            rank=-1,
            hyp=hyp,
            mask_head=True
)
cv2.namedWindow("mosaic", cv2.WINDOW_NORMAL)

for i, (imgs, targets, paths, _, masks) in enumerate(dataloader):
    #     print(targets)
    # print(targets)

    # show masks
    # mxywh = targets[:, 2:].cpu()
    # mxyxy = xywh2xyxy(mxywh) * torch.tensor((640, 640))[[0, 1, 0, 1]]
    # maskssss = masks.permute(1, 2, 0).contiguous()
    # for k in range(maskssss.shape[-1]):
    #     c1 = (int(mxyxy[k][0]), int(mxyxy[k][1]))
    #     c2 = (int(mxyxy[k][2]), int(mxyxy[k][3]))
    #     img = maskssss[:, :, k].cpu().numpy().astype(np.uint8) * 255
    #     cv2.rectangle(img, c1, c2, (255, 255, 255), -1, cv2.LINE_AA)  # filled
    #     cv2.imshow('p', cv2.resize(img, (640, 640)))
    #     if cv2.waitKey(0) == ord('q'):
    #         break
    # exit()

    if getattr(dataset, 'downsample_ratio', 1) != 1 and masks is not None:
        masks = F.interpolate(
            masks[None, :],
            (640, 640),
            mode="bilinear",
            align_corners=False,
        ).squeeze(0)

    result = plot_images_boxes_and_masks(images=imgs, targets=targets, paths=paths, masks=masks)
    cv2.imshow("mosaic", result[:, :, ::-1])
    if cv2.waitKey(0) == ord("q"):  # q to quit
        break
    continue
    # imgs = imgs.numpy().astype(np.uint8).transpose((1, 2, 0))
    # imgs = cv2.cvtColor(imgs, cv2.COLOR_BGR2RGB)
    # targets = targets.numpy()
    # temp = targets[:, 2:6].copy()
    # targets[:, 2:6] = xywh2xyxy(temp * 960)
    # for idx, cls, *xyxy in targets:
    #     label = '%s' % (names[int(cls)])
    #     plot_one_box(xyxy, imgs, label=label, color=colors[int(cls)], line_thickness=2)
    # # print(targets[targets[:, 0] == 0])
    # if save and len(targets[targets[:, 1] == 0]) > 0:
    #     cv2.imwrite(os.path.join(save_dir, os.path.split(paths)[-1]), imgs)
    # cv2.imshow('p', imgs)
    # cv2.waitKey(0)


# ---
# jupyter:
#   jupytext:
#     formats: ipynb,py:light
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.1
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# +
import praice
from praice import Trainer

print(praice.__version__)
# -

trainer = Trainer(
    ticker="MSFT",
    learners_cnf="default_ml_config",
    datasets_cnf="default_instrument_config",
)

# +
# %%time

trainer.run()
# -


from celery_app import celery_app, forward_to_fastapi
import logging

logging.basicConfig(level=logging.DEBUG)

def main():
    commentary = "Rodrigo"
    result = forward_to_fastapi.apply_async(args=[commentary])

    print("Sent the task.")
    response = result.get(timeout=30)
    print(f"Task Result: {response}")
    print(f"Task State: {result.state}")

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-

class Solution:
    def firstMissingPositive(self, nums):
        i = 0
        while i < len(nums):
            if 0 <= nums[i] - 1 < len(nums) and nums[i] != nums[nums[i] - 1]:
                nums[nums[i] - 1], nums[i] = nums[i], nums[nums[i] - 1]
            else:
                i += 1

        for i, num in enumerate(nums, 1):
            if i != num:
                return i
        return len(nums) + 1

if __name__ == "__main__":
    solution = Solution()

    assert 3 == solution.firstMissingPositive([1, 2, 0])
    assert 2 == solution.firstMissingPositive([3, 4, -1, 1])
    assert 1 == solution.firstMissingPositive([7, 8, 9, 11, 12])


from django.shortcuts import render,HttpResponse,redirect
from home.models import Contact_Us , Doctor
from datetime import datetime
from django.contrib import messages
from django.contrib.auth.models import User
from django.contrib.auth import authenticate,login,logout

# Create your views here.
def index (request):
    return render(request, 'index.html')

def contactus (request):
    if (request.method =="POST") :
        name = request.POST.get('name')
        email = request.POST.get('email')
        phone = request.POST.get('phone')
        desc = request.POST.get('desc')
        contact=Contact_Us(
            name=name,
            email=email,
            phone=phone,
            description=desc,
            date=datetime.today()
            )
        contact.save()
        messages.success(request, ' THANK YOU! Your information is received')
    return render(request, 'contactus.html')

def aboutus (request):
    return render(request,'aboutus.html')

def doctors (request):
    doc=Doctor.objects.all()
    return render(request,'doctors.html',{"data":doc})


# -*- coding: utf-8 -*-
"""
Created on Tue Dec 19 16:57:00 2017

@author: Maitreya
"""
import cv2
import numpy as np

def grabcut_func(img1, img2):
	resized_img = cv2.resize(img1, (0, 0), fx=0.2, fy=0.2)
	resized_img1 = cv2.resize(img2, (0, 0), fx=0.2, fy=0.2)
	print ("Image1...")
	mask = np.zeros(resized_img.shape[:2],np.uint8)
	bgdModel = np.zeros((1,65),np.float64)
	fgdModel = np.zeros((1,65),np.float64)
	rect = (10,130,resized_img.shape[1]-20,resized_img.shape[0]-100)
	cv2.grabCut(resized_img,mask,rect,bgdModel,fgdModel,30,cv2.GC_INIT_WITH_RECT)
	mask2 = np.where((mask==2)|(mask==0),0,1).astype('uint8')
	resized_img = resized_img*mask2[:,:,np.newaxis]
	print ("Image1...")
	mask1 = np.zeros(resized_img1.shape[:2],np.uint8)
	bgdModel1 = np.zeros((1,65),np.float64)
	fgdModel1 = np.zeros((1,65),np.float64)
	rect1 = (10,130,resized_img1.shape[1]-20,resized_img1.shape[0]-100)
	cv2.grabCut(resized_img1,mask1,rect1,bgdModel1,fgdModel1,30,cv2.GC_INIT_WITH_RECT)
	mask21 = np.where((mask1==2)|(mask1==0),0,1).astype('uint8')
	resized_img1 = resized_img1*mask21[:,:,np.newaxis]
	return (resized_img, resized_img1)

def hisComparision(img1, img2):
	print ("Histogram Comparision Started...")
	hist1 = cv2.calcHist([img1],[0],None,[256],[0,256])
	hist2 = cv2.calcHist([img2], [0], None, [256], [0,256])
	print ("Histogram Comparision Completed...")
	return cv2.compareHist(hist1,hist2,cv2.HISTCMP_CORREL)

def featureMatching(img1, img2):
	print ("Feature Matching Started...")

	img1 = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)
	img2 = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY)

	orb = cv2.ORB_create()

	kp1, des1 = orb.detectAndCompute(img1, None)
	kp2, des2 = orb.detectAndCompute(img2, None)

	bf = cv2.BFMatcher(cv2.NORM_HAMMING2, crossCheck = True)

	matches = bf.match(des1, des2);
	matches = sorted(matches, key = lambda x:x.distance)

	img3 = cv2.drawMatches(img1, kp1, img2, kp2, matches[:100], (0, 0, 255), flags = 2)
	cv2.imshow("img3", img3)
	cv2.waitKey(0)
	cv2.destroyAllWindows()
	print ("Feature Matching Completed...\nPrinting number of Matches...")
	return len(matches)

img1 = cv2.imread("C:\\Users\\hp\\Desktop\\MadeIT\\TEST\\P3.jpg")
img2 = cv2.imread("C:\\Users\\hp\\Desktop\\MadeIT\\TEST\\Z6.jpg")

#img1 = cv2.imread("C:\\Users\\Madhulika\\Documents\\GitHub\\OCR\\medicines\\P3.jpg")
#img2 = cv2.imread("C:\\Users\\Madhulika\\Documents\\GitHub\\OCR\\medicines\\Z5.jpg")
print ("Foreground Extraction Started...")
(img1, img2) = grabcut_func(img1, img2)
print("Foreground Extraction Complete...")

print (hisComparision(img1, img2))
print (featureMatching(img1, img2))


"""
View: For management info about items.
"""
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.contrib.auth.mixins import LoginRequiredMixin
from django.db.models import Q
from django.http import HttpResponseRedirect
from django.shortcuts import render, get_object_or_404
from django.urls import reverse_lazy, reverse
from django.views.generic import ListView, CreateView

from carts.forms import CartAddItemForm
from catalogue.forms import ItemBasicForm, ItemModelForm
from catalogue.models import ItemModel, CategoryModel

# GET - query param
@login_required(login_url="users:login")
def items_list(request):
    """

    """
    items = ItemModel.objects.filter(pvp__gt=5)  # Select * from item_model; -> [Queryset]
    # render(object request, <name_html>, context-> dict
    user = request.user  # None si no inicia sesion

    return render(request, "catalogue/index.html", {"products": items, "message": "Promociones"})

# POST - CREATE
def create_item(request):
    if request.method == "POST":
        form = ItemBasicForm(request.POST)
        if form.is_valid():  # Valida si los datos ingresados son correctos
            data = form.cleaned_data  # dict
            ItemModel.objects.create(**data)  # Crear el registro en bd
            messages.success(request, "Item creado exitosamente!")
            return HttpResponseRedirect(reverse("catalogue:item_list"))
        else:
            messages.error(request, "Data enviada no es válida!")
    form = ItemBasicForm()
    return render(request, "create.html", {"form": form})

# CRUD - Create , Read , Update , Delete
class ItemViewList(LoginRequiredMixin, ListView):
    """

    """
    model = ItemModel
    template_name = "catalogue/index.html"
    context_object_name = "products"
    redirect_field_name = reverse_lazy("users:login")
    queryset = ItemModel.objects.filter(pvp__gt=5)  # >

class ItemCreateView(CreateView):
    model = ItemModel
    form_class = ItemModelForm
    template_name = "create.html"
    success_url = reverse_lazy("catalogue:item_list")  # devuelve la url recibiendo el name.

# Catalogue
def item_catalogue(request, category_slug: str = None):
    category = None
    qs_categories = CategoryModel.objects.order_by("name")
    items = ItemModel.objects.order_by("name")
    if category_slug:
        category = get_object_or_404(CategoryModel, slug=category_slug)
        items = items.filter(category=category)  # []
    return render(request, "catalogue/list.html",
                  {"category": category, "categories": qs_categories, "items": items})

def item_detail(request, id: int, slug: str):
    item = get_object_or_404(ItemModel, id=id, slug=slug)
    cart_item_from = CartAddItemForm()
    return render(request, "catalogue/detail.html", {"item": item, "cart_item_form": cart_item_from})

# Search
def item_search(request):
    """

    """
    search_param = request.get("search")
    qs_items = ItemModel.objects.filter(
        Q(name__icontains=search_param) or Q(category__name__icontains=search_param)
    )
    return render(request, "catalogue/list.html", {"items": qs_items})


__author__ = 'Marvin Smith'

#  Python Libraries
import random, math, logging

class Exit_Condition(object):

    max_repeats      = None
    repeated_counter = 0
    current_best     = None

    def __init__(self, max_repeats = 100):

        #  Set the max repeat value
        self.max_repeats = max_repeats

    def Check_Exit(self, test_fitness):
        """
        Evaluate the candidate item to see if we have reached the desired exit condition
        :param candidate:
        :return:
        """

        #  Check if we have even started yet
        if self.current_best is None:
            self.current_best     = test_fitness
            self.repeated_counter = 0
            return False

        #  If the candidate and best are equivalent
        elif math.fabs(test_fitness - self.current_best) < 1:
            self.repeated_counter+= 1
            if self.repeated_counter >= self.max_repeats:
                return True
            else:
                return False

        #  Otherwise, they are not the same
        else:
            self.repeated_counter = 0
            if self.current_best < test_fitness:
                logging.debug("warning: replacing the current exit test with an inferior result")
            self.current_best     = test_fitness
            return False

class Genetic_Algorithm(object):

    #  Current Population
    population = None

    #  Fitness Function
    fitness_function = None

    #  Selection Rate
    selection_rate = None

    #  Crossover Algorithm
    crossover_algorithm = None

    #  Mutation Algorithm
    mutation_algorithm = None

    #  Mutation Rate
    mutation_rate = None

    #  Preservation rate
    preservation_rate = None

    #  Exit Object
    exit_object = None

    #----------------------------#
    #-        Constructor       -#
    #----------------------------#
    def __init__(self, population,
                       fitness_function,
                       crossover_algorithm,
                       mutation_algorithm,
                       selection_rate=0.25,
                       mutation_rate=0.02,
                       preservation_rate=0.01,
                       exit_on_repeats=1000):

        self.exit_object = Exit_Condition(exit_on_repeats)

        #  Set the population
        self.population = population

        #  Set the fitness function
        self.fitness_function = fitness_function

        #  Set the selection_rate
        self.selection_rate = selection_rate

        #  Set the crossover algorithm
        self.crossover_algorithm = crossover_algorithm

        #  Set the mutation algorithm
        self.mutation_algorithm = mutation_algorithm

        #  Set the mutation rate
        self.mutation_rate = mutation_rate

        #  Set the preservation rate
        self.preservation_rate = preservation_rate

    def Run_Iterations(self, max_iterations):

        #  Size of the subset
        subset_size = int(self.selection_rate * len(self.population))
        preserve_size = int(self.preservation_rate * len(self.population))
        mutation_size = int(self.mutation_rate * len(self.population))

        #  Population size
        pop_size = len(self.population)

        #  For each iteration
        for i in range(0, max_iterations):

            #  Select Preservation Set
            preservation_list = self.population[0:preserve_size]

            #  Run Selection
            sub_population = self.population[0:subset_size]

            #  Create result population
            self.population = preservation_list + sub_population

            #  Run Crossover
            while len(self.population) < pop_size:

                #  Select two indeces
                idA = random.randint(0, subset_size-1)
                idB = random.randint(0, subset_size-1)

                #  Merge two samples
                samples = self.crossover_algorithm(sub_population[idA],
                                                   sub_population[idB])

                #  Add the samples
                if len(sub_population) < pop_size:
                    self.population.append(samples[0])
                if len(sub_population) < pop_size:
                    self.population.append(samples[1])

            #  Run Mutation
            for x in range(preserve_size,mutation_size):

                #  Select a random index
                idx = random.randint(preserve_size, len(self.population)-1)
                self.mutation_algorithm(self.population[idx])

            #  Recompute fitness
            for x in range(0,len(self.population)):
                self.population[x].Update_Fitness()

            #  Sort
            self.population = sorted(self.population,
                                     key=self.fitness_function)

            #  Exit Condition
            if self.exit_object.Check_Exit(self.population[0].distance) == True:
                logging.debug('Exiting early on exit condition.')
                break

        #  Return the population
        return self.population


from setuptools import setup, find_packages

with open("./README.md") as f:
    long_description = f.read()

setup(
    name='sech3r',
    version='4.9',
    author="Captain Nick Lucifer",
    author_email="naryal2580@gmail.com",
    url="https://github.com/naryal2580/sech3r",
    download_url='https://github.com/naryal2580/sech3r/tarball/master',
    description="HTTP Security Header Checker",
    long_description=long_description,
    long_description_content_type="text/markdown",
    packages=find_packages(),
    py_modules=['sech3r'],
    install_requires=[
                        'stoyled',
                        'docopt'
                    ],
    classifiers=[
         "Programming Language :: Python :: 3",
         'Environment :: Console',
         "License :: OSI Approved :: MIT License",
         'Topic :: Security',
         "Operating System :: OS Independent",
     ],
    python_requires='>=3',
    entry_points={
        'console_scripts': [
            'sech3r = sech3r:run',
        ],
    }
 )


from sqlalchemy import exists, and_
from random import sample
from pathlib import Path
from shutil import copyfile, Error

from .Models import (Experiment, Reviewer, Session, Folder, BlindFolder, Trial,)

def copy_blind_folder(reviewer, blind_folder):
    blind_folder_dir = Path(reviewer.toScore_dir).joinpath(blind_folder.blind_name)
    blind_folder_dir.mkdir()

    for blind_trial in blind_folder.blind_trials:
        trial = Trial.query.get(blind_trial.trial_id)
        blind_trial_dir = str(
            blind_folder_dir.joinpath(
                f"{blind_folder.blind_name}_{blind_trial.blind_trial_num}.mp4"))
        try:
            copyfile(trial.trial_dir, blind_trial_dir)
        except (Error, PermissionError) as err:
            print(f"shutil.copy Error: {err}\n"
                  f"Trial Directory: {trial.trial_dir}\n"
                  f"BlindTrial Directory: {blind_trial_dir}\n")
            continue

def create_blind_folders(experiment_name, reviewer_name, num_files=1):
    experiment = Experiment.get_by_name(experiment_name)
    reviewer_first_name, reviewer_last_name = reviewer_name.split(' ')
    reviewer = Reviewer.query.filter_by(first_name=reviewer_first_name, last_name=reviewer_last_name).first()

    # Folders with no BlindFolders
    all_folders_not_blinded = Folder.query\
        .join(Session, Folder.session_id == Session.session_id)\
        .filter(Session.experiment_id == experiment.experiment_id)\
        .filter(~exists()
                .where(and_(Folder.folder_id == BlindFolder.folder_id)))\
        .all()

    # Select Folders to blind
    folders_to_blind = sample(all_folders_not_blinded, num_files)

    for folder in folders_to_blind:

        blind_folder = folder.create_blind_folder(reviewer)
        copy_blind_folder(reviewer, blind_folder)


from flask import Flask
from flask import request
from flask import make_response
import json

app = Flask(__name__)

_api_deployment_state = {}

def getFile(fileName):
    filePath = "/cara/responses/%s" % fileName
    F = open(filePath, "r")
    resp = make_response(F.read())
    resp.headers['Content-Type'] = 'application/json'
    return resp

@app.route('/')
def index():
    app.logger.info('Slash...')
    return "Hello, CARA!"

@app.route('/datamanagement/a/api/<versionId>/run-deployments', methods=['POST'])
def runDeployment(versionId):
    app.logger.info('run deployment for %s' % versionId)
    data = request.data
    dataDict = json.loads(data)
    app.logger.info("dataDict %s" % dataDict)
    if "DEV" in dataDict['deployment']:
        id = '81'
    else:
        id = '82'

    _api_deployment_state[id] = 0
    return getFile("runDeployment-%s.json" % id)

@app.route('/datamanagement/a/api/<versionId>/deployment-state/<deploymentId>', methods=['GET'])
def deploymentState(versionId, deploymentId):
    app.logger.info('deployment state for %s' % versionId)
    _api_deployment_state[deploymentId] = _api_deployment_state[deploymentId] + 1
    app.logger.info('state of deployment  %s %s' % (deploymentId, _api_deployment_state[deploymentId]))
    return getFile("deploymentState-{0}-{1}.json".format(deploymentId, _api_deployment_state[deploymentId]))

if __name__ == '__main__':
    app.run(debug=True)


from rest_framework.views import APIView
from rest_framework.response import Response
from coin_spider import serializers
from coin_db import get_db_by_channel_source
from django.http import StreamingHttpResponse

class DataExportView(APIView):

    def get_serializer(self, *args, **kwargs):
        return serializers.DataExportSerializer(*args,**kwargs)

    def get(self,request,*args,**kwargs):
        return Response("数据导出接口")

    def post(self,request,*args,**kwargs):
        serializer = self.get_serializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        channel = serializer.data['channel']
        instrument_id = serializer.data['instrument_id']
        crypto_margined = serializer.data['crypto_margined']
        if channel.endswith('swap') and crypto_margined:
            instrument_id = instrument_id.replace('-USDT','-USD')
        start = serializer.data['start']
        end = serializer.data['end']
        source = serializer.data['source']
        db = get_db_by_channel_source(channel,source,instrument_id)
        try:
            result = db.query_csv(instrument_id,start,end)
            resp = StreamingHttpResponse(result)
            start_str = start.strftime('%Y%m%d%H%M%S')
            end_str = end.strftime('%Y%m%d%H%M%S')
            resp["Content-Disposition"] = f'attachment; filename={"_".join([channel,instrument_id,start_str,end_str])}.csv'
            resp["Access-Control-Expose-Headers"] = "Content-Disposition"
            return resp
        except Exception as e:
            return Response({'error':str(e)})


from os import path
import polynomials as p
from datetime import datetime

# Have a log file open for the purposes of showing the encoding process!
try:
    log = open(path.abspath('./logs/process.txt'), 'w')
except:
    log = None
n = 0

def cyclic_encode(code: dict[int, str], generator: list[int]):
    """
    For each binary string in the code dictionary, perform polynomial multiplication.

    Clarification
    --------
    Here, we express polynomials with coefficients in {0, 1} as lists of ints

    e.g: [1,0,1,1,0] is equivalent to 1 + x^2 + x^3

    Process
    --------
    For each binary string in the code dictionary, we say that it represents the
    coefficients of the polynomial m(x). Moreover, we say that the generator coefficients represent g(x).

    To obtain a cyclic linear codeword, we simply do c(x) = m(x)g(x)
    """
    print(f"[{datetime.now()}] Started encoding file into a cyclic linear code")
    global log
    k = len(list(code.values())[0])
    deg = p.deg(generator)

    global n
    n = k + deg

    # Add padding
    generator += [0] * (k - 1)
    cyclic_code = {}

    print(f"n = {n}", file=log)
    print(f"k = {k}", file=log)
    print(f"deg(g(x)) = {deg}\n-------------", file=log)

    # Foreach message, we multiply it by the generator polynomial
    # PADDING: to the right
    for key in code.keys():
        message = [int(bit) for bit in code[key]] + [0]*deg
        codeword = []

        # Loop through each bit (or coefficient) of the message (or polynomial) 'm'
        # Polynomial multiplication in binary form
        print(f"Generating codeword for message: {message}", file=log)
        for i in range(n):
            polynomial_coef = message[i]

            if polynomial_coef == 1:
                gen = p.shift(generator, i)
                print(f"{generator} --shift({i})--> {gen}", file=log)
                if len(codeword) == 0:
                    for g in gen:
                        codeword.append(g)
                else:
                    temp = codeword.copy()
                    for j in range(n):
                        codeword[j] ^= gen[j]
                    print(f"{temp} XOR {gen} = {codeword}", file=log)
        if len(codeword) == 0:
            for _ in range(n):
                codeword.append(0)
        print(f"Codeword: {codeword}", file=log)
        print(file=log)
        codeword = [str(i) for i in codeword]
        cyclic_code[key] = ''.join(codeword)

    print("Generated cyclic linear code", file=log)
    for key in cyclic_code.keys():
        print(f"{key}: {cyclic_code[key]}", file=log)
    return cyclic_code

def encode_file(file: bytes, code: dict[int, str], generator: list[int]) -> str:
    """
    Encodes a file using cyclic linear encoding

    Parameters
    ------
    file: The bytes object representing the file to be encoded
    generator: The coefficients of the generator polynomial

    Returns
    ------
    The encoded binary in string format
    """
    cyclic_code = cyclic_encode(code, generator)
    print(f"[{datetime.now()}] Encoding process finished!")

    encoded_bin = ''
    for b in file:
        encoded_bin = ''.join((encoded_bin, cyclic_code[b]))

    print(f"[{datetime.now()}] File encoded successfully!")
    return encoded_bin

def convert_to_bytes(bins: list[str]) -> tuple[bytes, int]:
    """
    Converts binary string representations to bytes
    """
    k = len(bins[0])
    num_of_bytes = (k + 7) // 8
    byte_list = []
    print(f"[{datetime.now()}] For each binary string of length {k}, generating {num_of_bytes} bytes")
    # Each b is constant length k, same padding is applied foreach b
    for b in bins:
        padded_bin = b.zfill(num_of_bytes * 8)
        byte_list.append(int(padded_bin, 2).to_bytes(num_of_bytes))
    return (b''.join(byte_list), num_of_bytes)


from multiprocessing import Queue, Process
import requests
from datetime import datetime, timedelta
import pandas as pd
import sqlite3
from random import randint
import numpy as np
import requests_cache
from time import sleep
import io
import sys
import time
import random

class announcement():
    def __init__(self, df, today):

        if len(sys.argv) == 0 or not today:
            #print('setting up cache')
            requests_cache.install_cache('estimize_cache')
        self.conn = sqlite3.connect('estimize_data.sqlite', timeout=30)

        self.df = df
        #print(self.df)

        self.get_yahoo()
        self.add_day_if_amc()
        self.get_change()
        self.get_day_change()
        #self.get_rev_surprise()
        self.get_eps_growth_rate()
        self.get_revs_growth_rate()
        self.get_avg_eps_beat_rate()
        self.get_average_ROI()
        self.get_industry()
        self.get_sue()

        self.df = pd.DataFrame(self.df).transpose()
        #print(self.df)

    def add_day_if_amc(self):
        if self.df['Time'] == "AMC":
            date = datetime.strptime(self.df['Date'], "%Y-%m-%d")
            date + timedelta(days=1)
            while date.isoweekday()>=6:
                date + timedelta(days=1)
            self.df['Date'] = str(date).split(" ")[0]

    def get_yahoo(self):
        try:
            url = "http://chart.finance.yahoo.com/table.csv?s={0}&a=12&b=01&c=2010&d=12&e=01&f=2018&g=d&ignore=.csv".format(self.df['Symbol'])
            data = io.StringIO(requests.get(url).text)
            self.yahoo_df = pd.read_csv(data)
        except Exception as e:
            self.yahoo_df = None

    def get_eps_growth_rate(self):
        # (q315-q314)/q315
        self.df['EPS_Growth'] = None
        try:
            prev_qtr = self.df['Qtr'][:2]+str(int(self.df['Qtr'][2:])-1)

            sql = "select `EPS` from estimize_processed where Symbol='%s' and Qtr = '%s'" % (self.df['Symbol'], prev_qtr)
            df = pd.read_sql(sql, self.conn)
            if not df.empty:
                eps_growth_rate = (float(self.df['EPS']) - float(df['EPS'])) / float(df['EPS'])

                self.df['EPS_Growth'] = eps_growth_rate

        except Exception as e:
            print(e)

    def get_revs_growth_rate(self):
        # (q315-q314)/q315
        self.df['Rev_Growth'] = None
        try:
            prev_qtr = self.df['Qtr'][:2]+str(int(self.df['Qtr'][2:])-1)

            sql = "select `Revs` from estimize_processed where Symbol='%s' and Qtr = '%s'" % (self.df['Symbol'], prev_qtr)
            df = pd.read_sql(sql, self.conn)
            if not df.empty:
                eps_growth_rate = (float(self.df['Revs']) - float(df['Revs'])) / float(df['Revs'])

                self.df['Rev_Growth'] = eps_growth_rate
        except Exception as e:
            print("here", e)

    def get_avg_eps_beat_rate(self):
        # average percent beat for last 4 quarters
        self.df['Avg_Beat'] = None
        self.df['Avg_Estimize_Beat'] = None
        try:
            sql = "select `Percent_Beat` from estimize_processed where Symbol='%s' order by Date desc limit 4" % (self.df['Symbol'])
            df = pd.read_sql(sql, self.conn)

            if len(df)>=3:
                avg_beat = sum(df['Percent_Beat'].astype(float))/len(df['Percent_Beat'])

                self.df['Avg_Beat'] = avg_beat

        except Exception as e:
            #print("here 2", e)
            pass

        try:
            sql = "select `Estimize_Percent_Beat` from estimize_processed where Symbol='%s' order by Date desc limit 4" % (self.df['Symbol'])
            df = pd.read_sql(sql, self.conn)

            if len(df)>=3:
                avg_beat = sum(df['Estimize_Percent_Beat'].astype(float))/len(df['Estimize_Percent_Beat'])

                self.df['Avg_Estimize_Beat'] = avg_beat

        except Exception as e:
            #print("here 3", e)
            pass

    def get_average_ROI(self):
        # average return of the last four quarters
        self.df['Avg_ROI'] = None
        try:
            sql = "select `ROI` from estimize_processed where Symbol='%s' order by Date desc limit 4" % (self.df['Symbol'])
            df = pd.read_sql(sql, self.conn)

            if len(df)>=3:

                avg_roi = sum(df['ROI'])/len(df['ROI'])

                self.df['Avg_ROI'] =avg_roi

        except Exception as e:
            #print("here 4", e)
            pass

    def get_industry(self):
        self.df['Industry'] = None
        try:
            html_text = requests.get("http://finviz.com/quote.ashx?t="+self.df['Symbol']).text
            df = pd.read_html(io.StringIO(html_text))[2]

            industry = df[5][6].split(" | ")[0]
            industry_num = None

            if industry=="Basic Materials":
                industry_num = 0
            elif industry=="Conglomerates":
                industry_num = 1
            elif industry=="Consumer Goods":
                industry_num = 2
            elif industry=="Financial":
                industry_num = 3
            elif industry=="Healthcare":
                industry_num = 4
            elif industry=="Industrial Goods":
                industry_num = 5
            elif industry=="Services":
                industry_num = 6
            elif industry=="Technology":
                industry_num = 7
            elif industry=="Utilities":
                industry_num = 8

            self.df['Industry'] = industry_num
        except Exception as e:
            #print("here 5", e)
            pass

    def get_sue(self):
        self.df['SUE'] = None
        self.df['Estimize_SUE'] = None
        try:
            sql = "select Surprise as SUE from estimize_processed where Symbol == '%s' order by Date desc limit 4;" % (self.df['Symbol'])

            df = pd.read_sql(sql, self.conn)

            if len(df)>=3 and sue != 0:
                sue = np.std(df['SUE'].astype('float'));
                sue = (float(self.df['Surprise'])) / sue

                self.df['SUE'] = sue

        except Exception as e:
            #print("here 6", e)
            pass

        try:
            sql = "select Estimize_Surprise as SUE from estimize_processed where Symbol == '%s' order by Date desc limit 4;" % (self.df['Symbol'])

            df = pd.read_sql(sql, self.conn)

            if len(df)>=3:
                sue = np.std(df['SUE'].astype('float'));
                sue = (float(self.df['Estimize_Surprise'])) / sue

                self.df['Estimize_SUE'] = sue

        except Exception as e:
            #print("here 7", e)
            pass

    def get_day_change(self):
        self.df['Before_ROI'] = None

        try:

            start_date = str(self.df['Date']).split(" ")[0]

            start_index = self.yahoo_df[self.yahoo_df['Date']==start_date].index

            open = self.yahoo_df.iloc[start_index]['Open'].values[0]
            close = self.yahoo_df.iloc[start_index+3]['Close'].values[0]

            overnight_roi = (open-close)/close

            self.df['Before_ROI'] = (overnight_roi * 100)
        except Exception as e:
            #print("here 8", e)
            pass

    def get_change(self):
        self.df['ROI'] = None
        self.df['Open'] = None
        self.df['Close'] = None
        try:
            start_date = str(self.df['Date']).split(" ")[0]

            change_df = self.yahoo_df[self.yahoo_df['Date']>=start_date].tail()
            change_df = change_df.reset_index(drop=True)

            open = change_df.ix[4,'Open']
            close = change_df.ix[0,'Close']

            roi = (close - open)/open
            if roi<2 and roi>-2:
                self.df['ROI'] = (roi * 100)
                self.df['Open'] = open
                self.df['Close'] = close
        except Exception as e:
            print("get change exception", e)


"""The handler for training and evaluation."""

import os
from argparse import ArgumentParser

import torch
import wandb
# os.environ["WANDB_MODE"] = "offline"
# wandb.init()
from pytorch_lightning import Trainer
from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint
from pytorch_lightning.callbacks.progress import TQDMProgressBar
from pytorch_lightning.loggers import WandbLogger
from pytorch_lightning.strategies.ddp import DDPStrategy
from pytorch_lightning.utilities import rank_zero_info

from pl_tsp_model import TSPModel
from pl_mis_model import MISModel

torch.cuda.amp.autocast(enabled=True)
torch.cuda.empty_cache()

def arg_parser():
  parser = ArgumentParser(description='Train a Pytorch-Lightning diffusion model on a TSP dataset.')
  parser.add_argument('--task', type=str, required=True)
  parser.add_argument('--storage_path', type=str, required=True)
  parser.add_argument('--validation_split', type=str, default='data/tsp/tsp50_val_concorde.txt')
  parser.add_argument('--test_split', type=str, default='data/tsp/tsp50_test_concorde.txt')
  parser.add_argument('--validation_examples', type=int, default=64)

  parser.add_argument('--num_workers', type=int, default=16)
  parser.add_argument('--fp16', action='store_true')
  parser.add_argument('--use_activation_checkpoint', action='store_true')

  parser.add_argument('--diffusion_schedule', type=str, default='linear')
  parser.add_argument('--diffusion_steps', type=int, default=1000)
  parser.add_argument('--inference_diffusion_steps', type=int, default=1000)
  parser.add_argument('--inference_schedule', type=str, default='linear')
  parser.add_argument('--inference_trick', type=str, default="ddim")
  parser.add_argument('--sequential_sampling', type=int, default=1)
  parser.add_argument('--parallel_sampling', type=int, default=1)

  parser.add_argument('--n_layers', type=int, default=12)
  parser.add_argument('--hidden_dim', type=int, default=256)
  parser.add_argument('--sparse_factor', type=int, default=-1)
  parser.add_argument('--aggregation', type=str, default='sum')
  parser.add_argument('--two_opt_iterations', type=int, default=1000)
  parser.add_argument('--save_numpy_heatmap', action='store_true')

  parser.add_argument('--project_name', type=str, default='tsp_diffusion')
  parser.add_argument('--wandb_entity', type=str, default=None)
  parser.add_argument('--wandb_logger_name', type=str, default=None)
  parser.add_argument("--resume_id", type=str, default=None, help="Resume training on wandb.")
  parser.add_argument('--ckpt_path', type=str, default=None)
  parser.add_argument('--resume_weight_only', action='store_true')

  parser.add_argument('--do_valid_only', action='store_true')
  parser.add_argument('--do_test_only', action='store_true')

  parser.add_argument('--rewrite_ratio', type=float, default=0.25)
  parser.add_argument('--norm', action='store_true')
  parser.add_argument('--rewrite', action='store_true')
  parser.add_argument('--rewrite_steps', type=int, default=3)
  parser.add_argument('--inference_steps', type=int, default=5)

  args = parser.parse_args()
  return args

def main(args):
    args.wandb_logger_name += "_norm" if args.norm else ""
    args.wandb_logger_name += "_{}".format(args.rewrite_ratio)
    args.wandb_logger_name += "_{}".format(args.parallel_sampling)
    args.wandb_logger_name += "_{}".format(args.inference_diffusion_steps)

    project_name = args.project_name

    if args.task == 'tsp':
      model_class = TSPModel
      saving_mode = 'min'
    elif args.task == 'mis':
      model_class = MISModel
      saving_mode = 'max'
    else:
        raise NotImplementedError

    model = model_class(param_args=args)
    os.makedirs(os.path.join(args.storage_path, f'models'), exist_ok=True)
    wandb_id = os.getenv("WANDB_RUN_ID") or wandb.util.generate_id()
    wandb_logger = WandbLogger(
      name=args.wandb_logger_name,
      project=project_name,
      entity=args.wandb_entity,
      save_dir=os.path.join(args.storage_path, f'models'),
      id=args.resume_id or wandb_id,
    )
    rank_zero_info(f"Logging to {wandb_logger.save_dir}/{wandb_logger.name}/{wandb_logger.version}")

    checkpoint_callback = ModelCheckpoint(
      monitor='val/solved_cost', mode=saving_mode,
      save_top_k=3, save_last=True,
      dirpath=os.path.join(wandb_logger.save_dir,
                            args.wandb_logger_name,
                            wandb_logger._id,
                            'checkpoints'),
    )
    lr_callback = LearningRateMonitor(logging_interval='step')

    trainer = Trainer(
      accelerator="auto",
      devices=torch.cuda.device_count() if torch.cuda.is_available() else None,
      # devices=None,
      callbacks=[TQDMProgressBar(refresh_rate=20), checkpoint_callback, lr_callback],
      logger=wandb_logger,
      check_val_every_n_epoch=1,
      strategy=DDPStrategy(static_graph=True),
      precision=16 if args.fp16 else 32,
      inference_mode=False
    )

    rank_zero_info(
      f"{'-' * 100}\n"
      f"{str(model.model)}\n"
      f"{'-' * 100}\n"
    )

    ckpt_path = args.ckpt_path

    if args.do_valid_only:
      trainer.validate(model, ckpt_path=ckpt_path)
    elif args.do_test_only:
      trainer.test(model, ckpt_path=ckpt_path)
    else:
      trainer.validate(model, ckpt_path=ckpt_path)
      trainer.test(model, ckpt_path=ckpt_path)
    trainer.logger.finalize("success")

if __name__ == '__main__':
    args = arg_parser()
    main(args)


from dataclasses import dataclass
from datetime import datetime, timedelta
import os
import pandas as pd
from pymongo.database import Database
from DbConnector import DbConnector
from pathlib import Path
from collections import OrderedDict, defaultdict
from pprint import pprint
from statistics import mean
from tabulate import tabulate
from haversine import haversine
from heapq import nlargest
import sys

@dataclass
class User:
    _id: str
    has_labels: bool
    activities: list

@dataclass
class Activity:
    _id: int
    user_id: str
    transportation_mode: str
    start_date_time: datetime
    end_date_time: datetime
    # trackpoints: list

@dataclass
class TrackPoint:
    _id: int
    latitude: float
    longitude: float
    altitude: int
    date_days: float
    date_time: datetime
    activity_id: int

class Database_instance:
    def __init__(self):
        self.connection = DbConnector()
        self.client = self.connection.client
        self.db = self.connection.db

    def create_collection(self, collection_name):
        collection = self.db.create_collection(collection_name)
        print('Created collection: ', collection)

    def insert_documents(self, collection_name: str, documents: list):
        """Insert array of documents

        Args:
            collection_name (str): Name of collection
            documents (list): List of documents on json/dict format
        """
        collection = self.db[collection_name]
        collection.insert_many(documents)

    def fetch_documents(self, collection_name: str, criteria={}):
        """Fetch all documents or by criteria

        Args:
            collection_name (str): Name of collection
            criteria (dict, optional): Criteria for find. Defaults to {}.
        """
        collection = self.db[collection_name]
        documents = collection.find(criteria)
        for doc in documents:
            pprint(doc)

    def drop_collection(self, collection_name: str):
        collection = self.db[collection_name]
        collection.drop()

    def show_collections(self):
        collections = self.client[self.db.name].list_collection_names()
        print(collections)

    def _fill_with_data(self):
        db_users: list(User) = []
        db_activities: list(Activity) = []
        db_trackpoints: list(TrackPoint) = []
        base_path = Path(__file__).parents[1]

        # Find all labeled users
        labeled_users = set()
        with open(base_path / "./dataset/dataset/labeled_ids.txt", "r") as f:
            for line in f.readlines():
                labeled_users.add(line.strip())

        # Find all paths to users trackings
        data_folder_path = os.path.join(base_path, "./dataset/dataset/Data")
        activity_files = defaultdict(list)
        for plt_path in Path(data_folder_path).rglob("**/*/*.plt"):
            activity_files[str(os.path.normpath(plt_path)).split(os.sep)[-3]].append(plt_path)

        activity_id = 0
        trackpoint_id = 0
        for key, value in sorted(activity_files.items()):
            is_labeled = True if key in labeled_users else False
            user_activities = []
            if is_labeled:
                labels_df = pd.read_csv(os.path.join(base_path, f"./dataset/dataset/Data/{key}/labels.txt"),
                                        sep="\t", header=1, names=["Start Time", "End Time", "Transportation Mode"])

            # For paths in users data-folder, each plt_path is a .plt file with TrackPoints
            i = 0
            for plt_path in sorted(value):
                n_trackpoints = (sum(1 for _ in open(plt_path)) - 6)  # Ignoring 6 rows of metadata at the top
                if n_trackpoints <= 2500:
                    trackpoints = OrderedDict()
                    with open(plt_path, "r") as f:
                        for index, row in enumerate(f.readlines()[6:]):
                            split_row = row.strip().split(",")
                            helper = [activity_id] + [split_row[k] for k in range(len(split_row) - 2) if k != 2]
                            date_time_str = ",".join([split_row[5], split_row[6]])
                            helper.append(date_time_str)
                            trackpoints[index] = helper

                    plt_start_datetime = datetime.strptime(trackpoints[0][5], '%Y-%m-%d,%H:%M:%S')
                    plt_end_datetime = datetime.strptime(trackpoints[len(trackpoints) - 1][5], '%Y-%m-%d,%H:%M:%S')

                    # Check if a label exactly matches the beginning and ending times of an activity.
                    # If not, don't label the activity.
                    activity_label = None
                    if is_labeled and i < len(labels_df.index):
                        label_start_datetime = datetime.strptime(labels_df["Start Time"].iloc[i], '%Y/%m/%d %H:%M:%S')
                        while plt_start_datetime > label_start_datetime and i < (len(labels_df.index) - 1):
                            label_start_datetime = datetime.strptime(labels_df["Start Time"].iloc[i + 1], '%Y/%m/%d %H:%M:%S')
                            i += 1

                        label_end_datetime = datetime.strptime(labels_df["End Time"].iloc[i], '%Y/%m/%d %H:%M:%S')
                        if label_start_datetime == plt_start_datetime and label_end_datetime == plt_end_datetime:
                            activity_label = labels_df["Transportation Mode"].iloc[i]

                    db_activities.append(Activity(activity_id, key, activity_label, plt_start_datetime, plt_end_datetime).__dict__)
                    user_activities.append(activity_id)

                    for row in trackpoints.values():
                        db_trackpoints.append(TrackPoint(_id=trackpoint_id,
                                                         activity_id=activity_id,
                                                         latitude=row[1],
                                                         longitude=row[2],
                                                         altitude=row[3],
                                                         date_days=row[4],
                                                         date_time=datetime.strptime(row[5], '%Y-%m-%d,%H:%M:%S'),
                                                         ).__dict__)
                        trackpoint_id += 1
                    activity_id += 1
            db_users.append(User(key, is_labeled, user_activities).__dict__)
        self.insert_documents(collection_name="User", documents=db_users)
        self.insert_documents(collection_name="Activity", documents=db_activities)
        self.insert_documents(collection_name="TrackPoint", documents=db_trackpoints)

def tabulate_rows(rows, column_names):
    print(tabulate(rows, headers=column_names, floatfmt=".4f"))

def task_1(db: Database):
    """
    Find the amount of users, db_activities and db_trackpoints.
    """
    # Number of users
    n_users = db.User.count_documents({})

    # Number of activities
    n_activities = db.Activity.count_documents({})

    # Number of trackpoints
    n_trackpoints = db.TrackPoint.count_documents({})

    print("\nTask 1")
    tabulate_rows([[n_users, n_activities, n_trackpoints]], ["n_users", "n_activities", "n_trackpoints"])

def task_2(db: Database):
    """
    Find the average, minimum and maximum number of activities per user.
    """
    # This need the User to have a list of activity_ids, which may be unnecessary
    users: list = db.User.find()
    n_activities = []
    for user in users:
        n_activities.append(len(user["activities"]))
    average = mean(n_activities)
    minimum = min(n_activities)
    maximum = max(n_activities)

    print("\nTask 2")
    tabulate_rows([[average, maximum, minimum]], ["average", "maximum", "minimum"])

def task_2_w_pipeline(db: Database):
    """
    Find the average, minimum and maximum number of activities per user.
    """
    # Should outer left join for including the users without activities
    pipeline = [
        {"$lookup": {
            "from": "Activity",
            "localField": "_id",
            "foreignField": "user_id",
            "as": "activities"},
         },
        {"$group": {"_id": "$_id", "count": {"$sum": {"$size": "$activities"}}}},
        {"$sort": {"_id": 1}}
    ]

    res = db.User.aggregate(pipeline)
    n_activities = []
    for row in res:
        n_activities.append(list(row.values())[1])
    average = mean(n_activities)
    minimum = min(n_activities)
    maximum = max(n_activities)

    print("\nTask 2")
    tabulate_rows([[average, maximum, minimum]], ["average", "maximum", "minimum"])

def task_3(db: Database):
    """
    Find the top 10 users with the highest number of activities.
    """
    pipeline = [
        {"$group": {"_id": "$user_id", "count": {"$sum": 1}}},
        {"$sort": {"count": -1}},
        {"$limit": 10}
    ]

    count_objects = db.Activity.aggregate(pipeline)
    print("\nTask 3")
    tabulate_rows([(obj["_id"], obj["count"]) for obj in count_objects], ["Id", "Count"])

def task_4(db: Database):
    """
    Find the number of users that have started the activity in one day and ended the activity the next day.
    """
    activities = db.Activity.find({})
    user_ids = set()
    for activity in activities:
        if (activity["end_date_time"].date() - activity["start_date_time"].date()) == timedelta(days=1):
            user_ids.add(activity["user_id"])
    print("\nTask 4")
    tabulate_rows([[len(user_ids)]], ["users with multi-day activities"])

def task_5(db: Database):
    """
    Find activities that are registered multiple times. You should find the query even if you get zero results.
    """
    # This query returns no results, so there are no activities fulfilling the requirements.
    pipeline = [
        {
            "$group": {
                "_id": {
                    "uid": "$user_id",
                    "start": "$start_date_time"
                },
                "count": {"$sum": 1}
            },
        },
        {"$match": {"count": {"$gt": 1}}},  # Find entries that have count greater than 1
    ]

    res = db.Activity.aggregate(pipeline)
    print("\nTask 5")
    tabulate_rows([row for row in res], ["activities registred multiple times"])

def task_6(db: Database):
    """
    An infected person has been at position (lat, lon) (39.97548, 116.33031) at time ‘2008-08-24 15:38:00’.
    Find the user_id(s) which have been close to this person in time and space (pandemic tracking).
    Close is defined as the same minute (60 seconds) and space (100 meters).
    """
    infected_coords = (39.97548, 116.33031)
    infected_time = datetime.strptime("2008-08-24 15:38:00", "%Y-%m-%d %H:%M:%S")

    activities_matching = set()
    trackpoints = db.TrackPoint.find({})

    def check_time_window(d1, d2):
        return True if abs(d2 - d1).total_seconds() <= 60 else False

    def check_distance(cord1, cord2):
        dist = haversine(cord1, cord2)
        closeby = True if dist <= 100 else False
        return (closeby, dist)

    for tp in trackpoints:
        if check_time_window(tp["date_time"], infected_time):
            coords = (float(tp["latitude"]), float(tp["longitude"]))
            close, _ = check_distance(coords, infected_coords)
            if close:
                activities_matching.add(tp["activity_id"])

    activities = db.Activity.find({"_id": {"$in": list(activities_matching)}})
    users_in_contact = [a["user_id"] for a in activities]
    print("\nTask 6")
    print(f"Users in contact: {users_in_contact}")

def task_7(db: Database):
    """
    Find all users that have never taken a taxi.
    """
    taxi_activities = db.Activity.distinct("user_id", {"transportation_mode": "taxi"})
    users_not_taxi = list(db.User.find({"_id": {"$nin": taxi_activities}}))

    print("\nTask 7")
    print("users: ", len(users_not_taxi))
    tabulate_rows(sorted([(row["_id"], row["has_labels"]) for row in users_not_taxi],
                         key=lambda x: -int(x[1])), ["Users that have never taken a taxi", "labeled"])

def task_8(db: Database):
    """
    Find all types of transportation modes and count how many distinct users that have used the different transportation modes.
    Do not count the rows where the transportation mode is null.
    """
    activities = db.Activity.find({})
    transportation_res = defaultdict(set)
    for activity in activities:
        transportation_res[activity["transportation_mode"]].add(activity["user_id"])
    del transportation_res[None]
    print("\nTask 8")
    tabulate_rows([(mode, len(count)) for mode, count in sorted(transportation_res.items())], ["transportation_mode", "Count"])

def task_9(db: Database):
    """
    a) Find the year and month with the most activities.
    b) Which user had the most activities this year and month, and how many recorded hours do they have?
        Do they have more hours recorded than the user with the second most activities?
    """
    # a
    pipeline = [
        {
            "$group": {
                "_id": {
                    "y": {"$year": "$start_date_time"},
                    "m": {"$month": "$start_date_time"}
                },
                "count": {"$sum": 1},
            },
        },
        {"$sort": {"count": -1}},
        {"$limit": 1}
    ]

    res = db.Activity.aggregate(pipeline)
    print("\nTask 9a")
    tabulate_rows([[row["_id"]["y"], row["_id"]["m"]] for row in res], ["Year with most activities", "Month with most activities in this year"])

    # b
    p = [
        {
            "$group": {
                "_id": {
                    "uid": "$user_id",
                    "y": {"$year": "$start_date_time"},
                    "m": {"$month": "$start_date_time"}
                },
                "count": {"$sum": 1},
                "timeSpent": {
                    "$sum": {
                        "$dateDiff": {
                            "endDate": "$end_date_time",
                            "startDate": "$start_date_time",
                            "unit": "second"
                        }
                    }
                },
            }
        },
        {"$sort": {"count": -1}},
    ]

    res = db.Activity.aggregate(p)
    top_2 = []
    while len(top_2) < 2:
        el = next(res)
        if el != None:
            if el["_id"]["y"] == 2008 and el["_id"]["m"] == 11:
                el["timeSpent"] = el["timeSpent"] / 3600
                top_2.append(el)

    print("\nTask 9b")
    tabulate_rows([[row["_id"]["uid"], row["count"], row["timeSpent"]] for row in top_2], [
                  "uid", "n_activities", "hours"])

def task_10(db: Database):
    """
    Find the total distance (in km) walked in 2008, by user with id = 112.
    """
    start_2008 = datetime(2008, 1, 1)
    end_2008 = datetime(2008, 12, 31, 23, 59, 59)
    activities_112_walk_2008 = db.Activity.find({"user_id": "112", "transportation_mode": "walk", "start_date_time": {"$gte": start_2008, "$lte": end_2008}})
    trackpoints = db.TrackPoint.find({"activity_id": {'$in': [activity["_id"] for activity in activities_112_walk_2008]}})

    dist = 0
    prev_tp = trackpoints[0]
    for tp in trackpoints[1:]:
        if tp["activity_id"] != prev_tp["activity_id"]:
            continue
        prev_coords = (float(prev_tp["latitude"]), float(prev_tp["longitude"]))
        new_coords = (float(tp["latitude"]), float(tp["longitude"]))
        dist += haversine(prev_coords, new_coords)
        prev_tp = tp
    print("\nTask 10")
    print(f"User_id: 112\nDistance in 2008: {dist}")

def task_11(db: Database):
    """
    Find the top 20 users who have gained the most altitude meters.
        ○ Output should be a table with (id, total meters gained per user).
        ○ Remember that some altitude - values are invalid
        ○ Tip: ∑(tp_{n}.altitude − tp_{n−1}.altitude), tp_{n}.altitude > tp_{n−1}.altitude
    """
    # Short helper function to sum altitude gains
    def gain_alt(alt1, alt2):
        # return (alt2 - alt1) if alt1 < alt2 else 0
        return (alt2 - alt1) * 0.3048 if alt1 < alt2 else 0 # Convert from feet to metres

    num_users = db.User.count_documents({})

    trackpoints = db.TrackPoint.aggregate([
        {"$match": {"altitude": {"$ne": -777}}},
        {"$sort": {"activity_id": 1, "_id": 1}}
    ], allowDiskUse=True)

    activities = defaultdict(dict)
    activities_from_db = db.Activity.find({})
    for a in activities_from_db:
        activities[a["_id"]] = a

    trackpoints = list(trackpoints)

    alt_per_user = [0] * num_users
    prev_tp = trackpoints[0]
    for tp in trackpoints[1:]:
        # Ensure same activity and user
        uid = int(activities[tp["activity_id"]]["user_id"])
        if tp["activity_id"] != prev_tp["activity_id"] or uid != int(activities[prev_tp["activity_id"]]["user_id"]):
            prev_tp = tp
            continue

        alt_gain = gain_alt(float(prev_tp["altitude"]), float(tp["altitude"]))
        alt_per_user[uid] += alt_gain
        prev_tp = tp

    n_highest_gainers = nlargest(20, enumerate(alt_per_user), key=lambda x: x[1])
    print("\nTask 11")
    tabulate_rows(n_highest_gainers, ["uid", "m gained"])

    return n_highest_gainers

def task_12(db: Database):
    """
    Find all users that have invalid activities, and the number of invalid activities per user
        ○ An invalid activity is defined as an activity with consecutive trackpoints
            where the timestamps deviate with at least 5 minutes.
    """
    n_users = db.User.count_documents({})
    trackpoints_activity_fields = db.TrackPoint.aggregate([
        {"$lookup": {
            "from": "Activity",
            "localField": "activity_id",
            "foreignField": "_id",
            "pipeline": [
                {"$project": {"id": 1, "user_id": 1}}
            ],
            "as": "activity_fields"}
         },
        {"$sort": {"activity_id": 1, "date_time": 1}
         },
    ], allowDiskUse=True)
    trackpoints = list(trackpoints_activity_fields)
    invalid_per_user = [0] * n_users

    invalid = False
    prev_tp = trackpoints[0]
    for tp in trackpoints[1:]:
        # Ensure same acitivity and user
        if invalid:
            if tp["activity_fields"][0]["_id"] != prev_tp["activity_fields"][0]["_id"]:
                invalid = False
                prev_tp = tp
            continue

        if tp["activity_fields"][0]["_id"] != prev_tp["activity_fields"][0]["_id"] or tp["activity_fields"][0]["user_id"] != prev_tp["activity_fields"][0]["user_id"]:
            prev_tp = tp
            continue

        time_diff = (tp["date_time"] - prev_tp["date_time"]).total_seconds()
        if time_diff >= 300:
            invalid = True
            invalid_per_user[int(tp["activity_fields"][0]["user_id"])] += 1

        prev_tp = tp

    print("\nTask 12")
    tabulate_rows([(i, invalid_per_user[i]) for i in range(n_users)], ["uid", "Invalid activities"])

def init_db(instance: Database_instance):
    collections = ["User", "Activity", "TrackPoint"]

    for collection in collections:
        instance.drop_collection(collection_name=collection)
        instance.create_collection(collection_name=collection)

    instance.show_collections()
    instance._fill_with_data()

def task_0(db: Database):
    """
    This prints out the first 10 documents in each collection for part 1
    """
    rows_user = [f"\u007b'_id': {x['_id']}, 'has_labels': {x['has_labels']}, 'activities': [...]\u007d" for x in db.User.find({}).limit(10)]
    rows_activity = [x for x in db.Activity.find({}).limit(10)]
    rows_trackpoint = [x for x in db.TrackPoint.find({}).limit(10)]
    print("\nUser:")
    for row in rows_user:
        print(row)
    print("\nActivity:")
    for row in rows_activity:
        print(row)
    print("\nTrackPoint:")
    for row in rows_trackpoint:
        print(row)

if __name__ == '__main__':
    db_instance = None
    try:
        while len(sys.argv) == 1:
            ans = input("Which task you want to run? [i: init db, a: all, 1..12: task 1..12]: ")
            if ans == "a" or ans == "i" or ans.isdigit() and int(ans) in range(1, 13):
                sys.argv.append(ans)

        db_instance = Database_instance()
        tasks = [task_1,
                 task_2,
                 task_3,
                 task_4,
                 task_5,
                 task_6,
                 task_7,
                 task_8,
                 task_9,
                 task_10,
                 task_11,
                 task_12,
                 task_0]

        # db_instance.show_collections()
        # Run tasks
        if sys.argv[1] == "i":
            init_db(db_instance)
        elif sys.argv[1] == "a":
            for task in tasks:
                task(db_instance.db)
        elif int(sys.argv[1]) in range(0, 13):
            tasks[int(sys.argv[1]) - 1](db_instance.db)
        else:
            print(f"Wrong task argument, argv = {sys.argv}")

    finally:
        if db_instance:
            db_instance.connection.close_connection()

TDT4225 - Very Large, Distributed Data Volumes/Assignment 3 w:Jonas 25%/src/GeoLifeProgram.py


"""
Created on 2020/6/11 18:41 周四
@author: Matt zhuhan1401@126.com
Description: description
"""

from torch.utils.data import DataLoader, Dataset
from torchvision import datasets
from torchvision.transforms import transforms

def build_data_loader(cfg, name='MNIST', mode='train'):
    if name == 'MNIST':
        if mode == 'train':
            dataset = datasets.MNIST(root=r'F:\PythonCode\data\MNIST',
                                     train=True,
                                     transform=transforms.ToTensor(),
                                     download=False)
        else:
            dataset = datasets.MNIST(root=r'F:\PythonCode\data\MNIST',
                                     train=False,
                                     transform=transforms.ToTensor(),
                                     download=False)

    elif name == 'CIFAR10':
        if mode == 'train':
            dataset = datasets.CIFAR10(root=r'F:\PythonCode\data',
                                     train=True,
                                     transform=transforms.ToTensor(),
                                     download=False)
        else:
            dataset = datasets.CIFAR10(root=r'F:\PythonCode\data',
                                     train=False,
                                     transform=transforms.ToTensor(),
                                     download=False)

    batch_size = cfg.GLOBAL.TRAIN.BATCH_SIZE if mode == 'train' else cfg.GLOBAL.TEST.BATCH_SIZE

    data_loader = DataLoader(dataset,
                             batch_size=batch_size,
                             shuffle=False) # mode=='train'

    return data_loader

if __name__ == '__main__':
    import argparse
    from utils import config

    def get_parser():
        parser = argparse.ArgumentParser(description='PyTorch Point Cloud Semantic Segmentation')
        parser.add_argument('--config_file', type=str,
                            default='F:\PythonCode\ModelZoo-2D\configs\default.yaml', help='config file')
        parser.add_argument('opts', help='see config/s3dis/s3dis_pointweb_win.yaml for all options', default=None,
                            nargs=argparse.REMAINDER)
        args = parser.parse_args()
        assert args.config_file is not None
        cfg = config.load_cfg_from_file(args.config_file)
        if args.opts is not None:
            cfg.merge_from_list(args.opts)
        return cfg


import socket
import sys

server = ('localhost', 12345)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(20)
socks = [sys.stdin, s]

try:
    s.connect(server)
except:
    print('Unable to connect')
    sys.exit()

print('Connected to the server')
while True:
    for sock in socks:                 # цикл поочередно
        if sock == s:                  # получает и отправляет
            try:                       # сообщения
                data = sock.recv(1024)
            except:
                print('No new msgs')
                continue
            print(data.decode('utf-8'))
        else:
            msg = input('Input: ')
            s.send(msg.encode('utf-8'))