"""
Grid Search
===========
"""
from __future__ import annotations

import itertools
import logging

import numpy

from orion.algo.base import BaseAlgorithm
from orion.algo.space import Categorical, Dimension, Fidelity, Integer, Real, Space
from orion.core.utils import format_trials

log = logging.getLogger(__name__)

def grid(dim: Dimension, num: int):
    """Build a one-dim grid of num points"""

    if dim.type == "categorical":
        # NOTE: Following lines have type errors, because we check the type using the `type`
        # attribute rather than using `isinstance`. This is the right thing to do, since the
        # TransformedDimension subclasses wouldn't be handled correctly using `isinstance`.
        # TODO: Would be nice for the different TransformedSpace subclasses to be generics w.r.t.
        # the type of the wrapped space. This way, we could annotate the functions below with e.g.
        # `Integer | TransformedSpace[Integer]`.
        return categorical_grid(dim, num)  # type: ignore
    elif dim.type == "integer":
        return discrete_grid(dim, num)  # type: ignore
    elif dim.type == "real":
        return real_grid(dim, num)  # type: ignore
    elif dim.type == "fidelity":
        return fidelity_grid(dim, num)  # type: ignore
    else:
        raise TypeError(
            "Grid Search only supports `real`, `integer`, `categorical` and `fidelity`: "
            f"`{dim.type}`\n"
            "For more information on dimension types, see "
            "https://orion.readthedocs.io/en/stable/user/searchspace.html"
        )

# pylint: disable=unused-argument
def fidelity_grid(dim: Fidelity, num: int):
    """Build fidelity grid, that is, only top value"""
    return [dim.interval()[1]]

def categorical_grid(dim: Categorical, num: int):
    """Build categorical grid, that is, all categories"""
    categories = dim.interval()
    if len(categories) != num:
        log.warning(
            f"Categorical dimension {dim.name} does not have {num} choices: {categories}. "
            "Will use {len(categories)} choices instead."
        )
    return categories

def discrete_grid(dim: Integer, num: int):
    """Build discretized real grid"""
    grid = real_grid(dim, num)

    _, b = dim.interval()

    discrete_grid = [int(numpy.round(grid[0]))]
    for v in grid[1:]:
        int_v = int(numpy.round(v))
        if int_v <= discrete_grid[-1]:
            int_v = discrete_grid[-1] + 1

        if int_v > b:
            log.warning(
                f"Cannot list {num} discrete values for {dim.name}. "
                "Will use {len(discrete_grid)} points instead."
            )
            break

        discrete_grid.append(int_v)

    return discrete_grid

def real_grid(dim: Real, num: int):
    """Build real grid"""
    if dim.prior_name.endswith("reciprocal"):
        a, b = dim.interval()
        return list(_log_grid(a, b, num))
    elif dim.prior_name.endswith("uniform"):
        a, b = dim.interval()
        return list(_lin_grid(a, b, num))
    else:
        raise TypeError(
            "Grid Search only supports `loguniform`, `uniform` and `choices`: "
            f"`{dim.prior_name}`"
        )

def _log_grid(a, b, num: int) -> numpy.ndarray:
    return numpy.exp(_lin_grid(numpy.log(a), numpy.log(b), num))

def _lin_grid(a, b, num: int) -> numpy.ndarray:
    return numpy.linspace(a, b, num=num)

class GridSearch(BaseAlgorithm):
    """Grid Search algorithm

    Parameters
    ----------
    n_values: int or dict
        Number of trials for each dimensions, or dictionary specifying number of trials for each
        dimension independently (name, n_values). For categorical dimensions, n_values will not be
        used, and all categories will be used to build the grid.
    """

    deterministic = True
    requires_type = None
    requires_dist = None
    requires_shape = "flattened"

    def __init__(
        self,
        space: Space,
        n_values: int | dict[str, int] = 100,
    ):
        super().__init__(space)

        # pylint: disable=invalid-name
        self.n = 0
        self.n_values = n_values
        n_values_dict = (
            {name: n_values for name in self.space.keys()}
            if not isinstance(n_values, dict)
            else n_values
        )
        max_trials = 10_000 if self.max_trials is None else self.max_trials
        self.grid = self.build_grid(
            self.space,
            n_values_dict,
            max_trials=max_trials,
        )
        self.index = 0

    @staticmethod
    def build_grid(space: Space, n_values: dict[str, int], max_trials: int = 10000):
        """Build a grid of trials

        Parameters
        ----------
        n_values: int or dict
            Dictionary specifying number of trials for each dimension independently
            (name, n_values). For categorical dimensions, n_values will not be used, and all
            categories will be used to build the grid.
        max_trials: int
            Maximum number of trials for the grid. If n_values lead to more trials than max_trials,
            the n_values will be adjusted down. Will raise ValueError if it is impossible to build
            a grid smaller than max_trials (for instance if choices are too large).

        """
        adjust = 0
        n_trials = float("inf")
        coordinates: list[list] = []

        while n_trials > max_trials:
            coordinates = []
            capped_values = []
            for name, dim in space.items():
                capped_value = max(n_values[name] - adjust, 1)
                capped_values.append(capped_value)
                coordinates.append(list(grid(dim, capped_value)))

            if all(value <= 1 for value in capped_values):
                raise ValueError(
                    f"Cannot build a grid smaller than {max_trials}. "
                    "Try reducing the number of choices in categorical dimensions."
                )

            n_trials = numpy.prod([len(dim_values) for dim_values in coordinates])
            # TODO: Use binary search instead of incrementing by one.
            adjust += 1

        if adjust > 1:
            log.warning(
                f"`n_values` reduced by {adjust-1} to limit number of trials below {max_trials}."
            )

        return list(itertools.product(*coordinates))

    @property
    def state_dict(self) -> dict:
        """Return a state dict that can be used to reset the state of the algorithm."""
        state_dict = super().state_dict
        state_dict["grid"] = self.grid
        state_dict["index"] = self.index
        return state_dict

    def set_state(self, state_dict: dict) -> None:
        """Reset the state of the algorithm based on the given state_dict

        Parameters
        ----------
        state_dict: dict
            Dictionary representing state of an algorithm
        """
        super().set_state(state_dict)
        self.grid = state_dict["grid"]
        self.index = state_dict["index"]

    def suggest(self, num):
        """Return the entire grid of suggestions

        Returns
        -------
        list of trials or None
            A list of lists representing trials suggested by the algorithm. The algorithm may opt
            out if it cannot make a good suggestion at the moment (it may be waiting for other
            trials to complete), in which case it will return None.

        """
        trials = []
        while len(trials) < num and self.index < len(self.grid):
            trial = format_trials.tuple_to_trial(self.grid[self.index], self.space)
            if not self.has_suggested(trial):
                self.register(trial)
                trials.append(trial)
            self.index += 1

        return trials

    @property
    def is_done(self):
        """Return True when all grid has been covered."""
        # NOTE: GridSearch doesn't care about the space cardinality, it can just check if the grid
        # has been completely explored.
        return (
            self.has_completed_max_trials
            or self.grid is not None
            and self.n_suggested >= len(self.grid)
        )

    @property
    def configuration(self):
        """Return tunable elements of this algorithm in a dictionary form
        appropriate for saving.

        """
        # NOTE: Override parent method to ignore `seed`
        return {self.__class__.__name__.lower(): {"n_values": self.n_values}}


def binary_search(nums, target):
    idx = len(nums)//2
    return_idx = idx
    # if len(nums) == 1 and nums[0] == target:
    #     return 0
    if len(nums) == [2]:
        if nums[0] == target:
            return 0
        elif nums[1] == target:
            return 1
        else:
            return -1
    while idx >= 0 and idx < len(nums):
        if len(nums) == 1 and nums[0] != target:
            return -1
        a =nums[idx]
        if target > nums[idx]:
            nums =  nums[idx:]
            idx = len(nums)//2
            return_idx = return_idx + idx
        elif target < nums[idx]:
            nums = nums[:idx]
            idx = len(nums)//2
            return_idx = return_idx - idx
        else:
            return return_idx
    return -1

nums =  [-1,0,3,5,9,12]

target = 0
print(binary_search(nums, target))


def get_sequence_until(n = 1):
  if n <= 0: return False
  if n == 1: return [1]

  sequence = [1, 1]

  for i in range(2, n):
    sequence.append(sequence[i - 2] + sequence[i - 1])

  return sequence

def fib_index(n = 1):
  if n <= 0: return 0

  n1 = n2 = 1

  for _ in range(2, n):
    n1, n2 = n2, n1 + n2

  return n2


import sys
import os
import argparse
import torch
from itertools import product
from torch import nn
from torch.nn import functional as F
from torch.autograd import Variable
from nltk.translate.bleu_score import SmoothingFunction, corpus_bleu
from verb_preprocess import load_data
from arguments import s2s_inter as parseParams
from time import time

class model(nn.Module):
  def __init__(self,args):
    super().__init__()
    self.args = args

    # encoder decoder stuff
    self.encemb = nn.Embedding(args.svsz,args.hsz,padding_idx=0)
    self.enc = nn.LSTM(args.hsz,args.hsz//2,bidirectional=True,num_layers=args.layers,batch_first=True)
    self.decemb = nn.Embedding(args.vsz,args.hsz,padding_idx=0)
    self.dec = nn.LSTM(args.hsz*2,args.hsz,num_layers=args.layers,batch_first=True)
    self.gen = nn.Linear(args.hsz,args.vsz)

    self.vemb = nn.Embedding(args.verbvsz,args.hsz,padding_idx=0)
    self.vdec = nn.LSTM(args.hsz,args.hsz,num_layers=args.layers,batch_first=True)
    self.venc = nn.LSTM(args.hsz,args.hsz,num_layers=args.layers,bidirectional=True,batch_first=True)
    self.vgen = nn.Linear(args.hsz,args.verbvsz)
    self.vlinin = nn.Linear(args.hsz,args.hsz,bias=False)
    self.vlinout = nn.Linear(args.hsz*2,args.hsz,bias=False)

    # attention stuff
    self.linin = nn.Linear(args.hsz,args.hsz,bias=False)
    self.dvlinin = nn.Linear(args.hsz,args.hsz,bias=False)
    self.sm = nn.Softmax()
    self.linout = nn.Linear(args.hsz*2,args.hsz,bias=False)
    self.tanh = nn.Tanh()
    self.drop = nn.Dropout(args.drop)
    self.pretrain = False
    self.beamsize = args.beamsize

  def forward(self,inp,out=None,verbs=None):
    encenc = self.encemb(inp)
    enc,(he,ce) = self.enc(encenc)

    #enc hidden has bidirection so switch those to the features dim
    h = torch.cat([he[0:he.size(0):2], he[1:he.size(0):2]], 2)
    c = torch.cat([ce[0:ce.size(0):2], ce[1:ce.size(0):2]], 2)
    hx = torch.cat([he[0:he.size(0):2], he[1:he.size(0):2]], 2)
    cx = torch.cat([ce[0:ce.size(0):2], ce[1:ce.size(0):2]], 2)

    #decode verbs
    #op = Variable(torch.cuda.FloatTensor(inp.size(0),self.args.hsz).zero_())
    voutputs = []
    vencoding = []

    if verbs is None:
      outp = self.args.vmaxlen
    else:
      outp = verbs.size(1)

    for i in range(outp):
      if i == 0:
        prev = Variable(torch.cuda.LongTensor(inp.size(0),1).fill_(3))
      else:
        if verbs is None:
          prev = self.vgen(op).max(2)
          prev = prev[1]
        else:
          prev = verbs[:,i-1].unsqueeze(1)
        #op = op.squeeze(1)

      decin = self.vemb(prev)
      #dembedding = self.vemb(prev)
      #decin = torch.cat((dembedding.squeeze(1),op),1).unsqueeze(1)
      decout, (h,c) = self.vdec(decin,(h,c))
      #vencoding.append(decout)

      #attend on enc
      q = self.vlinin(decout.squeeze(1)).unsqueeze(2)
      w = torch.bmm(enc,q).squeeze(2)
      w = self.sm(w)
      cc = torch.bmm(w.unsqueeze(1),enc)
      op = torch.cat((cc,decout),2)
      op = self.drop(self.tanh(self.vlinout(op)))
      voutputs.append(self.vgen(op))

    voutputs = torch.cat(voutputs,1)
    if self.pretrain:
      return voutputs

    _, vidxs = torch.max(voutputs,2)
    vembedding = self.vemb(vidxs)
    vencoding, _ = self.vdec(vembedding,(hx,cx))
    h = hx
    c = cx

    '''
    vmask = torch.cuda.ByteTensor(verbs.size(0),verbs.size(1),args.hsz).zero_()
    for i in range(verbs.size(0)):
      for j in range(verbs.size(1)):
        if verbs[i,j].data[0]==0:
          vmask[i,j,:].fill_(1)
    vencoding = torch.cat(vencoding,1)
    vencoding.data.masked_fill_(vmask,0)
    '''

    #decode text
    op = Variable(torch.cuda.FloatTensor(inp.size(0),self.args.hsz).zero_())
    outputs = []

    if out is None:
      outp = self.args.maxlen
    else:
      outp = out.size(1)

    for i in range(outp):
      if i == 0:
        prev = Variable(torch.cuda.LongTensor(inp.size(0),1).fill_(3))
      else:
        if out is None:
          prev = self.gen(op).max(2)
          prev = prev[1]
        else:
          prev = out[:,i-1].unsqueeze(1)
        op = op.squeeze(1)

      dembedding = self.decemb(prev)
      decin = torch.cat((dembedding.squeeze(1),op),1).unsqueeze(1)
      decout, (h,c) = self.dec(decin,(h,c))

      '''
      #attend on enc
      q = self.linin(decout.squeeze(1)).unsqueeze(2)
      w = torch.bmm(enc,q).squeeze(2)
      w = self.sm(w)
      cc = torch.bmm(w.unsqueeze(1),enc)
      '''

      #attend on vencoding
      vq = self.dvlinin(decout.squeeze(1)).unsqueeze(2)
      vw = torch.bmm(vencoding,vq).squeeze(2)
      vw = self.sm(vw)
      vcc = torch.bmm(vw.unsqueeze(1),vencoding)

      #op = torch.cat((cc,decout,vcc),2)
      op = torch.cat((decout,vcc),2)
      op = self.drop(self.tanh(self.linout(op)))
      outputs.append(self.gen(op))

    outputs = torch.cat(outputs,1)
    return outputs

  def beamsearch(self,inp):
    inp = inp.unsqueeze(0)
    encenc = self.encemb(inp)
    enc,(he,ce) = self.enc(encenc)

    #enc hidden has bidirection so switch those to the features dim
    hx = torch.cat([he[0:he.size(0):2], he[1:he.size(0):2]], 2)
    cx = torch.cat([ce[0:ce.size(0):2], ce[1:ce.size(0):2]], 2)
    h = [torch.cat([he[0:he.size(0):2], he[1:he.size(0):2]], 2) for i in range(self.beamsize)]
    c = [torch.cat([ce[0:ce.size(0):2], ce[1:ce.size(0):2]], 2) for i in range(self.beamsize)]
    encbeam = [enc for i in range(self.beamsize)]

    #decode verbs
    #op = Variable(torch.cuda.FloatTensor(inp.size(0),self.args.hsz).zero_())
    op = Variable(torch.FloatTensor(inp.size(0),self.args.hsz).zero_())
    voutputs = []
    vencoding = []

    outp = self.args.vmaxlen

    for i in range(outp):
      if i == 0:
        #prev = Variable(torch.cuda.LongTensor(inp.size(0),1).fill_(3))
        prev = Variable(torch.LongTensor(inp.size(0),1).fill_(3))
      else:
        prev = self.vgen(op).max(2)
        prev = prev[1]
        if prev[0,0].data[0]==1:
          break
        #op = op.squeeze(1)

      #dembedding = self.vemb(prev)
      #decin = torch.cat((dembedding.squeeze(1),op),1).unsqueeze(1)
      decin = self.vemb(prev)
      decout, (hx,cx) = self.vdec(decin,(hx,cx))
      vencoding.append(decout)

      #attend on enc
      q = self.vlinin(decout.squeeze(1)).unsqueeze(2)
      w = torch.bmm(enc,q).squeeze(2)
      w = self.sm(w)
      cc = torch.bmm(w.unsqueeze(1),enc)
      op = torch.cat((cc,decout),2)
      op = self.drop(self.tanh(self.vlinout(op)))
      voutputs.append(self.vgen(op))

    #vencoding_ = torch.cat(vencoding,1)
    #vencoding = [vencoding_ for i in range(self.beamsize)]
    voutputs = torch.cat(voutputs,1)
    _, vidxs = torch.max(voutputs,2)
    verbs = vidxs.data
    vembedding = self.vemb(vidxs)
    vencoding_, _ = self.vdec(vembedding)
    vencoding = [vencoding_ for _ in range(self.beamsize)]

    ops = [Variable(torch.FloatTensor(1,1,self.args.hsz).zero_()) for i in range(self.beamsize)]
    prev = [Variable(torch.LongTensor(1,1).fill_(3)) for i in range(self.beamsize)]
    #ops = [Variable(torch.cuda.FloatTensor(1,1,self.args.hsz).zero_()) for i in range(self.beamsize)]
    #prev = [Variable(torch.cuda.LongTensor(1,1).fill_(3)) for i in range(self.beamsize)]
    beam = [[] for x in range(self.beamsize)]
    scores = [0]*self.beamsize
    sents = [0]*self.beamsize
    done = []
    donescores = []
    doneattns = []
    attns = [[] for x in range(self.beamsize)]
    for i in range(self.args.maxlen):
      tmp = []
      for j in range(len(beam)):
        dembedding = self.decemb(prev[j])
        decin = torch.cat((dembedding.squeeze(1),ops[j].squeeze(1)),1).unsqueeze(1)
        decout, (hx,cx) = self.dec(decin,(h[j],c[j]))

        '''
        #attend on enc
        q = self.linin(decout.squeeze(1)).unsqueeze(2)
        #q = decout.view(decout.size(0),decout.size(2),decout.size(1))
        w = torch.bmm(enc,q).squeeze(2)
        w = self.sm(w)
        cc = torch.bmm(w.unsqueeze(1),enc)
        '''

        #attend on vencoding
        vq = self.dvlinin(decout.squeeze(1)).unsqueeze(2)
        vw = torch.bmm(vencoding[j],vq).squeeze(2)
        vw = self.sm(vw)
        attns[j].append(vw)
        vcc = torch.bmm(vw.unsqueeze(1),vencoding[j])

        op = torch.cat((decout,vcc),2)
        #op = torch.cat((cc,decout,vcc),2)
        op = self.drop(self.tanh(self.linout(op)))

        op2 = self.gen(op)
        op2 = op2.squeeze()
        probs = F.log_softmax(op2)
        vals, pidx = probs.topk(self.beamsize*2,0)
        vals = vals.squeeze()
        pidx = pidx.squeeze()
        for k in range(self.beamsize):
          tmp.append((vals[k].data[0]+scores[j],pidx[k],j,hx,cx,op))
      tmp.sort(key=lambda x: x[0],reverse=True)
      newbeam = []
      newscore = []
      newsents = []
      newh = []
      newc = []
      newops = []
      newprev = []
      newattns = []
      added = 0
      j = 0
      while added < len(beam):
        v,pidx,beamidx,hx,cx,op = tmp[j]
        pdat = pidx.data[0]
        new = beam[beamidx]+[pdat]
        if pdat in self.punct:
          newsents.append(sents[beamidx]+1)
        else:
          newsents.append(sents[beamidx])
        if pdat == self.endtok or newsents[-1]>4:
          if new not in done:
            done.append(new)
            donescores.append(v)
            doneattns.append(attns[beamidx])
            added += 1
        else:
          if new not in newbeam:
            newbeam.append(new)
            newscore.append(v)
            newh.append(hx)
            newc.append(cx)
            newops.append(op)
            newprev.append(pidx)
            newattns.append(attns[beamidx])
            added += 1
        j+=1
      beam = newbeam
      prev = newprev
      scores = newscore
      sents = newsents
      h = newh
      c = newc
      ops = newops
      attns = newattns
    if len(done)==0:
      done.extend(beam)
      donescores.extend(scores)
      doneattns = attns
    donescores = [x/len(done[i]) for i,x in enumerate(donescores)]
    topscore =  donescores.index(max(donescores))
    return done[topscore], doneattns[topscore], verbs

def validate(M,DS,args):
  data = DS.val_batches
  cc = SmoothingFunction()
  M.eval()
  M = M.cpu()
  refs = []
  hyps = []
  i=0
  print(time())
  for x in data:
    if args.debug:
      sources, targets, verbs = DS.pad_batch(x,targ=False)
    else:
      sources, targets = DS.pad_batch(x,targ=False,v=False)
    sources = Variable(sources.cpu(),volatile=True)
    M.zero_grad()
    logits = []
    attns = []
    inter = []
    for s in sources:
      l,a,v = M.beamsearch(s)
      logits.append(l)
      attns.append(a)
      inter.append(v)
    hyp = [x[:x.index(1)] if 1 in x else x for x in logits]
    hyp = [[DS.vocab[x] for x in y] for y in hyp]
    for b in inter:
      ihyp = [[DS.verb_vocab[x] for x in list(y)] for y in b]
      print(ihyp)
    print(hyp)
    print(targets)
    hyps.extend(hyp)
    refs.extend(targets)
  bleu = corpus_bleu(refs,hyps,emulate_multibleu=True,smoothing_function=cc.method3)
  M = M.cuda()
  M.train()
  with open(args.savestr+"hyps"+args.epoch,'w') as f:
    hyps = [' '.join(x) for x in hyps]
    f.write('\n'.join(hyps))
  try:
    os.stat(args.savestr+"refs")
  except:
    with open(args.savestr+"refs",'w') as f:
      refstr = []
      for r in refs:
        r = [' '.join(x) for x in r]
        refstr.append('\n'.join(r))
      f.write('\n'.join(refstr))
  return bleu

def train(M,DS,args,optimizer):
  data = DS.train_batches
  weights = torch.cuda.FloatTensor(args.vsz).fill_(1)
  weights[0] = 0
  criterion = nn.CrossEntropyLoss(weights)
  trainloss = []
  for x in data:
    sources, targets, verbs = DS.pad_batch(x)
    sources = Variable(sources.cuda())
    targets = Variable(targets.cuda())
    verbs = Variable(verbs,requires_grad=False)
    M.zero_grad()
    logits= M(sources,targets,verbs)
    logits = logits.view(-1,logits.size(2))
    targets = targets.view(-1)

    loss = criterion(logits, targets)
    loss.backward()
    trainloss.append(loss.data.cpu()[0])
    optimizer.step()

    if len(trainloss)%100==99: print(trainloss[-1])
  return sum(trainloss)/len(trainloss)

def pretrain(M,DS,args,optimizer):
  M.pretrain=True
  data = DS.train_batches
  weights2 = torch.cuda.FloatTensor(args.verbvsz).fill_(1)
  weights2[0] = 0
  criterion2 = nn.CrossEntropyLoss(weights2)
  trainloss = []
  for x in data:
    sources, targets,verbs = DS.pad_batch(x)
    sources = Variable(sources)
    targets = Variable(targets)
    verbs = Variable(verbs)
    M.zero_grad()
    vlogits = M(sources,out=None,verbs=verbs)
    vlogits = vlogits.view(-1,vlogits.size(2))
    verbs = verbs.view(-1)
    loss = criterion2(vlogits,verbs)
    loss.backward()
    trainloss.append(loss.data.cpu()[0])
    optimizer.step()

    if len(trainloss)%100==99:
      print(trainloss[-1])
  M.pretrain=False
  return sum(trainloss)/len(trainloss)

def main(args):
  DS = torch.load(args.datafile)
  if args.debug:
    DS.train_batches= DS.train_batches[:2]
    DS.val_batches= DS.train_batches[:2]
    args.epochs = 100
  args.vsz = DS.vsz
  args.svsz = DS.svsz
  args.verbvsz = DS.vvsz
  if args.resume:
    M,optimizer = torch.load(args.resume)
    M.enc.flatten_parameters()
    M.dec.flatten_parameters()
    e = args.resume.split("/")[-1] if "/" in args.resume else args.resume
    e = e.split('_')[0]
    e = int(e)+1
  else:
    M = model(args).cuda()
    optimizer = torch.optim.Adam(M.parameters(), lr=args.lr)
    e=0
  M.endtok = DS.vocab.index("<eos>")
  M.punct = [DS.vocab.index(t) for t in ['.','!','?']]
  print(M)
  print(args.datafile)
  print(args.savestr)
  for epoch in range(args.pretrainepochs):
    trainloss = pretrain(M,DS,args,optimizer)
    print("train loss epoch",epoch,trainloss)
  for epoch in range(e,args.epochs):
    args.epoch = str(epoch)
    trainloss = train(M,DS,args,optimizer)
    print("train loss epoch",epoch,trainloss)
    if args.debug:
      if epoch%2==1:
        b = validate(M,DS,args)
        print("valid bleu ",b)
    else:
      b = validate(M,DS,args)
      print("valid bleu ",b)
      torch.save((M,optimizer),args.savestr+args.epoch+"_bleu-"+str(b))

if __name__=="__main__":
  args = parseParams()
  main(args)


import nltk
from nltk.stem import WordNetLemmatizer
lemmatizer= WordNetLemmatizer()
import pickle
import numpy as np
import pandas as pd
from keras. models import Sequential
from keras.layers import Dense,Activation, Dropout
from keras.optimizers import Adam
import random
import json

files = ['agriculture','culture','economics','health','politics']

for file in files:
    words = []
    tags= []
    documents= []
    ignore_words=['?','!']
    data_file = open(file+'.json').read()
    purposes = json.loads(data_file)

    for purpose in purposes['purpose']:
        for question in purpose['questions']:
            w = nltk.word_tokenize(question)
            words.extend(w)

            documents.append((w,purpose['id_name']))

            if purpose['id_name'] not in tags:
                tags.append(purpose['id_name'])

    words = [lemmatizer.lemmatize(w.lower()) for w in words if w not in ignore_words]
    words = sorted(list(set(words)))
    tags = sorted(list(set(tags)))
    pickle.dump(words,open(file+'_words.pkl','wb'))
    pickle.dump(tags,open(file+'_tags.pkl','wb'))

    training=[]
    output = [0]*len(tags)
    for doc in documents:
        bags=[]
        question = doc[0]
        question = [lemmatizer.lemmatize(w.lower()) for w in question]

        for w in words:
            bags.append(1) if w in question else bags.append(0)

            out = list(output)
            out[tags.index(doc[1])]=1
            training.append([bags,out])

    random.shuffle(training)
    training = np.array(training)
    X_train = list(training[:,0])
    y_train = list(training[:,1])

    model=Sequential()
    model.add(Dense(128,activation="relu",input_shape=(len(X_train[0]),)))
    model.add(Dropout(0.2))
    model.add(Dense(64,activation="relu"))
    model.add(Dropout(0.2))
    model.add(Dense(len(y_train[0]),activation="softmax"))
    adam = Adam(learning_rate=0.01)
    model.compile(optimizer=adam,loss="categorical_crossentropy",metrics=["accuracy"])

    hist = model.fit(np.array(X_train),np.array(y_train),epochs=40,batch_size=5,verbose=1)

    model.save(file+'_model.h5',hist)


#!/usr/bin/env python3 -u
# coding: utf-8

__author__ = ["Markus Löning"]
__all__ = []

import numpy as np
from sktime.utils._testing.series_as_features import \
    make_classification_problem

from benchmarks.utils import ak_3d_arr
from benchmarks.utils import ak_record_arr
from benchmarks.utils import np_3d_arr

def _slice(X):
    return X[10:20, 5:15, 50:60]

def _nested_slice(X):
    x = X.iloc[10:20, 5:15]
    return np.asarray([[x.iloc[i, j].iloc[50:60].to_numpy()
                        for j in range(x.shape[1])]
                       for i in range(x.shape[0])])

X, _ = make_classification_problem(n_instances=100,
                                   n_timepoints=100,
                                   n_columns=20)

expected = _slice(np_3d_arr(X))

def test_ak_3d_slice(benchmark):
    x = ak_3d_arr(X)
    actual = benchmark(_slice, x)
    np.testing.assert_array_equal(actual, expected)

def test_ak_record_slice(benchmark):
    x = ak_record_arr(X)
    actual = benchmark(_slice, x)
    np.testing.assert_array_equal(actual["value"], expected)

def test_np_slice(benchmark):
    x = np_3d_arr(X)
    actual = benchmark(_slice, x)
    np.testing.assert_array_equal(actual, expected)

def test_nested_slice(benchmark):
    actual = benchmark(_nested_slice, X)
    np.testing.assert_array_equal(actual, expected)


"""Main file that processes raw data from the Arduino"""

import serial
from datetime import datetime
import itertools
import time
from constants import *

#arduinoSerial = serial.Serial('COM4', 9600) #For Windows
arduinoSerial = serial.Serial('/dev/ttyACM0', 9600) #For Linux

previous_altitude = 0
previous_time = 0
altitudeBucket = []
altGPSArray = []
heightCalibrated = 0
calibrated = False
localQNH = 0

#If the GPS altitude is similar 20 times in raw. Adapt the barometer to local QNH
def adapt_local_pressure(altGPS,alt):
    global localQNH
    global heightCalibrated
    global calibrated

    altGPSArray.append(altGPS)

    if len(altGPSArray) > 20:
        altGPSArray.pop(0)
        if all(abs(x - y) < 3.5 for x, y in itertools.combinations(altGPSArray, 2) if x != 0.0):
            localQNH = SEALEVELPRESSURE + ((sum(altGPSArray) / len(altGPSArray)) - alt) * METERSOFAIRHETCOPASCAL
            heightCalibrated = round((localQNH - SEALEVELPRESSURE) * HETCOPASCALMETERSOFAIR, 1)
            calibrated = True

#Verical speed calculated from a Delta between time and height
def vertical_speed(alt):
    global previous_altitude, previous_time
    current_time = time.time()
    delta_time = current_time - previous_time
    altitude_difference = alt - previous_altitude
    vertical_speed = altitude_difference / delta_time

    # Update previous values for the next iteration
    previous_altitude = alt
    previous_time = current_time

    vertical_speed = round(vertical_speed, 1)

    # If the vertical speed is too small, set it to 0.0
    if vertical_speed == -0.0 or vertical_speed == 0.1 or vertical_speed == -0.1 or vertical_speed == 0.2 or vertical_speed == -0.2:
        vertical_speed = 0.0
    return vertical_speed

#Transform raw data into JSON
def data():
    #time.sleep(1)
    while arduinoSerial.inWaiting() == 0:
        pass
    dataString = arduinoSerial.readline().decode('utf-8')
    dataArray = dataString.replace('\r\n', '').split(',')

    localtime = datetime.now().strftime("%H:%M:%S.%f")[:-3]
    returnDict = {
        "time": localtime,
        "localQNH": 0,
        "lat": 0,
        "long": 0,
        "speed": 0,
        "altGPS": 0,
        "sat": 0,
        "vert": 0,
        "alt": 0
    }

    if len(dataArray) == 1:
        altitude = float(dataArray[0])
        vert_speed = vertical_speed(altitude)
        if calibrated:
            altitude += heightCalibrated
        returnDict.update({"vert": vert_speed, "alt": altitude, "localQNH":localQNH})
    elif len(dataArray) >= 5:
        latitude = float(dataArray[0])
        longitude = float(dataArray[1])
        speed = float(dataArray[2])
        altitudeGPS = float(dataArray[3])
        satellites = float(dataArray[4])
        altitude = float(dataArray[-1])

        vert_speed = vertical_speed(altitude)

        if not calibrated:
            adapt_local_pressure(altitudeGPS, altitude)

        if calibrated:
            altitude += heightCalibrated

        returnDict.update({
            "lat": latitude,
            "long": longitude,
            "speed": speed,
            "altGPS": altitudeGPS,
            "sat": satellites,
            "vert": vert_speed,
            "alt": altitude,
            "localQNH":localQNH
        })
    print(returnDict)
    return returnDict

#while True:
#    data()


#!/usr/bin/env python3

# file    :  fuzz-webview.py
# repo    :  https://github.com/fuzzah/fuzzaide
# author  :  https://github.com/fuzzah
# license :  MIT
# check repository for more information

"""
fuzz webview: simple Flask App to aggregate and display data from fuzzer_stats files
Code here may partially overlap with fuzzman.py
"""

import os
import sys
import glob
import time
import argparse
from datetime import datetime
from threading import Thread, Lock, Event

from flask import Flask, render_template, send_from_directory

from fuzzaide.common.fuzz_stats import get_afl_stat_name, is_afl_fuzzer_stats_old

class StatsLoader(Thread):
    def __init__(self, dir, lock):
        super().__init__()
        self.dir = dir  # fuzzers sync dir
        self.stats = list()  # list of dicts
        self.common_stats = dict()
        self.lock = lock  # lock for self.stats
        self._stop_evt = Event()

        self.start()

    def get_fuzzer_stats(self, fname):
        """
        Form a dictionary from fuzzer_stats
        """

        try:
            with open(fname, "rt") as f:
                data = f.read()
        except OSError:
            return None

        if data is None:  # file is empty
            return None

        stats = dict()
        for line in data.split("\n"):
            if ":" in line:
                k, v = line.split(":", 1)
                k = k.strip()
                v = v.strip()
                stats[k] = v
        if len(stats) < 1:
            return None

        return stats

    @staticmethod
    def update_stat_timestamp(stats_dict, stat_name, saved_newest_stamp):
        """
        Use this method to update last path (crash, hang, etc) timestamp.
        Example: newest_path_stamp = update_stat_timestamp(stats, "last_path", newest_path_stamp)
        """
        stamp = stats_dict.get(stat_name)

        if stamp is None:
            return saved_newest_stamp

        try:
            stamp = int(stamp)
        except ValueError:
            return saved_newest_stamp

        if stamp > saved_newest_stamp:
            return stamp

        return saved_newest_stamp

    @staticmethod
    def format_seconds(seconds):
        s = seconds % 60
        m = (seconds // 60) % 60
        h = (seconds // 3600) % 24
        d = seconds // 86400

        if d > 0:
            return "%d days, %d hrs, %d min, %d sec" % (d, h, m, s)
        elif h > 0:
            return "%d hrs, %d min, %d sec" % (h, m, s)
        elif m > 0:
            return "%d min, %d sec" % (m, s)

        return "%d sec" % (s,)

    def load_stats(self):
        fuzzer_stats = "fuzzer_stats"
        filenames = glob.glob(os.path.join(self.dir, "*", fuzzer_stats))
        single_fuzzer_stats = os.path.join(self.dir, fuzzer_stats)
        if os.path.isfile(single_fuzzer_stats):
            filenames.append(single_fuzzer_stats)
        filenames = sorted(filenames)

        newest_path_stamp = 0
        newest_hang_stamp = 0
        newest_crash_stamp = 0

        sum_execs = 0
        sum_paths = 0
        sum_hangs = 0
        sum_crashes = 0

        common_stats = {}
        all_stats = []

        now = int(datetime.now().timestamp())

        for fname in filenames:
            stats = self.get_fuzzer_stats(fname)
            if not stats:
                continue

            use_old_style = is_afl_fuzzer_stats_old(stats)

            last_upd = int(
                stats.get(get_afl_stat_name("last_update", use_old_style), 0)
            )
            if last_upd > 0:
                delta = now - last_upd
                mess = self.format_seconds(delta) + " ago"
                if delta > 90:
                    mess += " (NOT RUNNING?)"
                stats["last_update"] = mess
            all_stats.append(stats)

            crashes = int(
                stats.get(get_afl_stat_name("unique_crashes", use_old_style), 0)
            )
            hangs = int(stats.get(get_afl_stat_name("unique_hangs", use_old_style), 0))
            paths_total = int(
                stats.get(get_afl_stat_name("paths_total", use_old_style), 0)
            )

            sum_crashes += crashes
            sum_hangs += hangs
            sum_paths += paths_total
            sum_execs += int(stats.get("execs_done", 0))

            newest_path_stamp = self.update_stat_timestamp(
                stats, get_afl_stat_name("last_path", use_old_style), newest_path_stamp
            )
            newest_hang_stamp = self.update_stat_timestamp(
                stats, get_afl_stat_name("last_hang", use_old_style), newest_hang_stamp
            )
            newest_crash_stamp = self.update_stat_timestamp(
                stats,
                get_afl_stat_name("last_crash", use_old_style),
                newest_crash_stamp,
            )

        if newest_path_stamp == 0:
            return

        e = float(sum_execs)
        c = ""
        if e >= 1000000000:
            e /= 1000000000
            c = "B"
        elif e >= 1000000:
            e /= 1000000
            c = "M"
        elif e >= 1000:
            e /= 1000
            c = "K"

        execs = "Unknown"
        if len(c) > 0:
            if c == "B":
                execs = "%.4f%c" % (e, c)
            else:
                execs = "%.2f%c" % (e, c)
        else:
            execs = "%.0f" % (e,)

        common_stats["execs"] = execs

        newest_path_delta = now - newest_path_stamp
        newest_path_fmt = self.format_seconds(newest_path_delta)

        common_stats["paths"] = sum_paths
        common_stats["last_new_path"] = newest_path_delta
        common_stats["paths_info_fmt"] = "Paths: %d.\tLast new path: %s ago" % (
            sum_paths,
            newest_path_fmt,
        )

        common_stats["hangs"] = sum_hangs

        if sum_hangs > 0:
            delta = now - newest_hang_stamp
            seconds_fmt = self.format_seconds(delta)
            common_stats["last_new_hang"] = delta
            common_stats["hangs_info_fmt"] = "Hangs: %d.\tLast new hang: %s ago" % (
                sum_hangs,
                seconds_fmt,
            )
        else:
            common_stats["last_new_hang"] = -1
            common_stats["hangs_info_fmt"] = "Hangs: 0"

        common_stats["crashes"] = sum_crashes
        if sum_crashes > 0:
            delta = now - newest_crash_stamp
            seconds_fmt = self.format_seconds(delta)
            common_stats["last_new_crash"] = delta
            common_stats[
                "crashes_info_fmt"
            ] = "Crashes: %d.\tLast new crash: %s ago" % (sum_crashes, seconds_fmt)
        else:
            common_stats["last_new_crash"] = -1
            common_stats["crashes_info_fmt"] = "Crashes: 0"

        self.lock.acquire()
        self.stats = all_stats
        self.common_stats = common_stats
        self.lock.release()

    def run(self):
        while True:
            self.load_stats()
            time.sleep(1.0)
            if self._stop_evt.is_set():
                break

    def stop(self):
        self._stop_evt.set()

class WebApp(Flask):
    def __init__(self, appname, syncdir):
        super().__init__(appname)
        self.stats_lock = Lock()
        self.stats_loader = StatsLoader(syncdir, self.stats_lock)

        @self.route("/", methods=["GET"])
        def _index():
            return self.render_index()

        @self.route("/favicon.ico")
        def _favicon():
            return send_from_directory(
                os.path.join(self.root_path, "static"),
                "favicon.ico",
                mimetype="image/x-icon",
            )

    def stop_stats_loader(self):
        self.stats_loader.stop()

    def load_stats(self):
        self.stats_lock.acquire()
        fuzz_stats = self.stats_loader.stats
        common_stats = self.stats_loader.common_stats
        self.stats_lock.release()
        return common_stats, fuzz_stats

    def render_index(self):
        now = datetime.now()
        curtime_fmt = now.strftime(r"%H:%M:%S %d.%m.%Y")
        comm_stats, fuzz_stats = self.load_stats()
        comm_stats["curtime_fmt"] = curtime_fmt
        return render_template(
            "index.html", fuzzer_stats=fuzz_stats, common_stats=comm_stats
        )

def main():
    parser = argparse.ArgumentParser(
        prog=os.path.basename(sys.argv[0]),
        description="%(prog)s - simple dashboard for AFL-like fuzzers",
        epilog="by default, listens on http://127.0.0.1:8080",
    )

    parser.add_argument(
        "-o", "--sync-dir", help="fuzzer sync directory", type=str, required=True
    )
    parser.add_argument(
        "-p", "--port", help="port to bind to", type=int, default="8080"
    )
    parser.add_argument(
        "-l", "--addr", help="address to bind to", default="127.0.0.1", type=str
    )
    parser.add_argument(
        "-v", "--verbose", help="verbose mode", action="count", default=0
    )

    if len(sys.argv) < 2:
        parser.print_help()
        return 0

    args = parser.parse_args()

    print("Using sync dir:", args.sync_dir)
    print("Trying to start web server on http://%s:%s.." % (args.addr, args.port))

    app = WebApp(__name__, args.sync_dir)
    app.run(host=args.addr, port=args.port, debug=args.verbose > 0)
    print("\nLeaving..")
    app.stop_stats_loader()
    return 0

if __name__ == "__main__":
    sys.exit(main())


#Find biggest of 3 numbers entered.
a = int(input("Enter first number : "))
b = int(input("Enter second number : "))
c = int(input("Enter third number : "))
if a>b and a>c:
    print("first number is greatest number")
elif b>c and b>a:
    print("second number is greatest")
elif c>a and c>b:
    print("third number  is greatest")
else:
    print("not defined....!")


import pandas as pd
import state_abbreviations
import re
import os
import requests
import json
from pandas.io.json import json_normalize

dir = os.getcwd() + '/Data Sets/'

def mergeRosterWithGeoCoords(filename):

    team_name = filename.split('_')[0]

    #Import the datasets to be merged
    roster_df = pd.read_csv(dir + filename, "*", header=0)
    cities_df = pd.read_csv("worldcities.csv")

    #Subset the data to only include the USA as retain only the necessary variables for WebMap
    cities_df = cities_df[cities_df['iso3'] == 'USA']
    cities_df = cities_df[['city', 'admin_name', 'lat', 'lng']]

    #Grab the city and state since the High School was included in the variable string
    roster_df['city'] = roster_df['city'].apply(lambda city: city.split('(')[0].strip())

    #In order to merge the two data sets, the states must both be abbrevatiated.  A custom function
    #was created to abbreviate the states.
    cities_df['admin_name'] = cities_df['admin_name'].apply(lambda x: state_abbreviations.abbreviateState(x))

    #Manipulate the city variable in cities_df to match the city variable from the roster_df
    cities_df['city'] = cities_df['city'] + ', ' + cities_df['admin_name']

    #Merge the datasets by city
    new_df = pd.merge(cities_df, roster_df, on="city")
    print(new_df.head(10))

    #Export to csv
    new_df.to_csv(team_name + '_merged.csv')

#mergeRosterWithGeoCoords('lsu_roster.csv')

# The following function concatenates all of the college football teams into one
# main dataset.
def createMasterDataset():
    frames = []
    for file in os.listdir(dir):
        df = pd.read_csv(dir + file, "*", header=0)
        frames.append(df)
    master_df = pd.concat(frames)
    print(master_df)

#createMasterDataset()

def queryAPI(url):
    response = requests.get(url)
    data = response.json()
    df = pd.DataFrame.from_dict(data)
    df.to_csv('college.csv')

queryAPI('https://api.collegefootballdata.com/recruiting/players?year=2019&classification=HighSchool')


# Code adapted from Tensorflow Object Detection Framework
# https://github.com/tensorflow/models/blob/master/research/object_detection/object_detection_tutorial.ipynb
# Tensorflow Object Detection Detector

import logging
import os
import sys
import numpy as np
import tensorflow.compat.v1 as tf
import cv2
import argparse
import smtplib

logging.getLogger('tensorflow').disabled = True
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
tf.disable_v2_behavior()

def parse_command_line():
    parser = argparse.ArgumentParser(add_help=False)
    parser.add_argument('--path', type=str, default=None, required=True)
    args = parser.parse_args()
    return args.path

def alert_user(filename):

    server = smtplib.SMTP('smtp.gmail.com', 587)
    server.ehlo()
    server.starttls()
    myEmail = "farmCamAlert293@gmail.com"
    myEmailPass = "richisawesome"
    destEmail = "gwynethallwright@gmail.com"
    # Log in to the server using user provided credentials
    server.login(myEmail, myEmailPass)
    # Send the mail
    subject = "Human detected"
    body = "The device has detected human activity in video " + filename + "."
    message = ("From: %s\r\n" % myEmail
             + "To: %s\r\n" % destEmail
             + "Subject: %s\r\n" % subject
             + "\r\n"
             + body)
    server.sendmail(myEmail, destEmail, message)
    server.close()

class DetectorAPI:
    def __init__(self, path_to_ckpt):
        self.path_to_ckpt = path_to_ckpt
        self.detection_graph = tf.Graph()
        with self.detection_graph.as_default():
            od_graph_def = tf.GraphDef()
            with tf.gfile.GFile(self.path_to_ckpt, 'rb') as fid:
                serialized_graph = fid.read()
                od_graph_def.ParseFromString(serialized_graph)
                tf.import_graph_def(od_graph_def, name='')
        self.default_graph = self.detection_graph.as_default()
        self.sess = tf.Session(graph=self.detection_graph)
        # Definite input and output Tensors for detection_graph
        self.image_tensor = self.detection_graph.get_tensor_by_name('image_tensor:0')
        # Each box represents a part of the image where a particular object was detected.
        self.detection_boxes = self.detection_graph.get_tensor_by_name('detection_boxes:0')
        # Each score represent how level of confidence for each of the objects.
        # Score is shown on the result image, together with the class label.
        self.detection_scores = self.detection_graph.get_tensor_by_name('detection_scores:0')
        self.detection_classes = self.detection_graph.get_tensor_by_name('detection_classes:0')
        self.num_detections = self.detection_graph.get_tensor_by_name('num_detections:0')

    def processFrame(self, image):
        # Expand dimensions since the trained_model expects images to have shape: [1, None, None, 3]
        image_np_expanded = np.expand_dims(image, axis=0)
        # Actual detection.
        (boxes, scores, classes, num) = self.sess.run(
            [self.detection_boxes, self.detection_scores, self.detection_classes, self.num_detections],
            feed_dict={self.image_tensor: image_np_expanded})
        im_height, im_width,_ = image.shape
        boxes_list = [None for i in range(boxes.shape[1])]
        for i in range(boxes.shape[1]):
            boxes_list[i] = (int(boxes[0,i,0] * im_height),
                        int(boxes[0,i,1]*im_width),
                        int(boxes[0,i,2] * im_height),
                        int(boxes[0,i,3]*im_width))
        return boxes_list, scores[0].tolist(), [int(x) for x in classes[0].tolist()], int(num[0])

    def close(self):
        self.sess.close()
        self.default_graph.close()

if __name__ == "__main__":
    filename = parse_command_line()
    model_path = '/home/ubuntu/ssd_mobilenet_v1_coco_2018_01_28/frozen_inference_graph.pb'
    odapi = DetectorAPI(path_to_ckpt=model_path)
    threshold = 0.4
    humanCountThreshold = 2
    x_dim = 1280
    y_dim = 720
    skip_frames = 1
    videos_path = "./"
    print("Started processing for " + filename)
    cap = cv2.VideoCapture(videos_path + filename)
    num_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
    if num_frames <= 0 :
        print("Video could not be read.")
        sys.exit()
    humanCount = 0
    frame_num = 0
    while True:

        r, img = cap.read()

        if r==False:
            print("Human not found in "+filename)
            break

        img = cv2.resize(img, (x_dim, y_dim))
        boxes, scores, classes, num = odapi.processFrame(img)

        for i in range(len(boxes)):
            # Class 1 represents human
            if classes[i] == 1 and scores[i] > threshold:
                humanCount = humanCount+1
                break

        if humanCount > humanCountThreshold:
            print("Human found in "+filename)
            alert_user(filename)
            sys.exit()


# -*- coding: utf-8 -*-
"""

"""

import cv2 as cv
import numpy as np

class FindingOrientationOfContours(object):

    def FindOrientation(self, contourA, contourB, contourC):
        """Here famous Triangle Method is used to determine the Orientation
        of the three contours and identify which one is top, left and
        right contour.Here we find the distance between the centres of
        mass of these three countours and which ever is longest , base
        of # TODO: he triangle formed by three contours and then we find the
        position of remaining contour using this base line to know
        the orientation"""

        # calculating the centre of mass of three contours
        MomentA = cv.moments(contourA)
        MomentB = cv.moments(contourB)
        MomentC = cv.moments(contourC)

        # finding the centre of mass of three contours
        centreOfMassA_X = int(MomentA['m10'] / MomentA['m00'])
        centreOfMassA_Y = int(MomentA['m01'] / MomentA['m00'])
        PointA = (centreOfMassA_X, centreOfMassA_Y)

        centreOfMassB_X = int(MomentB['m10'] / MomentB['m00'])
        centreOfMassB_Y = int(MomentB['m01'] / MomentB['m00'])
        PointB = (centreOfMassB_X, centreOfMassB_Y)

        centreOfMassC_X = int(MomentC['m10'] / MomentC['m00'])
        centreOfMassC_Y = int(MomentC['m01'] / MomentC['m00'])
        PointC = (centreOfMassC_X, centreOfMassC_Y)

        # finding the distance of the distance between points
        distance_AB = self.__findDistanceBetweenTwoPoints(PointA, PointB)
        distance_BC = self.__findDistanceBetweenTwoPoints(PointB, PointC)
        distance_AC = self.__findDistanceBetweenTwoPoints(PointA, PointC)

        largestLine = np.argmax(
             np.array([distance_AB, distance_BC, distance_AC]))

        if largestLine == 0:
            # largest line is between points A and B
            Right, Bottom, Top = self.findOrientationBetwwenPoints(
                 PointC, PointA, PointB)
            return Right, Bottom, Top

        if largestLine == 1:
            # LargestLine is Between B and C
            Right, Bottom, Top = self.findOrientationBetwwenPoints(
                 PointA, PointB, PointC)
            return Right, Bottom, Top

        if largestLine == 2:
            # LargestLine is between A and C
            Right, Bottom, Top = self.findOrientationBetwwenPoints(
                 PointB, PointA, PointC)
            return Right, Bottom, Top

    def __findDistanceBetweenTwoPoints(self, PointA, PointB):
        return np.sqrt(np.square(
             PointA[0] - PointB[0]) + np.square(PointA[1] - PointB[1]))

    def findOrientationBetwwenPoints(
              self, DistancePoint, SlopePointA, SlopePointB):
        slope, distance = self.CalculatePerpendicularDistance(
             DistancePoint, SlopePointA, SlopePointB)
        Right = None
        Bottom = None
        Top = None

        print (slope, distance)

        if (slope >= 0) and (distance >= 0):
            # if slope and distance are positive A is bottom while B is right
            Right = SlopePointB
            Bottom = SlopePointA
            Top = DistancePoint

        if (slope > 0) and (distance < 0):
            # if slope is positive and distance is negative then B is bottom
            # while A is right
            Right = SlopePointA
            Bottom = SlopePointB
            Top = DistancePoint

        if (slope < 0) and (distance > 0):
            Right = SlopePointB
            Bottom = SlopePointA
            Top = DistancePoint

        if (slope < 0) and (distance < 0):
            Right = SlopePointA
            Bottom = SlopePointB
            Top = DistancePoint

        return (Right, Bottom, Top)

    def CalculatePerpendicularDistance(
              self, DistancePoint, SlopePointA, SlopePointB):
        coeffA, coeffB, constant = self.__findCoefficientsOftheLine(
             SlopePointA, SlopePointB)
        slope = self.__findSlope(SlopePointA, SlopePointB)

        return (slope, coeffA * DistancePoint[0] + coeffB *
                DistancePoint[1] + constant / (np.sqrt(coeffA**2 + coeffB**2)))

    def __findCoefficientsOftheLine(self, pointA, pointB):
        slope = self.__findSlope(pointA, pointB)
        coefficientA = -slope
        coefficientB = 1
        constant = slope * pointA[0] - pointA[1]

        return (coefficientA, coefficientB, constant)

    def __findSlope(self, pointA, pointB):
        return (pointB[1] - pointA[1]) / (pointB[0] - pointA[0] + 1e-5)


import pytest
from sstar.cal_threshold import cal_threshold

@pytest.fixture
def data():
    pytest.simulated_data = "./examples/data/simulated_data/gravel_asn_scale_60k.simulated.data"
    pytest.score_file = "./tests/results/test.score.exp.results"
    pytest.recomb_map = "./examples/data/real_data/hum.windows.50k.10k.recomb.map"
    pytest.output = "./tests/results/test.threshold.results"
    pytest.exp_output = "./tests/results/test.threshold.exp.results"

def test_cal_threshold(data):
    cal_threshold(simulated_data=pytest.simulated_data, score_file=pytest.score_file, recomb_rate=0, recomb_map=pytest.recomb_map, quantile=0.99, output=pytest.output, k=8)

    f1 = open(pytest.output, 'r')
    res = f1.read()
    f1.close()
    f2 = open(pytest.exp_output, 'r')
    exp_res = f2.read()
    f2.close()

    assert res == exp_res


import boto3, argparse, base64

def main(function_name):
    client = boto3.client("lambda")
    response = client.invoke(
        FunctionName=function_name,
        InvocationType="RequestResponse",
    )
    print(response['Payload'].read().decode("utf-8"))

if __name__ == "__main__":
    # parse arguments
    parser = argparse.ArgumentParser()
    parser.add_argument("function_name", type=str)
    args = parser.parse_args()

    main(args.function_name)


import datetime
import select
import subprocess
import time
from sys import argv
import sys
import os
from dataclasses import dataclass
from typing import Dict, DefaultDict
from collections import defaultdict
import logging
import boto3
import kubernetes
from kubernetes.client.rest import ApiException

@dataclass
class ProcessOutput:
  stderr: str
  stdout: str
  other_fds: Dict[int, str]
  proc: subprocess.Popen

def stream_while_running(cwd, args, line_delimiter='', log_output=True):
  """Helper function for streaming a process' output whilst gathering it into a string"""
  proc = subprocess.Popen(cwd=cwd,
                          args=args, stderr=subprocess.PIPE,
                          stdout=subprocess.PIPE, text=True, bufsize=1)

  stderr = []
  stdout = []
  fds: DefaultDict[str] = defaultdict(list)
  while True:
    poll = proc.poll()
    if poll is not None:
      break
    else:
      # Poll the stdout/err streams every 0.05 seconds to see if anything has been written
      rdrs, _, _ = select.select([proc.stdout, proc.stderr], [], [], 0.05)

      # figure out which was returned
      if rdrs != []:
        for r in rdrs:
          # The FD might return multiple lines, so keep calling readline()
          # until we get a '' implying nothing more to consume
          keep_reading = True
          while keep_reading:
            line = r.readline()
            if len(line) > 0:
              if line[-1] == "\n":
                line = line[:-1]
              if r.name == proc.stdout.fileno():
                stdout.append(line)
                if log_output:
                  #logging.info(f"PID {str(proc.pid)} stdout: {line}")
                  logging.info(f"{line}")
              elif r.name == proc.stderr.fileno():
                stderr.append(line)
                if log_output:
                  #logging.info(f"PID {str(proc.pid)} stderr: {line}")
                  logging.info(f"{line}")
              else:
                fds[r.name].append(line)
                if log_output:
                  #logging.info(f"PID {str(proc.pid)} fd {r.name}: {line}")
                  logging.info(f"{line}")
            else:
              keep_reading = False
      else:
        # no new output
        continue

  # Flatten the list of list of strings that are the file descriptor outputs.
  # It makes it consistent with the subprocess output capture format
  for fd in fds:
    fds[fd] = line_delimiter.join(fds[fd])

  return ProcessOutput(line_delimiter.join(stderr), line_delimiter.join(stdout), fds, proc)

def slurp_ec2_instances(client):
  next_token = None
  instances = []
  keep_going = True
  while keep_going:
    if next_token is None:
      tmp = client.describe_instances()
    else:
      tmp = client.describe_instances(NextToken=next_token)
    for reservation in tmp['Reservations']:
      for instance in reservation['Instances']:
        instances.append(instance)
      if 'NextToken' in tmp and tmp['NextToken'] is not None:
        next_token = tmp['NextToken']
      else:
        keep_going = False
  return instances

def get_asgs_names(client, target):
  next_token = None
  asgs_names_list = []
  keep_going = True
  while keep_going:
    if next_token is None:
      tmp = client.describe_auto_scaling_groups()
    else:
      tmp = client.describe_auto_scaling_groups(NextToken=next_token)
    for asg in tmp['AutoScalingGroups']:
      if target == 'dbs':
        if 'db' in asg['AutoScalingGroupName']:
          asgs_names_list.append(asg['AutoScalingGroupName'])
      elif target == 'apps':
        if 'db' not in asg['AutoScalingGroupName']:
          asgs_names_list.append(asg['AutoScalingGroupName'])
      elif target == 'all':
        asgs_names_list.append(asg['AutoScalingGroupName'])
    if 'NextToken' in tmp and tmp['NextToken'] is not None:
      next_token = tmp['NextToken']
    else:
      keep_going = False
  return asgs_names_list

def main(region, target):

  logging.basicConfig(
    level=logging.INFO,
    format='%(levelname)s: %(asctime)s: %(message)s'
  )

  ec2_client = boto3.client('ec2', region_name=region)
  asg_client = boto3.client('autoscaling', region_name=region)

  namespace = 'default'
  snooze_seconds = 30
  pods_to_ignore = ['datadog-agent']
  errored = False

  # Get ASGS
  asgs_names_list = get_asgs_names(asg_client, target)

  # Get K8s nodes information
  kubernetes.config.load_kube_config()
  k8s_core_client = kubernetes.client.CoreV1Api()
  nodes = k8s_core_client.list_node()

  # Get EC2 instance tag information
  ec2_instances = slurp_ec2_instances(ec2_client)

  for k8s_node in nodes.items:
    instance_id = k8s_node.spec.provider_id.split('/')[-1]
    node_name = k8s_node.metadata.name
    matched_ec2_node = list(filter(lambda x: x['InstanceId'] == instance_id, ec2_instances))[0]
    asg_name = [x['Value'] for x in matched_ec2_node['Tags'] if x['Key'] == 'aws:autoscaling:groupName'][0]
    if asg_name not in asgs_names_list:
      continue

    print()
    logging.info(f"Dealing with instance: {instance_id}, in ASG: {asg_name}")

    logging.info(f"Will cordon instance: {instance_id}, with node_name: {node_name} so no new pods are scheduled on it")
    stream_while_running("/", ['kubectl', 'cordon', node_name])

    # Create a dict with node and pods details
    try:
      node_and_pods_dict = {}
      kubernetes.config.load_kube_config()
      k8s_core_client = kubernetes.client.CoreV1Api()
      field_selector = 'spec.nodeName=' + k8s_node.metadata.name
      ret = k8s_core_client.list_namespaced_pod(namespace, watch=False, field_selector=field_selector)
      node_and_pods_dict[instance_id] = {}
      node_and_pods_dict[instance_id]['node_name'] = k8s_node.metadata.name
      node_and_pods_dict[instance_id]['asg_name'] = asg_name
      node_and_pods_dict[instance_id]['pods'] = []

      for i in ret.items:
        if not i.metadata.name.startswith(tuple(pods_to_ignore)):
          node_and_pods_dict[instance_id]['pods'].append(i.metadata.name)
      node_and_pods_dict[instance_id]['number_of_pods'] = len(node_and_pods_dict[instance_id]['pods'])

    except ApiException as e:
      logging.error("Exception when calling Kubernetes CoreV1Api->list_namespaced_pod: %s\n" % e)

    logging.info(f"Instance and pod info: {node_and_pods_dict}")

    pod_controller_and_controller_kind_tuple_list = []

    # Create a list of tuples with controller name (derived from pod name) and controller kind
    for pod in node_and_pods_dict[instance_id]['pods']:
      try:
        kubernetes.config.load_kube_config()
        k8s_core_client = kubernetes.client.CoreV1Api()
        api_response = k8s_core_client.read_namespaced_pod(pod, namespace)
        controller_kind = (api_response.__dict__['_metadata'].__dict__['_owner_references'][0].__dict__['_kind'])
        pod_controller = pod.rpartition('-')[0]
        pod_controller_and_controller_kind_tuple = (pod_controller, controller_kind)
        pod_controller_and_controller_kind_tuple_list.append(pod_controller_and_controller_kind_tuple)
      except ApiException as e:
        logging.error("Exception when calling Kubernetes CoreV1Api->read_namespaced_pod: %s\n" % e)

    # Scale up ASG DesiredCapacity by 1, and also possibly increase MaxSize by 1 if DesiredCapacity + 1 > MaxSize
    logging.info(f"Dealing with ASG: {asg_name}")
    get_asg = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name])

    # Current ASG values
    current_desired_capacity = get_asg['AutoScalingGroups'][0]['DesiredCapacity']
    logging.info(f"Current ASG DesiredCapacity: {current_desired_capacity}")
    current_max_size = get_asg['AutoScalingGroups'][0]['MaxSize']
    logging.info(f"Current ASG MaxSize: {current_max_size}")

    # New ASG values
    new_desired_capacity = current_desired_capacity + 1
    logging.info(f"New ASG DesiredCapacity: {new_desired_capacity}")
    new_max_size = new_desired_capacity if new_desired_capacity > current_max_size else current_max_size
    logging.info(f"New ASG MaxSize: {new_max_size}")

    try:
      logging.info(f"Updating ASG: {asg_name} with new DesiredCapacity: {new_desired_capacity} and MaxSize: {new_max_size}")
      update_asg = asg_client.update_auto_scaling_group(AutoScalingGroupName=asg_name, DesiredCapacity=new_desired_capacity, MaxSize=new_max_size)

      # Wait for new instance to be InService
      logging.info("Waiting for new instance to be InService...")
      new_instance_not_up = True
      while new_instance_not_up:
        get_asg = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name])
        instances = get_asg['AutoScalingGroups'][0]['Instances']
        instances_lifecycle_states = [instance['LifecycleState'] for instance in instances]
        number_of_in_service_instances = instances_lifecycle_states.count("InService")
        if number_of_in_service_instances == new_desired_capacity:
          logging.info("New instance ready. Moving along...")
          new_instance_not_up = False
        else:
          logging.info(f"Still waiting for new instance to be InService. Will snooze for {snooze_seconds} seconds and try again...")
          time.sleep(snooze_seconds)

        # Set a 5 minute limit on the startup, in case something goes wrong
        start_time = datetime.datetime.now()
        if datetime.datetime.now() > (start_time + datetime.timedelta(minutes=5)):
          raise RuntimeError("Failed to get new node in 5 minutes. Bailing out!")

      # Drain the old node, and wait till it's fully drained
      logging.info(f"Draining old instance {instance_id}/{node_name} in ASG {asg_name}")
      stream_while_running("/", ['kubectl', 'drain', '--ignore-daemonsets', '--delete-local-data', node_name])
      all_pods_not_drained = True

      while all_pods_not_drained:
        try:
          node_and_pods_drained_node_dict = {}
          kubernetes.config.load_kube_config()
          k8s_core_client = kubernetes.client.CoreV1Api()
          field_selector = 'spec.nodeName=' + k8s_node.metadata.name
          ret = k8s_core_client.list_namespaced_pod(namespace, watch=False, field_selector=field_selector)
          node_and_pods_drained_node_dict[instance_id] = {}
          node_and_pods_drained_node_dict[instance_id]['node_name'] = k8s_node.metadata.name
          node_and_pods_drained_node_dict[instance_id]['asg_name'] = asg_name
          node_and_pods_drained_node_dict[instance_id]['pods'] = []

          for i in ret.items:
            if not i.metadata.name.startswith(tuple(pods_to_ignore)):
              node_and_pods_drained_node_dict[instance_id]['pods'].append(i.metadata.name)
          node_and_pods_drained_node_dict[instance_id]['number_of_pods'] = len(node_and_pods_drained_node_dict[instance_id]['pods'])

        except ApiException as e:
          logging.error("Exception when calling Kubernetes CoreV1Api->list_namespaced_pod: %s\n" % e)

        if node_and_pods_drained_node_dict[instance_id]['number_of_pods'] == 0:
          logging.info(f"All pods have been drained off instance {instance_id}. Moving along...")
          all_pods_not_drained = False
        else:
          logging.info(f"Still waiting for all pods to be drained off instance {instance_id}. {node_and_pods_drained_node_dict[instance_id]['number_of_pods']} pod(s) ({node_and_pods_drained_node_dict[instance_id]['pods']}) left. Will snooze for {snooze_seconds} seconds and try again...")
          time.sleep(snooze_seconds)

      # Wait and keep checking Kubernetes API to see if there are 100% replicas available for each ReplicaSet and StatefulSet
      logging.info(f"Will wait for all replicas to be available for each ReplicaSet/StatefulSet, while ignoring DaemonSet/Ad Hoc Pods ({pods_to_ignore})...")

      set_counter_list = []

      while len(pod_controller_and_controller_kind_tuple_list) > len(set_counter_list):
        for pod_controller, controller_kind in pod_controller_and_controller_kind_tuple_list:
          logging.info(f"Dealing with {pod_controller}, {controller_kind}")
          all_replicas_not_available = True
          while all_replicas_not_available:
            try:
              kubernetes.config.load_kube_config()
              k8s_app_client = kubernetes.client.AppsV1Api()
              if controller_kind == 'ReplicaSet':
                api_response = k8s_app_client.read_namespaced_replica_set(pod_controller, namespace)
              elif controller_kind == 'StatefulSet':
                api_response = k8s_app_client.read_namespaced_stateful_set(pod_controller, namespace)
              else:
                set_counter_list.append("UnknownController")
                continue
              number_of_replicas = api_response.__dict__['_status'].__dict__['_replicas']
              number_of_ready_replicas = api_response.__dict__['_status'].__dict__['_ready_replicas']
              if number_of_ready_replicas is None or number_of_ready_replicas < number_of_replicas:
                logging.info(f"{pod_controller} {controller_kind} does not have {number_of_replicas}/{number_of_replicas} replicas ready (only {number_of_ready_replicas or 0}/{number_of_replicas} ready). Will snooze for {snooze_seconds} seconds and try again...")
                time.sleep(snooze_seconds)
              elif number_of_ready_replicas >= number_of_replicas:
                logging.info(f"{pod_controller} {controller_kind} has {number_of_ready_replicas}/{number_of_replicas} replicas ready. Moving along...")
                all_replicas_not_available = False
                set_counter_list.append(pod_controller)
              else:
                logging.info(f"{pod_controller} {controller_kind} does not have {number_of_replicas}/{number_of_replicas} replicas ready (only {number_of_ready_replicas}/{number_of_replicas} ready). Will snooze for {snooze_seconds} seconds and try again...")
                time.sleep(snooze_seconds)
            except ApiException as e:
              logging.error("Exception when calling Kubernetes AppsV1Api: %s\n" % e)

      logging.info("100% replicas available for each ReplicaSet/StatefulSet. Moving along...")

      # Terminate the old node
      logging.info("Terminating old instance...")
      terminate_instance = ec2_client.terminate_instances(InstanceIds=[instance_id])

      # Scale ASG DesiredCapacity and MaxSize back down to original values so we are consistent with Terraform
      logging.info(f"Reverting ASG: {asg_name} with original DesiredCapacity: {current_desired_capacity} and MaxSize: {current_max_size}")
      update_asg = asg_client.update_auto_scaling_group(AutoScalingGroupName=asg_name, DesiredCapacity=current_desired_capacity, MaxSize=current_max_size)

    except (KeyboardInterrupt, TypeError) as e:
      errored = True
      logging.error(e)
      logging.error(f"Signal/TypeError caught! Will revert ASG: {asg_name} back to original DesiredCapacity: {current_desired_capacity} and MaxSize: {current_max_size} before exiting!")
      stream_while_running("/", ['kubectl', 'uncordon', node_name])
      update_asg = asg_client.update_auto_scaling_group(AutoScalingGroupName=asg_name, DesiredCapacity=current_desired_capacity, MaxSize=current_max_size)
      logging.error(f"ASG reverted back to original values. Instance {instance_id} has been uncordoned. Check state of Kubernetes and AWS before running the script again. Exiting!")
      try:
        sys.exit(0)
      except SystemExit:
        os._exit(0)

    except:
      errored = True
      logging.error("Unexpected error:", sys.exc_info()[0])
      logging.error(f"Will revert ASG: {asg_name} back to original DesiredCapacity: {current_desired_capacity} and MaxSize: {current_max_size} before exiting!")
      stream_while_running("/", ['kubectl', 'uncordon', node_name])
      update_asg = asg_client.update_auto_scaling_group(AutoScalingGroupName=asg_name, DesiredCapacity=current_desired_capacity, MaxSize=current_max_size)
      logging.error(f"ASG reverted back to original values. Instance {instance_id} has been uncordoned. Check state of Kubernetes and AWS before running the script again. Exiting!")
      try:
        sys.exit(0)
      except SystemExit:
        os._exit(0)
      raise

    try:
      if errored:
        logging.error(f"An error occurred above, and the ASG: {asg_name} has already been reverted to original DesiredCapacity: {current_desired_capacity} and MaxSize: {current_max_size}. Exiting!")
      else:
        # Wait for number of instances in ASG to scale back down
        logging.info("Waiting for number of instances in ASG to scale back down...")
        asg_instances_not_scaled_down = True
        while asg_instances_not_scaled_down:
          get_asg = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name])
          instances = get_asg['AutoScalingGroups'][0]['Instances']
          instances_lifecycle_states = [instance['LifecycleState'] for instance in instances]
          number_of_in_service_instances = instances_lifecycle_states.count("InService")
          if number_of_in_service_instances == current_desired_capacity:
            logging.info(f"Number of instances in ASG has scaled back down to original DesiredCapacity: {current_desired_capacity}")
            print("--------------------------------------------------------------------------------------------")
            print("--------------------------------------------------------------------------------------------")
            asg_instances_not_scaled_down = False
          else:
            logging.info(f"Still waiting for number of instances in ASG to scale back down. Will snooze for {snooze_seconds} seconds and try again...")
            time.sleep(snooze_seconds)
    except Exception as e:
      logging.error(e)
      logging.error(f"An error occurred above, and the infrastructure is possibily in an unstable state now. Check state of Kubernetes and AWS before running the script again. Exiting!")

if __name__ == '__main__':
  region = argv[1]
  target = argv[2]
  main(region, target)


from app.extensions import db,fake # Import your db and faker instance instance from your app
from app.models.Client import Client
from app.models.CodeAppointment import CodeAppointment
from app.models.DrivingAppointment import DrivingAppointment
from app.models.Quiz import Quiz
from app.models.Question import Question
from app.models.Response import Response
from main import create_app
import random

app = create_app()

# Generated client data
client_data= []
for _ in range(20):
    first_name = fake.first_name()
    last_name = fake.last_name()
    email = fake.email()
    phone_number = fake.random_int(min= 20000000, max = 99000000)
    cin_number = fake.random_int(min= 12000000, max = 99999999)
    cin_recto = fake.image_url()
    cin_verso = fake.image_url()
    address = fake.address()
    date_of_birth = fake.date_of_birth(minimum_age=18, maximum_age=65)
    nb_hours_code = fake.random_int(min=1, max= 99)
    nb_hours_driving = fake.random_int(min=1, max= 99)

    client_instance = Client(
        first_name=first_name,
        last_name=last_name,
        email=email,
        phone_number=phone_number,
        cin_number=cin_number,
        cin_recto=cin_recto,
        cin_verso=cin_verso,
        address=address,
        date_of_birth=date_of_birth,
        nb_hours_code=nb_hours_code,
        nb_hours_driving=nb_hours_driving
    )
    client_data.append(client_instance)
    # Add the generated client data to the database
    with app.app_context():
        db.session.add_all(client_data)
        db.session.commit()

# Generated code appointement data
with app.app_context():
        clients = Client.query.all() #Fetch all clients from database
        code_appointment_data= []

        for _ in range(10):
            test_date_code= fake.date_of_birth(minimum_age=18, maximum_age=65)
            client = random.choice(clients)
            code_appointment_instance = CodeAppointment(
                test_date_code=test_date_code,
                client_id = client.id
            )
            code_appointment_data.append(code_appointment_instance)
        db.session.add_all(code_appointment_data)
        db.session.commit()

# Generated driving appointement data
with app.app_context():
    clients = Client.query.all() #Fetch all clients from database
    driving_appointment_data = []

    for _ in range(10):
        test_date_driving= fake.date_of_birth(minimum_age=18, maximum_age=65)
        client = random.choice(clients)
        driving_appointment_instance= DrivingAppointment(
             test_date_driving=test_date_driving,
             client_id =client.id
        )
        driving_appointment_data.append(driving_appointment_instance)
    db.session.add_all(driving_appointment_data)
    db.session.commit()

# Generated Quiz data
with app.app_context():
    quiz_data = []
    for _ in range(10):
        uuid = fake.uuid4()
        score = fake.random_int(min=0,max=100)
        quiz_instance = Quiz(
             uuid=uuid,
             score=score
        )
        quiz_data.append(quiz_instance)
    db.session.add_all(quiz_data)
    db.session.commit()

# Generated Question data
with app.app_context():
    quizes = Quiz.query.all()  #Fetch all quizes from database

    question_data = []
    for _ in range(10):
        text = fake.paragraph()
        quiz = random.choice(quizes)
        question_instance = Question(
            text=text,
            quiz_id = quiz.id
        )
        question_data.append(question_instance)
    db.session.add_all(question_data)
    db.session.commit()

# Generated Response data
with app.app_context():
    questions = Question.query.all()  #Fetch all questions from database

    response_data = []
    for _ in range(10):
        text = fake.paragraph()
        is_correct = fake.boolean()
        question = random.choice(questions)
        response_instance = Response(
            text=text,
            is_correct=is_correct,
            question_id = question.id
        )
        response_data.append(response_instance)
    db.session.add_all(response_data)
    db.session.commit()


import inspect
import time

from django import VERSION as DJANGO_VERSION
from django.conf import settings
from django.http import Http404

from django_statsd.clients import statsd

try:
    from django.utils.deprecation import MiddlewareMixin
except ImportError:
    class MiddlewareMixin(object):
        pass

def is_authenticated(user):
    if DJANGO_VERSION < (1, 10):
        return user.is_authenticated()
    return user.is_authenticated

class GraphiteMiddleware(MiddlewareMixin):

    def process_response(self, request, response):
        statsd.incr('response.%s' % response.status_code)
        if hasattr(request, 'user') and is_authenticated(request.user):
            statsd.incr('response.auth.%s' % response.status_code)
        return response

    def process_exception(self, request, exception):
        if not isinstance(exception, Http404):
            statsd.incr('response.500')
            if hasattr(request, 'user') and is_authenticated(request.user):
                statsd.incr('response.auth.500')

class GraphiteRequestTimingMiddleware(MiddlewareMixin):
    """statsd's timing data per view."""

    def process_view(self, request, view_func, view_args, view_kwargs):
        view = view_func
        if not inspect.isfunction(view_func):
            view = view.__class__
        try:
            request._view_module = view.__module__
            request._view_name = view.__name__
            request._start_time = time.time()
        except AttributeError:
            pass

    def process_response(self, request, response):
        self._record_time(request)
        return response

    def process_exception(self, request, exception):
        self._record_time(request)

    def _record_time(self, request):
        if hasattr(request, '_start_time'):
            ms = int((time.time() - request._start_time) * 1000)
            data = dict(module=request._view_module, name=request._view_name,
                        method=request.method)
            statsd.timing('view.{module}.{name}.{method}'.format(**data), ms)
            if getattr(settings, 'STATSD_VIEW_TIMER_DETAILS', True):
                statsd.timing('view.{module}.{method}'.format(**data), ms)
                statsd.timing('view.{method}'.format(**data), ms)

class TastyPieRequestTimingMiddleware(GraphiteRequestTimingMiddleware):
    """statd's timing specific to Tastypie."""

    def process_view(self, request, view_func, view_args, view_kwargs):
        try:
            request._view_module = view_kwargs['api_name']
            request._view_name = view_kwargs['resource_name']
            request._start_time = time.time()
        except (AttributeError, KeyError):
            super(TastyPieRequestTimingMiddleware, self).process_view(
                request, view_func, view_args, view_kwargs)


a={'one':1,'two':2,'three':3,'four':4,'five':5}
for i in a:
    x=a.values()### ->output-->dict_values([1, 2, 3, 4, 5])
print(x)

##values():Returns a list of all the values in the dictionary

# for i in a.values():
#     print(i)### print only the values

# details ={
# "name":  "Bijender",
# "age":  17,
# "class":  "10th"
# }
# for x in details.values():##return values
#     print(x)


# HackerRank CCI:
# https://www.hackerrank.com/challenges/ctci-making-anagrams/problem
# Python 3

# -----------------------------------------------------------------------------
# Number of chars required to delete from each passed string a, b to get anagrams
# Constraints:
# 1 <= |a|, |b| <= 10000 (10^4)
# It is guaranteed that strings a and b consist of lowercase English alphabetic letters (i.e. a-z).
def number_needed(first, second):
    number_of_deletions = 0
    # Create hash/dict that maps chars a-z to their count
    letters = {}

    # First iterate over the first string and update frequency of letters found (increment)
    for char in first:
        if char in letters:
            letters[char] += 1
        else:
            letters[char] = 1

    # Now iterate over the second string and update the frequency of letters found (decrement)
    for char in second:
        if char in letters:
            letters[char] -= 1
        else:
            letters[char] = -1

    # Now sum the char frequencies in the letters array using ABS
    for char in letters:
        number_of_deletions += abs(letters[char])

    return number_of_deletions

# **NOTE: This passed most tests, but not efficient enough (don't need to get the actual strings anagrams)
def old_number_needed(a, b):
    # First remove any chars in a not in b, or extra occurrences in a
    new_string_a = remove_chars(a, b)
    # DEBUG
    print('NEW modified string A: ', new_string_a)
    # Total char deletions is original string a - new_string_a length
    total_deletions = len(a) - len(new_string_a)

    # Now remove any chars in b not in the new string a from above, or extra occurrences in b
    new_string_b = remove_chars(b, new_string_a)
    # DEBUG
    print('NEW modified string B: ', new_string_b)
    # Total char deletions is total deletions from above + original string_b - new_string_b length
    total_deletions += len(b) - len(new_string_b)

    return total_deletions

# -----------------------------------------------------------------------------
# Removes chars from string_one NOT found in string_two recursively
# **NOTE: This passed most tests, but not efficient enough (don't need to get the actual strings anagrams)
def remove_chars(string_one, string_two):
    new_string = None

    # Remove any chars in string_a not in string_b
    for char in string_one:
        # DEBUG
        print('String ONE: ', string_one)
        print('Char: ', char)

        # If char is not in string_two, remove any occurrences from string_one
        if char not in string_two:
            # Remove ALL occurrences of this char
            new_string = string_one.replace(char, '')
            # DEBUG
            print('Removed ALL occurrences of char: ', char)
            new_string = remove_chars(new_string, string_two)
            break
        # Else check the strings have the same number of occurrences of this char
        else:
            if string_one.count(char) > string_two.count(char):
                # Just remove ONE occurrence of this char
                new_string = string_one.replace(char, '', 1)
                # DEBUG
                print('Removed ONE occurrence of char: ', char)
                new_string = remove_chars(new_string, string_two)
                break

    new_string = string_one if new_string is None else new_string
    # DEBUG
    print('Returning NEW string: ', new_string)
    return new_string

# -----------------------------------------------------------------------------
# Main
# -----------------------------------------------------------------------------

# User Input
# a = input().strip()
# b = input().strip()

# Tests
#a = 'cde'
#b = 'abc'

# a = 2; b = 1; c = 2; d = 2; e = 1; f = 1; g = 1 ... z = 0
# a = 2; b = 1; c = 1; d = 1; e = 1; f = 1; g = -2 ... z = 2
# Total => 11
a = 'abccddefag'
b = 'cdgggzz'

print(number_needed(a, b))


## LINKs códigos:
            ##Link Collab: https://colab.research.google.com/drive/1SF3p2nz389SQnAgoKO5os2ZjlexW_O16?authuser=2#scrollTo=seR4-BXXb1f_
## Parte del código extraido de google collab

## Loading the required libraries:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import scipy.stats as st
#import warnings
#warnings.filterwarnings('ignore')

from pandasql import sqldf
pysqldf = lambda q: sqldf(q, globals())

## Reading the data:
order_items = pd.read_csv('olist_order_items_dataset.csv', on_bad_lines='skip')  ## Unclassified orders dataset
seller=pd.read_csv('olist_sellers_dataset.csv', on_bad_lines='skip') ## Seller information
translation=pd.read_csv('product_category_name_translation.csv', on_bad_lines='skip')
product = pd.read_csv('olist_products_dataset.csv', on_bad_lines='skip')  ## Product translation to english

# Generate data at product-item level:
order_item_products = pd.merge(order_items,product, how='left', on='product_id')
orders_item_products_english = pd.merge(order_item_products,translation, how='left', on='product_category_name')
orders = pd.merge(orders_item_products_english,seller, how='left', on='seller_id')

# Select only the columns we are going to use:
selected_columns = orders[['order_id','product_id', 'product_category_name_english','seller_id', 'seller_city','seller_state','price','freight_value']]

# Aggregate data: Count orders to get the quantity of sold items:
order_products = selected_columns.groupby(['product_id','seller_state','seller_city','product_category_name_english']).aggregate({'order_id':'count','price':'mean','freight_value':'mean'}).rename(columns={'order_id':'quantity'}).reset_index() #.sort_values(by='quantity',ascending=False).reset_index()

# Multiply 'price' and 'quantity' to get the Gross Monetary Value (GMV):
order_products['GMV'] = order_products['price']*order_products['quantity']
dataset = order_products
#### Fin extraccion Google Collab ###

## Renaming column
dataset.rename(columns ={"product_category_name_english" : "product_category"}, inplace = True)

############## FUNCTIONS ###############

def STD(dataframe):
    size = dataframe["quantity"].sum()
    mean = dataframe["GMV"].sum()/size
    SquareSum = 0
    i = 0
    while i<dataframe.shape[0]:
        SquareSum+= (dataframe["GMV"][i] - mean)**2
        i+=1
    Division = SquareSum/size
    std = Division**(1/2)
    #print(size)
    #print(mean)
    #print(SquareSum)
    #print(std)
    list = [mean, std]
    return list

############## END FUNCTIONS ###########

q1 = f"""SELECT product_category, SUM(quantity) as sum_quantity, SUM(GMV) as sum_GMV FROM dataset
GROUP BY product_category
ORDER BY sum_GMV DESC
LIMIT 1 """
n1 = pysqldf(q1)
#print(n1, "\n")

category = n1["product_category"][0]
q2 = f"""SELECT product_category, quantity, price, GMV FROM dataset WHERE product_category = "{category}"
ORDER BY price DESC"""
n2 = pysqldf(q2)
#print(n2, "\n")

###  SAMPLE N = 25 #####
Sample25 = n2["GMV"].sample(n = 25)
#print(Sample25)
Interval25t = st.t.interval(0.95, (len(Sample25) - 1),np.mean(Sample25), st.sem(Sample25))
print("\nIntervalo confianza n = 25")
Ancho25 = Interval25t[1] - Interval25t[0]
print(Interval25t)
print("Ancho ", Ancho25)

###  SAMPLE N = 100 #####
Sample100 = n2["GMV"].sample(n = 100)
#print(Sample100)
Interval100 = st.norm.interval(0.95, np.mean(Sample100), st.sem(Sample100))
print("\nIntervalo confianza n = 100")
print(Interval100)
Ancho100 = Interval100[1] - Interval100[0]
print("Ancho ", Ancho100)

###  SAMPLE N = 1000 #####
Sample1000 = n2["GMV"].sample(n = 1000)
#print(Sample100)
Interval1000 = st.norm.interval(0.95, np.mean(Sample1000), st.sem(Sample1000))
print("\nIntervalo confianza n = 1000")
print(Interval1000)
Ancho1000 = Interval1000[1] - Interval1000[0]
print("Ancho ", Ancho1000, "\n")


courses = [
	"EDUC-X158",
	"MATH-M212",
	"EALC-E111",
	"CSCI-C200"
]

def containsCourse(str):
	for c in courses:
		if c in str:
			return True
	return False

with open ("TODO.txt", "r") as todo:
	rawTodo = todo.read().strip()

lines = rawTodo.splitlines()

assignment = []
sub = []
complete = []

for l in lines:
	if len(l) > 0:
		# Complete Assignment Detected
		if l.strip()[0] == "/":
			complete.append(l.strip())
		# Assignment Detected
		elif l.strip()[0] == "-" and not containsCourse(l):
			assignment.append(l.strip())
		# Assignment Part
		elif l.strip()[0] == ".":
			sub.append(l.strip())

print('Assignments:')
for a in assignment:
	print(a)
print('Sub:')
for s in sub:
	print(s)
print('Completed:')
for c in complete:
	print(c)


"""
name:小沙雕计算器
edition：3.0.2   [eval复原版本]
author:yuanchuang
Email:3358851903@qq.com（欢迎交流）
"""
from tkinter import *
from tkinter import messagebox
from decimal import *

tip = 0

def shuru(a):
            global tip
            if shuchu.get()=="0":
                shuchu.set(a)
            else:

                if a == "%":
                    try:
                        shuchu.set(eval(shuchu.get()+"/100"))
                    except BaseException:
                        messagebox.showinfo("tip","计算力上限")

                else:
                    shuchu.set(shuchu.get() + str(a))
                    if len(shuchu.get())>6 and tip == 0:
                          messagebox.showinfo("tip","超出七个字符可能显示不全")

                          tip = 1

def de():
                shuchu.set("0")
def dell():
            shuchu.set(shuchu.get()[:-1])

def dengyu():
             try:
                 shuchu.set(shuchu.get().replace("×","*"))
                 shuchu.set(shuchu.get().replace("÷","/"))

                 shuchu.set(eval(shuchu.get()))

                 if len(shuchu.get())==7 and tip!=1:

                    messagebox.showinfo("tip","超出七个字符可能显示不全")

             except SyntaxError:
                     messagebox.showinfo("tip","输入有误")
                     de()
             except ZeroDivisionError:
                     messagebox.showinfo("tip","输入有误")
                     de()

def kuaijie():
          try:
              shuchu.set(shuchu.get()[:shuchu.get().index(".")+3])
          except ValueError:
              messagebox.showinfo("tip","无小数点，不需要精确！")
        #创建主窗口
root = Tk()
zibs = 50

        #输出台架构
shuchu = StringVar()
shuchu.set("0")
result = LabelFrame(root,text="输出台        欢迎使用       小沙雕计算器3.0",bg="bisque")

reword = Entry(result,textvar=shuchu,font=(None,zibs),fg="black",bg="bisque")

reword.pack(anchor="e")
result.pack(fill="x",anchor="center")
        #输入台架构
eve = LabelFrame(root,text="按键",bg="azure")

b1 = Button(eve,text=1,bd=10,bg="sandybrown",font=(None,40),command=lambda:shuru(1)).grid(row=1,column=0)

b2 = Button(eve,text=2,bd=10,bg="lavender",font=(None,40),command=lambda:shuru(2)).grid(row=1,column=1)

b3 = Button(eve,text=3,bd=10,bg="sandybrown",font=(None,40),command=lambda:shuru(3)).grid(row=1,column=2)

b4 = Button(eve,text=4,bd=10,bg="lavender",font=(None,40),command=lambda:shuru(4)).grid(row=2,column=0)

b5 = Button(eve,text=5,bd=10,bg="sandybrown",font=(None,40),command=lambda:shuru(5)).grid(row=2,column=1)

b6 = Button(eve,text=6,bd=10,bg="lavender",font=(None,40),command=lambda:shuru(6)).grid(row=2,column=2)

b7 = Button(eve,text=7,bd=10,bg="sandybrown",font=(None,40),command=lambda:shuru(7)).grid(row=3,column=0)

b8 = Button(eve,text=8,bd=10,bg="lavender",font=(None,40),command=lambda:shuru(8)).grid(row=3,column=1)

b9 = Button(eve,text=9,bd=10,bg="sandybrown",font=(None,40),command=lambda:shuru(9)).grid(row=3,column=2)

bcheng = Button(eve,text="×",bd=10,bg="wheat",font=(None,40),command=lambda:shuru("×")).grid(row=1,column=3)

bjian = Button(eve,text="-",bd=10,bg="wheat",font=(None,40),width=1,height=1,command=lambda:shuru("-")).grid(row=2,column=3)

bnone = Button(eve,text="C",bd=10,bg="wheat",font=(None,40),width=1,height=1,command=de).grid(row=0,column=0)

b0 = Button(eve,text=0,bd=10,bg="sandybrown",font=(None,40),command=lambda:shuru(0)).grid(row=4,column=1)

bshan = Button(eve,text="←",bd=10,bg="wheat",font=(None,40),width=1,height=1,command=dell).grid(row=0,column=1)

bbaifen = Button(eve,text="%",bd=10,bg="wheat",font=(None,40),width=1,height=1,command=lambda:shuru("%")).grid(row=0,column=2)

bchu = Button(eve,text="÷",bd=10,bg="wheat",font=(None,40),width=1,height=1,command=lambda:shuru("÷")).grid(row=0,column=3)

bjia = Button(eve,text="+",bd=10,bg="wheat",font=(None,40),width=1,height=1,command=lambda:shuru("+")).grid(row=3,column=3)

bpoint = Button(eve,text=".",bd=10,bg="wheat",font=(None,40),width=1,height=1,command=lambda:shuru(".")).grid(row=4,column=2)

bdengyu = Button(eve,text="=",bd=10,bg="coral",font=(None,40),width=1,height=1,command=dengyu).grid(row=4,column=3)

bgaoji = Button(eve,text="省",bd=10,bg="brown",width=1,height=1,font=(None,40),command=kuaijie).grid(row=4,column=0)

eve.pack(fill="x")

root.mainloop()


'''

'''

import subprocess, os.path, logging, tempfile, warnings

from ngs.alignment import Aligner

class BWA(Aligner):

    '''
    A class to run the BWA aligner

    '''

    #
    # in order to perform a BWA alignment first an
    # sam index (sai) should be constructed. Seperately
    # for the forward and reverse read in the case of a
    # paired end
    #
    #
    #
    #
    #
    #

    def __init__( self ):
        '''

        '''
        self._locs = {}
        logging.info( "Intializing the BWA object" )

        # set the nice loc
        self._locs['nice'] = 'nice'
        self._locs['bwa']  = 'bwa'

    def is_available( self, fname ):
        '''
        Checks whether the specified index is available

        Args:
        self   -
        fname  - the file path to check for

        Returns:
        True or False
        '''
        suffixes = [ 'amb', 'ann', 'bwt', 'fai', 'pac', 'sa', 'rbwt', 'rpac', 'rsa' ]
        present = True
        for s in suffixes:
            fn = fname + '.' + s
            if not os.path.exists( fn ):
                present = False

        # return whether the index is present or not
        return present

    def align( self, index, infiles, outfile, opt={}, saifiles=[] ):
        '''
        Use the BWA aligner to align the reads in infiles
        and output the data to the outfile

        Args:
        self     -  a reference to self
        infiles  -  the input FastQ file
        index    -  the index to which to align
        outfile  -  the output SAM file
        opt      -  options to pass to the alignment

        Returns:
        A tuple with the error, samfile and error file
        '''

        retval = None
        opt    = self._add_default_options( opt )
        if len( infiles ) == 2:
            self._index   = index
            self._samfile = outfile

            # fix the output file names
            if len(saifiles) != len(infiles):
                for f in infiles:
                    if not os.path.exists( f ):
                        logging.warn( "input file %f does not exist")
                        return None
                    saifiles.append( "%s.sai" % f )

            # actually perform the alignment
            for i in range( len( infiles ) ) :
                 self._infile  = infiles[i]
                 self._saifile = saifiles[i]
                 self._aln( opt=opt['aln'] )

            # gather the results
            self._infile  = infiles
            self._saifile = saifiles
            retval = self._sampe( opt=opt['sampe'] )

        elif len(infiles) == 1:
            self._index   = index
            self._infile  = infiles[0]
            if len(saifiles) == 1:
                self._saifile = saifiles[0]
            else:
                self._saifile = "%s.sai" % infiles[0]
            self._samfile = outfile

            # perform the alignment and results gathering
            self._aln( opt=opt['aln'] )
            retval = self._samse( opt=opt['samse'] )
        else:
            logging.warn( 'Incorrect number of input files' )

        #
        return retval

    def _aln( self, opt={} ):
        '''
        Perform the alignment and make the sam index file

        Args:
        self
        opt

        Returns:
        '''
        #
        cmd = []
        cmd.append( self._locs['nice'] )
        cmd.append( self._locs['bwa'] )
        cmd.append( 'aln' )

        # add the options
        cmd = self._add_options( cmd, opt )

        # add the index
        cmd.append( self._index )
        cmd.append( self._infile )

        # run the process
        (err,pout,perr) = self._subprocess( cmd, stdout=self._saifile )

        #
        return (err,pout,perr)

    def _samse( self, opt={} ):
        '''
        Assemble the single read alignment file

        Args:

        Returns:

        '''
        #
        cmd = []
        cmd.append( self._locs['nice'] )
        cmd.append( self._locs['bwa'] )
        cmd.append( 'samse' )

        #
        cmd = self._add_options( cmd, opt )

        #
        cmd.append( self._index )
        cmd.append( self._saifile )
        cmd.append( self._infile )

        #
        (err,pout,perr) = self._subprocess( cmd, stdout=self._samfile )

        #
        return (err,pout,perr)

    def _sampe( self, opt={} ):
        '''
        Assemble a paired end alignment

        Args:

        Returns:

        '''
        #
        cmd = []
        cmd.append( self._locs['nice'] )
        cmd.append( self._locs['bwa'] )
        cmd.append( 'sampe' )

        #
        cmd = self._add_options( cmd, opt )
        cmd.append( self._index )

        #
        cmd.append( self._saifile[0] )
        cmd.append( self._saifile[1] )

        #
        cmd.append( self._infile[0] )
        cmd.append( self._infile[1] )

        #
        (err,pout,perr) = self._subprocess( cmd, stdout=self._samfile )
        return (err,pout,perr)

    def _add_default_options( self, opt ):
        '''
        This method will add 'sensible' default options.
        So NARWHAL will not crash if these are absent. This
        function will not fill fields that are defined.

        Args:
        opt   -    the options hash

        Returns:
          return the filled options
        '''
        if not opt.has_key( 'sampe' ):
            # add the sampe options
            opt['sampe'] = {}

        if not opt.has_key( 'samse' ):
            # the samse options
            opt['samse'] = {}

        if not opt.has_key( 'aln' ):
            # the alignment options
            opt['aln'] = {}

        return opt

if __name__ == '__main__':
    pass


"""
Given an array of integers nums and an integer target,
return indices of the two numbers such that they add up to target.

You may assume that each input would have exactly one solution,
and you may not use the same element twice.

You can return the answer in any order.
"""

import unittest
from typing import List

class Solution:
    def twoSum(self, nums: List[int], target: int) -> List[int]:

        for i in range(len(nums)-1):
            current = nums[i]

            for j in range(i+1, len(nums)):
                next_c = nums[j]

                if next_c + current == target:
                    return [i, j]

class TestSolution(unittest.TestCase):
    def testTwoSum(self):
        s = Solution()

        nums = [2,7,11,15]
        target = 9
        expected = [0,1]
        actual = s.twoSum(nums, target)
        self.assertEqual(actual, expected)

        nums = [3,2,4]
        target = 6
        expected = [1,2]
        actual = s.twoSum(nums, target)
        self.assertEqual(actual, expected)

        nums = [3,3]
        target = 6
        expected = [0,1]
        actual = s.twoSum(nums, target)
        self.assertEqual(actual, expected)

unittest.main()


#!/usr/bin/env python3

import math
from collections import defaultdict
from typing import List
from typing import Set
try:
    from _tree import *
except ImportError:
    pass

class Solution:
    def shiftGrid(self, grid: List[List[int]], k: int) -> List[List[int]]:
        if k == 0:
            return grid

        m = len(grid)
        n = len(grid[0])
        new_grid = []
        for _ in range(m):
            new_grid.append([0] * n)

        for i in range(m):
            for j in range(n):
                new_j = (j + k) % n
                new_i = (i + (j + k) // n) % m
                new_grid[new_i][new_j] = grid[i][j]

        print(new_grid)
        return new_grid

if __name__ == '__main__':
    Solution().shiftGrid(grid = [[1,2,3],[4,5,6],[7,8,9]], k = 1)
    Solution().shiftGrid(grid = [[3,8,1,9],[19,7,2,5],[4,6,11,10],[12,0,21,13]], k = 4)
    Solution().shiftGrid(grid = [[1,2,3],[4,5,6],[7,8,9]], k = 9)