import torch
import copy
import wandb
import numpy as np

"""================================================================================================="""
ALLOWED_MINING_OPS  = None
REQUIRES_BATCHMINER = False
REQUIRES_OPTIM      = True
REQUIRES_LOGGING    = True

### MarginLoss with trainable class separation margin beta. Runs on Mini-batches as well.
class Criterion(torch.nn.Module):
    def __init__(self, opt, **kwargs):
        """
        Args:
            margin:             Triplet Margin.
            nu:                 Regularisation Parameter for beta values if they are learned.
            beta:               Class-Margin values.
            n_classes:          Number of different classes during training.
        """
        super(Criterion, self).__init__()

        self.n_classes = opt.n_classes
        self.name = 'margin_multisimilarity'

        #### MS loss params
        self.pos_weight = opt.loss_multisimilarity_pos_weight
        self.neg_weight = opt.loss_multisimilarity_neg_weight
        self.margin = opt.loss_multisimilarity_margin
        self.pos_thresh = opt.loss_multisimilarity_pos_thresh
        self.neg_thresh = opt.loss_multisimilarity_neg_thresh
        self.d_mode = opt.loss_multisimilarity_d_mode
        self.base_mode = opt.loss_multisimilarity_base_mode
        self.sampling_mode = opt.loss_multisimilarity_sampling_mode
        self.sampling_incl_pos = opt.loss_multisimilarity_sampling_incl_pos

        if 'pads' in self.sampling_mode:
            self.sampling_distr = Sampling_Distribution(opt.loss_multisimilarity_init_distr_nbins, [0., 1.],
                                                        update_step_size=25, ema_alpha=opt.loss_multisimilarity_sampling_ema_alpha,
                                                        update_type=self.sampling_mode, init_distr=opt.loss_multisimilarity_init_distr,
                                                        init_distr_mu=opt.loss_multisimilarity_init_distr_mu,
                                                        init_distr_sigma=opt.loss_multisimilarity_init_distr_sigma,
                                                        normalize_update=opt.loss_multisimilarity_sampling_normalize_update)
        else:
            self.sampling_distr = None

        #### margin loss params
        self.beta_const = opt.loss_multisimilarity_beta_const
        self.beta = torch.nn.Parameter(torch.ones(opt.n_classes)*opt.loss_multisimilarity_beta)
        self.lr = opt.loss_multisimilarity_beta_lr

        ####
        self.ALLOWED_MINING_OPS= ALLOWED_MINING_OPS
        self.REQUIRES_BATCHMINER = REQUIRES_BATCHMINER
        self.REQUIRES_OPTIM = REQUIRES_OPTIM
        self.REQUIRES_LOGGING = REQUIRES_LOGGING

    def forward(self, batch, labels, global_step, **kwargs):

        assert batch.size(0) == labels.size(0), \
            f"batch.size(0): {batch.size(0)} is not equal to labels.size(0): {labels.size(0)}"
        batch_size = batch.size(0)
        sim_mat = batch.mm(batch.T)

        epsilon = 1e-5
        bs, dim = batch.shape
        loss = list()
        neg_losses = list()
        neg_bins = list()

        labels = labels.unsqueeze(1)
        bsame_labels = (labels.T == labels.view(-1,1)).to(batch.device).T
        bdiff_labels = (labels.T != labels.view(-1,1)).to(batch.device).T

        for i in range(batch_size):
            pos_ixs = copy.deepcopy(bsame_labels[i])
            pos_ixs[i] = False
            neg_ixs = copy.deepcopy(bdiff_labels[i])

            pos_pair_ = sim_mat[i][pos_ixs]
            pos_pair_ = pos_pair_[pos_pair_ < 1 - epsilon] # exclude sample itself

            if self.sampling_incl_pos:
                neg_pair_ = sim_mat[i]
                neg_pair_ = neg_pair_[neg_pair_ < 1 - epsilon] # exclude sample itself
            else:
                neg_pair_ = sim_mat[i][neg_ixs]

            if self.sampling_mode == 'max_min':
                neg_pair = neg_pair_[neg_pair_ + self.margin > min(pos_pair_)]
                pos_pair = pos_pair_[pos_pair_ - self.margin < max(neg_pair_)]

                if len(neg_pair) < 1 or len(pos_pair) < 1:
                    continue

            elif self.sampling_mode == 'distance':
                pos_pair = pos_pair_[np.random.choice(len(pos_pair_))] # choose random positive
                q_d_inv = self.inverse_sphere_cosine_distances(dim, neg_pair_)
                neg_pair = neg_pair_[np.random.choice(len(neg_pair_), p=q_d_inv)]

            elif 'pads' in self.sampling_mode:
                pos_pair = pos_pair_[np.random.choice(len(pos_pair_))] # choose random positive
                q_d_inv, bins = self.sampling_distr.sample(neg_pair_)

                idx_tmp = np.random.choice(len(neg_pair_), p=q_d_inv)
                neg_pair = neg_pair_[idx_tmp]
                neg_bins.append(bins[idx_tmp])

            # weighting step
            if self.beta_const:
                pos_loss = 1.0 / self.pos_weight * torch.log(
                    1 + torch.sum(torch.exp(-self.pos_weight * (pos_pair - self.pos_thresh)))) # make self.thresh learnable
                neg_loss = 1.0 / self.neg_weight * torch.log(
                    1 + torch.sum(torch.exp(self.neg_weight * (neg_pair - self.neg_thresh)))) # make self.thresh learnable
                loss.append(pos_loss + neg_loss)
            else:
                # learnable threshold
                pos_loss = 1.0 / self.pos_weight * torch.log(
                    1 + torch.sum(torch.exp(-self.pos_weight * (pos_pair - self.beta[labels[i]])))) # make self.thresh learnable
                neg_loss = 1.0 / self.neg_weight * torch.log(
                    1 + torch.sum(torch.exp(self.neg_weight * (neg_pair - self.beta[labels[i]])))) # make self.thresh learnable
                loss.append(pos_loss + neg_loss)
                neg_losses.append(neg_loss)

        if len(loss) == 0:
            return torch.zeros([], requires_grad=True)

        if self.sampling_distr:
            if 'random' in self.sampling_distr.update_type:
                if global_step > 0 and global_step % self.sampling_distr.update_step_size == 0:
                    self.sampling_distr.update_sample_distr()
            elif 'ema' in self.sampling_distr.update_type:
                change_info = {'losses': neg_losses, 'bins': neg_bins}
                self.sampling_distr.update_sample_distr(change_info)

        loss = sum(loss) / batch_size
        return loss

    def inverse_sphere_cosine_distances(self, dim, anchor_to_all_dists):
            dists  = anchor_to_all_dists

            #negated log-distribution of distances of unit sphere in dimension <dim>
            log_q_d_inv = (float(3 - dim) / 2) * torch.log(1.0 - dists.pow(2))
            q_d_inv = torch.exp(log_q_d_inv - torch.max(log_q_d_inv)) # - max(log) for stability

            q_d_inv = q_d_inv/q_d_inv.sum()
            return q_d_inv.detach().cpu().numpy()

    def get_log_data(self):
        if 'pads' in self.sampling_mode:
            return self.sampling_distr.log_sampling_distr()
        else:
            distr = self.inverse_sphere_cosine_distances(384, torch.range(0, 0.7, 0.77/25))
            support = np.linspace(0, 0.7, 24)
            distr_data = {'Sampling Distr': wandb.Histogram(np_histogram=(np.array(distr), np.array(support))),
                          'Log Sampling Distr': wandb.Histogram(np_histogram=(np.log(np.clip(np.array(distr), 1e-20, None)) - np.log(1e-20), np.array(support)))}

            return distr_data

class Sampling_Distribution():
    def __init__(self, n_support, support_limit, update_step_size, update_type, ema_alpha=0.25,
                 init_distr='random', init_distr_mu=0.6, init_distr_sigma=0.05, normalize_update=True,
                 filter_support_limit=False):

        self.n_support, self.support_limit = n_support, support_limit
        self.init_distr = init_distr
        self.init_distr_mu = init_distr_mu
        self.init_distr_sigma = init_distr_sigma
        self.normalize_update = normalize_update
        self.support = np.linspace(support_limit[0], support_limit[1], self.n_support)
        self.update_step_size = update_step_size
        self.update_type = update_type
        self.ema_alpha = ema_alpha

        self.no_norm_distr = self.init_sample_distr()
        self.distr = self.norm(self.no_norm_distr)

        self.distr_collect = {'support_limit': support_limit, 'n_support': n_support, 'progression': [], 'nonorm_progression': []}
        self.filter_support_limit = filter_support_limit

    def give_distr(self, name):
        if name   == 'random':
            distr_to_init = np.array([1.]*(self.n_support-1))
        elif name == 'distance':
            distr_to_init = self.probmode(self.support, self.n_support, upper_lim=0.5, mode='distance')
        elif name == 'uniform_low_and_tight':
            distr_to_init = self.probmode(self.support, self.n_support, mu=0.4, sig=0.1, mode='uniform')
        elif name == 'uniform_lowish_and_tight':
            distr_to_init = self.probmode(self.support, self.n_support, mu=0.6, sig=0.1, mode='uniform')
        elif name == 'heavyside_low':
            distr_to_init = self.probmode(self.support, self.n_support, mu=0.5,  mode='heavyside')
        elif name == 'uniform_low':
            distr_to_init = self.probmode(self.support, self.n_support, mu=0.5, sig=0.2, mode='uniform')
        elif name == 'uniform_avg':
            distr_to_init = self.probmode(self.support, self.n_support, mu=1.,  sig=0.2, mode='uniform')
        elif name == 'uniform_high':
            distr_to_init = self.probmode(self.support, self.n_support, mu=1.5, sig=0.2, mode='uniform')
        elif name == 'normal_low':
            # distr_to_init = self.probmode(self.support, self.n_support, mu=0.6, sig=0.05, mode='gaussian')
            distr_to_init = self.probmode(self.support, self.n_support, mu=self.init_distr_mu, sig=self.init_distr_sigma, mode='gaussian')
        elif name == 'normal_avg':
            distr_to_init = self.probmode(self.support, self.n_support, mu=1.1,  sig=0.04, mode='gaussian')
        elif name == 'normal_high':
            distr_to_init = self.probmode(self.support, self.n_support, mu=1.6, sig=0.04, mode='gaussian')
        else:
            raise Exception('Init. Distr. >> {} << not available!'.format(name))
        return distr_to_init

    def norm(self, distr):
        return distr/np.sum(distr)

    def init_sample_distr(self):
        if 'random' in self.update_type:
            no_norm_distr = np.random.choice(2, self.n_support - 1) + 0.05
        elif 'rl' in self.update_type:
            no_norm_distr = self.give_distr(self.init_distr)
        elif 'ema' in self.update_type:
            # no_norm_distr = np.array([1.]*(self.n_support-1))
            no_norm_distr = self.give_distr(self.init_distr)

        return no_norm_distr

    def update_sample_distr(self, change_info=None):

        if 'random' in self.update_type:
            self.no_norm_distr = np.random.choice(2, self.n_support) + 0.05
        elif 'rl' in self.update_type:
            self.no_norm_distr = np.clip(self.no_norm_distr * change_info['distr_change'], 1e-25, 1e25)
        elif 'ema' in self.update_type:
            distr_change = np.zeros_like(self.no_norm_distr)
            counter = np.zeros_like(self.no_norm_distr)
            for i,val in zip(change_info['bins'], change_info['losses']):
                distr_change[i] += val.cpu().detach().numpy()
                counter[i] += 1

            if self.normalize_update:
                counter[counter == 0] = 1
                distr_change = distr_change / counter
            self.no_norm_distr = (1-self.ema_alpha) * self.no_norm_distr + self.ema_alpha * distr_change
            self.no_norm_distr = np.clip(self.no_norm_distr, 1e-25, 1e25)

        # normalize
        self.distr = self.norm(self.no_norm_distr)

        # save history
        self.distr_collect['progression'].append(copy.deepcopy(self.distr))
        self.distr_collect['nonorm_progression'].append(copy.deepcopy(self.no_norm_distr))

    def sample(self, distances):
        p_assigns = np.sum((distances.cpu().detach().numpy().reshape(-1)>self.support[1:-1].reshape(-1,1)).T,axis=1).reshape(distances.shape)
        sample_p = self.distr[p_assigns]

        # if not self.filter_support_limit:
        #     outside_support_lim = (distances.cpu().numpy().reshape(-1)<self.support_limit[0]) * (distances.cpu().numpy().reshape(-1)>self.support_limit[1])
        #     outside_support_lim = outside_support_lim.reshape(distances.shape)
        #     sample_ps[outside_support_lim] = 0

        # if self.include_pos:
        #     sample_p = sample_ps[i]
        #     sample_p = sample_p/sample_p.sum()
        #     negatives.append(np.random.choice(bs,p=sample_p))
        # else:

        sample_p = sample_p/sample_p.sum()
        # neg = np.random.choice(np.arange(len(distances)), p=sample_p)

        return sample_p, p_assigns

    def log_sampling_distr(self):
        distr_data = dict()
        distr_data['Sampling Distr'] = wandb.Histogram(np_histogram=(np.array(self.distr),np.array(self.support)))
        distr_data['Log Sampling Distr'] = wandb.Histogram(np_histogram=(np.log(np.clip(np.array(self.distr), 1e-20, None))-np.log(1e-20),np.array(self.support)))

        return distr_data

    def gaussian(self, x, mu=1, sig=1):
        return 1/np.sqrt(2*np.pi*sig)*np.exp(-(x-mu)**2/(2*sig**2))

    def uniform(self, x, mu=1, sig=0.25):
        sp = x.copy()
        sp[(x>=mu-sig) * (x<=mu+sig)] = 1
        sp[(x<mu-sig) + (x>mu+sig)]   = 0
        return sp

    def distance(self, x, upper_lim):
        sp = x.copy()
        sp[x<=upper_lim] = 1
        sp[x>=upper_lim] = 0
        return sp

    def heavyside(self, x, mu):
        sp = x.copy()
        sp[x<=mu] = 1
        sp[x>mu]  = 0
        return sp

    def probmode(self, support_space, support, mu=None, sig=None, upper_lim=None, mode='uniform'):
        if mode=='uniform':
            probdist = self.uniform(support_space[:-1]+2/support, mu=mu, sig=sig)
        elif mode=='gaussian':
            probdist = self.gaussian(support_space[:-1]+2/support, mu=mu, sig=sig)
        elif mode=='distance':
            probdist = self.distance(support_space[:-1]+2/support, upper_lim)
        elif mode=='heavyside':
            probdist = self.heavyside(support_space[:-1]+2/support, mu=mu)
        probdist = np.clip(probdist, 1e-15, 1)
        probdist = probdist/probdist.sum()
        return probdist


from django.urls import path
from . import views

urlpatterns = [
    path('', views.Home, name='home'),
    path('landing/', views.Landing.as_view(), name='landing'),
    path('login/', views.loginPage, name="login"),
    path('logout/', views.logoutPage, name="logout"),
    # path('signup/', SignupPageView.as_view(), name="home"),
]


from django.core.cache import cache
from django.db.models import Q
from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext

from soccer.games.models import Game
from soccer.teams.models import Team
from soccer.stats.models import SeasonStat

def index(request):
    """A list of all teams in the database."""

    teams = Team.reals.order_by("short_name")
    ctx = {"teams": teams}
    return render_to_response("teams/index.html",
                              ctx,
                              context_instance=RequestContext(request)
                              )

def defunct(request):
    """A list of all teams in the database."""

    teams = Team.defuncts.order_by("short_name")
    ctx = {"teams": teams}
    return render_to_response("teams/index.html",
                              ctx,
                              context_instance=RequestContext(request)
                              )

def team_schedule(request, slug):
    """
    Recent games played by this team.
    """
    games = Game.objects.filter(Q(home_team__slug=slug) | Q(away_team__slug=slug)
                                ).order_by("-date")

    context = {
        'games': games,
        }

    return render_to_response('teams/schedule.html',
                              context,
                              context_instance=RequestContext(request))

def team_schedule_year(request, slug, year):
    """
    Recent games played by this team.
    """
    games = Game.objects.filter(Q(home_team__slug=slug) | Q(away_team__slug=slug),
                                date__year=year).order_by("-date")
    context = {
        'games': games,
        }

    return render_to_response('teams/schedule.html',
                              context,
                              context_instance=RequestContext(request))

def team_and_year(request, id, year):
    """Detailed information for a team for a given year.
    Contains player stats and game details."""
    # Game details not yet implemented.

    team = Team.objects.get(id=id)
    stats = SeasonStat.objects.filter(team=team, year=year)
    context = {
        'team': team,
        'stats': stats,
        }
    return render_to_response("teams/team_and_year.html",
                              context,
                              context_instance=RequestContext(request)
                              )

def team_detail(request, slug):
    team = get_object_or_404(Team, slug=slug)
    years = team.years_with_stats()

    # Whoa whoa whoa what is this stuff?
    minutes = {}
    games = {}
    goals = {}
    assists = {}

    stats = SeasonStat.objects.filter(team=team)
    for stat in stats:
        player = stat.player
        if player not in minutes:
            minutes[player] = games[player] = goals[player] = assists[player] = 0
        minutes[player] += stat.minutes
        games[player] += stat.games_played
        goals[player] += stat.goals
        assists[player] += stat.assists

    sort_leaders = lambda d: sorted(d.items(), key=lambda e: -e[1])

    context = {
        'team': team,
        'years': years,
        "minutes": sort_leaders(minutes)[:10],
        "games": sort_leaders(games)[:10],
        "goals": sort_leaders(goals)[:10],
        "assists": sort_leaders(assists)[:10],
        }

    return render_to_response("teams/team.html",
                              context,
                              context_instance=RequestContext(request)
                              )


#! /usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import division, print_function, absolute_import
no_lcd = True

if not no_lcd:
    from st7735_ijl20.st7735 import ST7735
import os
import io
from timeit import time
import warnings
import sys
import argparse
import numpy as np
import signal
import picamera
from picamera import Color
from PIL import ImageDraw
from PIL import ImageFont
from PIL import ImageColor
from PIL import Image
#from yolo import YOLO
#from yolo_v3_tflite import YOLO_TFLITE
from ssd_mobilenet import SSD_MOBILENET
from intersection import any_intersection, intersection

from deep_sort import preprocessing
from deep_sort import nn_matching
from deep_sort.detection import Detection
from deep_sort.tracker import Tracker
from tools import generate_detections as gdet
from deep_sort.detection import Detection as ddet
warnings.filterwarnings('ignore')

class SigTerm(SystemExit): pass
def sigterm(sig,frm): raise SigTerm
signal.signal(15,sigterm)

FONT_SMALL_LCD = ImageFont.truetype('fonts/truetype/freefont/FreeSansBold.ttf', 10)
FONT_SMALL_PRV = ImageFont.truetype('fonts/truetype/freefont/FreeSansBold.ttf', 40)
FONT_LARGE_LCD = ImageFont.truetype('fonts/truetype/freefont/FreeSansBold.ttf', 12)
FONT_LARGE_PRV = ImageFont.truetype('fonts/truetype/freefont/FreeSansBold.ttf', 48)

FONT_SMALL = {'lcd': FONT_SMALL_LCD, 'prv': FONT_SMALL_PRV}
FONT_LARGE = {'lcd': FONT_LARGE_LCD, 'prv': FONT_LARGE_PRV}

DISPLAY_WIDTH = 160                # LCD panel width in pixels
DISPLAY_HEIGHT = 128               # LCD panel height

CAMERA_WIDTH = 640
CAMERA_HEIGHT = 480

def clamp(minvalue, value, maxvalue):
    return max(minvalue, min(value, maxvalue))

def main():
    ##################################################
    # Initialise LCD
    ## LCD = LCD_1in8.LCD()
    if not no_lcd:
        LCD = ST7735()
        ## Lcd_ScanDir = LCD_1in8.SCAN_DIR_DFT
        ## LCD.LCD_Init(Lcd_ScanDir)
        LCD.begin()
        ## screenbuf = Image.new("RGB", (LCD.LCD_Dis_Column, LCD.LCD_Dis_Page), "WHITE")
        screenbuf = Image.new("RGBA", (DISPLAY_WIDTH, DISPLAY_HEIGHT), (0,0,0,0))
        drawLCD = ImageDraw.Draw(screenbuf)
    else:
        drawLCD = None
    previewbuf = Image.new("RGBA", (CAMERA_WIDTH, CAMERA_HEIGHT), (0,0,0,0))
    drawPRV = ImageDraw.Draw(previewbuf)
    ratios = np.array([DISPLAY_WIDTH/CAMERA_WIDTH, DISPLAY_HEIGHT/CAMERA_HEIGHT],dtype=float)

    # draw* functions work in CAMERA coordinates
    def drawtext(xy, txt, fill = None, font = FONT_SMALL):
        nonlocal drawLCD, drawPRV
        if drawLCD is not None:
            xyl = list(np.int32(np.array(xy) * ratios))
            drawLCD.text(xyl, txt, fill=fill, font=font['lcd'])
        if drawPRV is not None:
            drawPRV.text(list(np.array(xy,dtype=int).reshape(-1)), txt, fill=fill, font=font['prv'])

    def drawline(pts, fill = None, width = 0):
        nonlocal drawLCD, drawPRV
        if drawLCD is not None:
            ptsl = list(np.int32(np.array(pts).reshape(-1,2) * ratios).reshape(-1))
            drawLCD.line(ptsl, fill = fill, width = width)
        if drawPRV is not None:
            drawPRV.line(list(np.array(pts,dtype=int).reshape(-1)), fill = fill, width = width)

    def drawrect(pts, outline = None, fill = None):
        nonlocal drawLCD, drawPRV
        if drawLCD is not None:
            ptsl = list(np.int32(np.array(pts).reshape(-1,2) * ratios).reshape(-1))
            drawLCD.rectangle(ptsl, fill = fill, outline = outline)
        if drawPRV is not None:
            drawPRV.rectangle(list(np.array(pts,dtype=int).reshape(-1)), fill = fill, outline = outline)

    def textsize(txt, font = FONT_SMALL):
        return font['prv'].getsize(txt)

    drawtext((100, 100), 'Initialising...', fill = "BLUE", font = FONT_LARGE)
    if not no_lcd:
        LCD.display(screenbuf) # force initial drawing on LCD

    ##################################################
    parser = argparse.ArgumentParser()
    parser.add_argument('--line', '-L', help="counting line: x1,y1,x2,y2",
                        default=None)
    parser.add_argument('--model', help='File path of .tflite file.', required=True)
    parser.add_argument('--labels', help='File path of labels file.', required=True)
    parser.add_argument('--no-preview', help='Disable picamera preview', required=False,
                        action='store_true')
    args = parser.parse_args()
    if args.no_preview:
        drawPRV = None

    objd = SSD_MOBILENET(wanted_label='person', model_file=args.model, label_file=args.labels)
    basedir = os.getenv('DEEPSORTHOME','.')

   # Definition of the parameters
    max_cosine_distance = 0.3
    nn_budget = None
    nms_max_overlap = 1.0

   # deep_sort
    model_filename = '{}/model_data/mars-small128.pb'.format(basedir)
    encoder = gdet.create_box_encoder(model_filename,batch_size=1)

    metric = nn_matching.NearestNeighborDistanceMetric("cosine", max_cosine_distance, nn_budget)
    tracker = Tracker(metric,max_iou_distance=0.7,max_age=10)
    delcount=0
    intcount=0
    poscount=0
    negcount=0
    db = {}
    def check_track(i):
        nonlocal delcount
        nonlocal ratios
        nonlocal cameracountline
        if i in db and len(db[i]) > 1:
            if any_intersection(cameracountline[0], cameracountline[1], np.array(db[i])):
                delcount+=1
                print("delcount={}".format(delcount))
            db[i] = []

    if args.line is None:
        w, h = DISPLAY_WIDTH, DISPLAY_HEIGHT
        countline = np.array([[w/2,0],[w/2,h]],dtype=int)
    else:
        countline = np.array(list(map(int,args.line.strip().split(','))),dtype=int).reshape(2,2)

    cameracountline = countline.astype(float) / ratios

    fps = 0.0
    frameTime = time.time()*1000
    with picamera.PiCamera(resolution=(CAMERA_WIDTH, CAMERA_HEIGHT), framerate=30) as camera:
        if not args.no_preview:
            camera.start_preview(alpha=255)
            camera.annotate_foreground = Color('black')
            camera.annotate_background = Color('white')
            previewarray = np.array(previewbuf.resize((CAMERA_WIDTH, CAMERA_HEIGHT),
                                   Image.ANTIALIAS))
            overlay = camera.add_overlay(previewarray.tobytes(), layer=3, alpha=64)
        try:
            stream = io.BytesIO()
            for _ in camera.capture_continuous(stream, format='jpeg', use_video_port=True):
                stream.seek(0)

                image = Image.open(stream)
                t1 = time.time()
                t1objd = time.time()
                boxs = objd.detect_image(image)
                # boxs is in tlwh format
                t2objd = time.time()

                stream.seek(0)
                stream.truncate()

                # set background image and clear screen
                t1bclr = time.time()
                # clear both screens
                drawrect([0,0,CAMERA_WIDTH,CAMERA_HEIGHT], fill=0, outline=0)
                # also blit camera image onto LCD screen
                if not no_lcd:
                    screenbuf.paste(image.resize((DISPLAY_WIDTH, DISPLAY_HEIGHT)))

                t2bclr = time.time()

                # features and tracking get more expensive as there are more things to track
                # writes to disk are fairly substantial as well

                # print("box_num",len(boxs))
                t1feat = t2objd
                frame = np.array(image)
                features = encoder(frame,boxs)
                t2feat = time.time()

                # score to 1.0 here).
                detections = [Detection(bbox, 1.0, feature) for bbox, feature in zip(boxs, features)]

                # Run non-maxima suppression.
                t1prep = time.time()
                boxes = np.array([d.tlwh for d in detections])
                scores = np.array([d.confidence for d in detections])
                indices = preprocessing.non_max_suppression(boxes, nms_max_overlap, scores)
                detections = [detections[i] for i in indices]
                t2prep = time.time()

                # Call the tracker
                t1trac = time.time()
                tracker.predict()
                tracker.update(detections)
                a = (cameracountline[0,0], cameracountline[0,1])
                b = (cameracountline[1,0], cameracountline[1,1])
                drawline([a, b], fill=(0,0,255), width=3)

                for track in tracker.deleted_tracks:
                    i = track.track_id
                    if track.is_deleted():
                        check_track(track.track_id)

                for track in tracker.tracks:
                    i = track.track_id
                    if not track.is_confirmed() or track.time_since_update > 1:
                        continue
                    if i not in db:
                        db[i] = []
                    db[i].append(track.mean[:2].copy())
                    if len(db[i]) > 1:
                        pts = (np.array(db[i]).reshape((-1,1,2))).reshape(-1)
                        drawline(pts, fill=(255,0,255), width=3)
                        p1 = cameracountline[0]
                        q1 = cameracountline[1]
                        p2 = np.array(db[i][-1])
                        q2 = np.array(db[i][-2])
                        cp = np.cross(q1 - p1,q2 - p2)
                        if intersection(p1,q1,p2,q2):
                            intcount+=1
                            print("track_id={} just intersected camera countline; cross-prod={}; intcount={}".format(i,cp,intcount))
                            drawline(pts[-4:], fill=(0,0,255), width=5)
                            if cp >= 0:
                                poscount+=1
                            else:
                                negcount+=1

                    bbox = track.to_tlbr()
                    drawrect(bbox,outline=(255,255,255))
                    drawtext(bbox[:2],str(track.track_id), fill=(0,255,0), font=FONT_SMALL)

                for det in detections:
                    bbox = det.to_tlbr()
                    drawrect(bbox,outline=(255,0,0))

                t2trac = time.time()

                t1draw = time.time()

                # draw counters along bottom of screen(s)
                (_, dy) = textsize(str(negcount), font = FONT_LARGE)
                drawtext((0, CAMERA_HEIGHT-dy), str(negcount), fill=(255,0,0), font=FONT_LARGE)
                (dx, dy) = textsize(str(abs(negcount-poscount)), font = FONT_LARGE)
                drawtext(((CAMERA_WIDTH-dx)/2, CAMERA_HEIGHT-dy), str(abs(negcount-poscount)), fill=(0,255,0), font=FONT_LARGE)
                (dx, dy) = textsize(str(poscount), font = FONT_LARGE)
                drawtext((CAMERA_WIDTH-dx, CAMERA_HEIGHT-dy), str(poscount), fill=(0,0,255), font=FONT_LARGE)
                if not no_lcd:
                    LCD.display(screenbuf) #50ms (RPi3)
                if not args.no_preview:
                    previewarray = np.array(previewbuf) #13ms (RPi3)
                    overlay.update(previewarray.tobytes()) #12ms (RPi3)
                t2draw = time.time()
                t2 = time.time()

                print("Frame processing time={:.0f}ms (objd={:.0f}ms prep={:.0f}ms feat={:.0f}ms trac={:.0f}ms draw={:.0f}ms)".format(1000*(t2 - t1), 1000*(t2objd - t1objd), 1000*(t2prep - t1prep), 1000*(t2feat - t1feat), 1000*(t2trac - t1trac), 1000*(t2draw - t1draw)))

            for track in tracker.tracks:
                check_track(track.track_id)

            print(delcount)

        finally:
            if not no_lcd:
                LCD.LCD_Clear()
            if not args.no_preview:
                camera.stop_preview()

if __name__ == '__main__':
    main()


#!/usr/bin/env python
import rospy
import time
import roslib; roslib.load_manifest('ur_driver')
import actionlib
from control_msgs.msg import *
from trajectory_msgs.msg import JointTrajectory
from trajectory_msgs.msg import JointTrajectoryPoint
from sensor_msgs.msg import JointState
from math import *

JOINT_NAMES = ['shoulder_pan_joint', 'shoulder_lift_joint', 'elbow_joint',
               'wrist_1_joint', 'wrist_2_joint', 'wrist_3_joint']

STARTING_POSITION = [radians(90), radians(-135), radians(-70), radians(-100), radians(90), 0]
#STARTING_POSITION = [radians(90), radians(-100), radians(-70), radians(-100), radians(0), 0]

client = None
joint_speed_pub = None
trajectory = JointTrajectory()

"""
def ThrowTrajectory_movement():

	rate = rospy.Rate(130)
	trajectory = JointTrajectory()

	trajectory.header.stamp=rospy.Time.now()
	trajectory.joint_names = JOINT_NAMES

	count = 0
	i = 0

	# working speed = 2, wrist_speed = 1.5
	# working speed = 2.5, wrist_speed = 2
	# working speed = 3, wrist_speed = 2.5
	# working speed = 4.5, wrist_speed = 2.5

	speed = 4.5
	wrist_speed = 2.5

	while True:
		if count < 100:
			trajectory.points = [JointTrajectoryPoint(positions=[0]*6, velocities=[0, 0, wrist_speed, speed, 0, 0])]
		else:
			speed -= 0.06
			if wrist_speed > 0.04:
				wrist_speed -= 0.04
			else:
				wrist_speed = 0
			trajectory.points = [JointTrajectoryPoint(positions=[0]*6, velocities=[0, 0, wrist_speed, speed, 0, 0])]
		joint_speed_pub.publish(trajectory)
		count += 1

	  	if speed < 0.05:
			break

        rate.sleep()

def BackHandThrowTrajectory():

	rate = rospy.Rate(130)
	trajectory = JointTrajectory()

	trajectory.header.stamp=rospy.Time.now()
	trajectory.joint_names = JOINT_NAMES

	count = 0

	speed = -2
	wrist_speed = -1.5

	while True:
		if count < 300:
			trajectory.points = [JointTrajectoryPoint(positions=[0]*6, velocities=[0, 0, wrist_speed, speed, 0, 0])]
		else:
			speed += 0.06
			if wrist_speed < -0.04:
				wrist_speed += 0.04
			else:
				wrist_speed = 0
			trajectory.points = [JointTrajectoryPoint(positions=[0]*6, velocities=[0, 0, wrist_speed, speed, 0, 0])]
		joint_speed_pub.publish(trajectory)
		count += 1

	  	if speed > -0.05:
			break

        rate.sleep()
"""

def MoveToStartPosition():
    global joints_pos
    g = FollowJointTrajectoryGoal()
    g.trajectory = JointTrajectory()
    g.trajectory.joint_names = JOINT_NAMES
    try:
        joint_states = rospy.wait_for_message("joint_states", JointState)
        joints_pos = joint_states.position
        g.trajectory.points = [
        JointTrajectoryPoint(positions=joints_pos, velocities=[0]*6, time_from_start=rospy.Duration(3.0)),
        JointTrajectoryPoint(positions=STARTING_POSITION ,velocities=[0]*6, time_from_start=rospy.Duration(6.0))]

        client.send_goal(g)
        client.wait_for_result()
    except:
        raise

def ThrowTrajectory(rate):
    trajectory.header.stamp=rospy.Time.now()
    trajectory.joint_names = JOINT_NAMES

    count = 0
    i = 0
    speed = 1.5
    wrist_speed = 2

    while True:
        if count < 100:
            trajectory.points = [JointTrajectoryPoint(positions=[0]*6, velocities=[0, 0, wrist_speed, speed, 0, 0])]
        else:
            speed -= 0.04
            if wrist_speed > 0.04:
                wrist_speed -= 0.04
            else:
                wrist_speed = 0
            trajectory.points = [JointTrajectoryPoint(positions=[0]*6, velocities=[0, 0, wrist_speed, speed, 0, 0])]
        joint_speed_pub.publish(trajectory)
        count += 1

        if speed < 0.05:
            break

        rate.sleep()

def main():
    global client, joint_speed_pub

    # create node
    rospy.init_node("throwing_trajectory_planning", anonymous=True, disable_signals=True)

    # create the client to move the arm to the starting position
    client = actionlib.SimpleActionClient('follow_joint_trajectory', FollowJointTrajectoryAction)

    print("Waiting for server...")
    client.wait_for_server()
    print("Connected to server")

    # create publisher to publish to the ur_driver/joint_speed
    joint_speed_pub = rospy.Publisher('ur_driver/joint_speed',JointTrajectory, queue_size=30)
    rate = rospy.Rate(125)

    #---------------Moving the arm------------------#
    MoveToStartPosition()

    time.sleep(3)

    #ThrowTrajectory(rate)

if __name__ == '__main__': main()


# ------------------------------------------------------------------------ #
# Title: Assignment 08
# Description: Working with classes

# ChangeLog (Who,When,What):
# RRoot,1.1.2030,Created started script
# RRoot,1.1.2030,Added pseudo-code to start assignment 8
# JIn,12/4/19,Modified code to complete assignment 8
# ------------------------------------------------------------------------ #

# Data -------------------------------------------------------------------- #
strFileName = 'products.txt'
lstOfProductObjects = []

class Product:
    """Stores data about a product:
    properties:
        product_name: (string) with the products's  name
        product_price: (float) with the products's standard price
    methods:
    changelog: (When,Who,What)
        RRoot,1.1.2030,Created Class
        Jin,12/4/19,Modified code to complete assignment 8
    """
    # --Fields--
    # --Constructor--
    def __init__(self, product_name: str, product_price: float):
        # --Attribute --
        self.__product_name = product_name
        self.__product_price = product_price
    #--Properties--
    @property
    #product name
    def product_name(self):
        return str(self.__product_name).title()

    @product_name.setter
    def product_name(self, name):
        if str(name).isnumeric() == False:
            self.__product_name = name
        else:
            raise Exception("Product Name cannot be numbers")

    #product_price
    @property
    def product_price(self):
        return float(self.__product_price).title()

    @product_price.setter
    def product_price(self, value):
        if float(value).isnumeric() == True:
            self.__product_price = value
        else:
            raise Exception("Product Price has to be numbers.")

# Data -------------------------------------------------------------------- #

# Processing  ------------------------------------------------------------- #
class FileProcessor:
    """Processes data to and from a file and a list of product objects:

    methods:
        save_data_to_file(file_name, list_of_product_objects):

        read_data_from_file(file_name): -> (a list of product objects)

    changelog: (When,Who,What)
        RRoot,1.1.2030,Created Class
        Jin,12/4/2019,Modified code to complete assignment 8
    """
    #methods:
    @staticmethod
    def read_data_from_file(file_name):
        """
        :param file_name: (string) with name of file:
        :return: list_of_products: (list) of products
        """
        list_of_products = []
        objFile = open(file_name, "r")
        for line in objFile:
            row = line.split(",")
            list_of_products.append(row)
        objFile.close()
        return list_of_products

    @staticmethod
    def save_data_to_file(file_name, list_of_product_objects):
        """
        :param file_name: (string) with name of file:
        :param list_of_product_objects: (list) you want to write into the file:
        """
        objFile = open(file_name,"w")
        for product in list_of_product_objects:
            objFile.write(product[0] + ","+product[1]+"\n")
        objFile.close()
        return "Data stored to file"

# Processing  ------------------------------------------------------------- #

# Presentation (Input/Output)  -------------------------------------------- #
class IO:
    """
    desc: class for input/output
    methods includes:
        print_menu(): show menu to user and ask for user choice
        print_data(list_of_products): prints out the current product list for user
        add_product(): add more product data from user
    """
    pass
    @staticmethod
    def print_menu():
        '''
        desc: method to print menu to display to user and return user choice
        :return: menu_choice: (str) user choice from menu
        '''
        print('''
        Menu:
        1) Show current data from file
        2) Add a new product and price
        3) Save Data
        4) Exit

        ''')
        menu_choice = input("What would you like to do?")
        return menu_choice

    @staticmethod
    def print_data(list_of_products):
        '''
        desc: prints out data from the list of products
        :param list_of_products: products to be displayed
        '''
        for product_data in list_of_products:
            print(product_data[0] + "  $"+product_data[1])
        print()

    @staticmethod
    def add_product():
        '''
        desc: asks user for product details and return that.
        :return:  product with the user input
        '''
        product_name = input("What's the name of the product you would like to add?")
        product_price = input("What's the price of the product you would like to add?")
        try:
            if product_price.isnumeric() == True:
                product=[product_name, product_price]
                return product
            else:
                raise Exception("Please Enter number for products")
        except Exception as e:
            print(e)

# Presentation (Input/Output)  -------------------------------------------- #

# Main Body of Script  ---------------------------------------------------- #
# Load data from file into a list of product objects when script starts
# Show user a menu of options
# Get user's menu option choice
    # Show user current data in the list of product objects
    # Let user add data to the list of product objects
    # let user save current data to file and exit program
lstOfProductObjects = FileProcessor.read_data_from_file(strFileName)
while True:
    user_choice = IO.print_menu()

    if user_choice =="1":
        IO.print_data(lstOfProductObjects)
        continue
    elif user_choice =="2":
        lstOfProductObjects.append(IO.add_product())
        continue
    elif user_choice =="3":
        FileProcessor.save_data_to_file(strFileName,lstOfProductObjects)
        continue
    elif user_choice == "4":
        break
    else:
        print("Please choose between 1-4.")
        continue

# Main Body of Script  ---------------------------------------------------- #


a = dict()
li = list()
cont = 0
soma = 0
mulheres = list()
while True:
    print('-+' * 40)
    a['Nome'] = str(input('Digite o seu nome: '))
    a['Idade'] = int(input('Digite a sua idade: '))
    a['Sexo'] = str(input('Digite o seu sexo[M/F]: ')).upper()
    if a['Sexo'] in 'F':
        mulheres.append(a['Nome'])
    li.append(a.copy())
    a.clear()
    b = str(input('Quer continuar?[S/N]: ')).upper()
    if b in 'Nn':
        break
print('-+' * 40)
for pe in li:
    cont += 1
    soma += pe['Idade']
media = soma/cont
print(f'Temos {cont} pessoas cadastradas')
print(f'A média de idade do grupo é igual a {media}')
print(f'As mulheres cadastradas foram: {mulheres}')
print('-+' * 40)
print('LISTA DE PESSOAS COM IDADE ACIMA DA MÉDIA')
print('-+' * 40)
for x in li:
    if x['Idade'] > media:
        print(f'Nome = {x["Nome"]}; Idade = {x["Idade"]}; Sexo = {x["Sexo"]}')
        print('')
print('-+' * 40)
print('Fim')
print('-+' * 40)


# Printe o nome do Ator que ganhou o Oscar em 1993.

# importando a biblioteca do pandas
import pandas as pd

# lendo e salvando o arquivo em uma variavel
arquivo = pd.read_csv('CSV.csv', encoding='UTF-8', sep=',')

# separando o nome do ator em uma variavel e pritando ele
ano = arquivo['Name'][arquivo['Year'] == 1993]
print(ano.to_string(index=True))

Exercícios Manipulação de Arquivos Python/Exercício 11.py


import tensorflow as tf
import os

def linear_regration():
    tf.compat.v1.disable_eager_execution()
    # 1.准备数据
    with tf.compat.v1.variable_scope("prepare_data"):
        X = tf.compat.v1.random_normal(shape=(1000, 1), name="data")
        y_true = tf.matmul(X, [[0.8]]) + 0.7

    # 2.构造模型
    with tf.compat.v1.variable_scope("create_model"):
        weights = tf.Variable(initial_value=tf.compat.v1.random_normal(shape=(1, 1)), name="Weight")
        bias = tf.Variable(initial_value=tf.compat.v1.random_normal(shape=(1, 1)), name="Bias")
        y_predict = tf.matmul(X, weights) + bias

    # 3.构造误差函数
    with tf.compat.v1.variable_scope("cost_function"):
        error = tf.reduce_mean(tf.square(y_true - y_predict))

    # 4.优化误差
    with tf.compat.v1.variable_scope("optimizer"):
        optimizer = tf.compat.v1.train.GradientDescentOptimizer(learning_rate=0.01).minimize(error)

    # 2_收集变量
    tf.compat.v1.summary.scalar("error", error)
    tf.compat.v1.summary.histogram("weight", weights)
    tf.compat.v1.summary.histogram("bias", bias)

    # 3_合并变量
    merged = tf.compat.v1.summary.merge_all()

    # 显示初始化变量
    init = tf.compat.v1.global_variables_initializer()

    with tf.compat.v1.Session() as sess:
        # 初始化变量
        sess.run(init)

        # 1_创建事件文件
        file_writer = tf.compat.v1.summary.FileWriter('./tmp/linear', sess.graph)

        # 创建Saver对象
        saver = tf.compat.v1.train.Saver()
        # 查看初始化模型参数后的值
        print("训练前模型参数为：权重{}, 偏置{}, 误差{}".format(weights.eval(), bias.eval(), error.eval()))

        for i in range(1000):
            sess.run(optimizer)
            print("训练后模型参数为：权重{}, 偏置{}, 误差{}".format(weights.eval(), bias.eval(), error.eval()))

            # 运行合并变量操作
            summer = sess.run(merged)
            file_writer.add_summary(summer, i)

            # 保存模型
            if i % 10 == 0:
                saver.save(sess, './tmp/model/my_linear.ckpt')
        # 加载模型
        # if os.path.exists("./tmp/model/checkpoint"):
        #     saver.restore(sess, "./tmp/model/my_linear.ckpt")
        # print("训练后模型参数为：权重{}, 偏置{}, 误差{}".format(weights.eval(), bias.eval(), error.eval()))

    return None

if __name__ == '__main__':
    linear_regration()


#
# @lc app=leetcode id=349 lang=python3
#
# [349] Intersection of Two Arrays
#

# @lc code=start
class Solution:
    def intersection(self, nums1: List[int], nums2: List[int]) -> List[int]:
        res = []
        nums1.sort()
        nums2.sort()
        c1, c2 = 0, 0
        while c1 < len(nums1) and c2 < len(nums2):
            if nums1[c1] == nums2[c2] and nums1[c1] not in res:
                res.append(nums1[c1])
                c1 += 1
                c2 += 1
            elif nums1[c1] > nums2[c2]:
                c2 += 1
            else:
                c1 += 1
        return res

# @lc code=end


import os
import ast
import pandas as pd
import numpy as np
import allennlp_models.pair_classification
import spacy
#import neuralcoref
import time
import torch
import json
from copy import deepcopy
from allennlp.predictors.predictor import Predictor
from sentence_transformers import SentenceTransformer
from scipy.spatial import distance
from sklearn.metrics.pairwise import cosine_similarity
from transformers import pipeline
from collections import defaultdict
from nltk import word_tokenize
from bert_score import score as bert_score

from pair_extraction.model.seq2seq import Seq2SeqModel

def abs_cosine_distance(X, Y):
    return 1 - abs(distance.cosine(X, Y) - 1)

def mean_pooling(model_output, attention_mask):
    token_embeddings = model_output[0] #First element of model_output contains all token embeddings
    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
    sum_embeddings = torch.sum(token_embeddings * input_mask_expanded, 1)
    sum_mask = torch.clamp(input_mask_expanded.sum(1), min=1e-9)
    return sum_embeddings / sum_mask

def extract_to_file(tokenized_reviews, output_path):
    review_pair_id = 0
    with open(os.path.join(output_path, 'ReviewRebuttalPredict.txt'), 'w') as output_file:
        for i in range(len(tokenized_reviews)):
            # Get dummy labels for all sentences in a review
            review = tokenized_reviews[i]
            for sentence in review:
                if sentence == ' ' or sentence == '\t':
                    continue
                row = ''
                row += sentence.replace('\n', '[line_break_token]') + '\t'
                row += 'O' + '\t'
                row += 'O' + '\t'
                row += 'Review' + '\t'
                row += str(review_pair_id)
                row += '\n'
                output_file.write(row)
            #output_file.write('\n')
            # Get all combinations of reviews with other reviews
            for j in range(len(tokenized_reviews)):
                if j == i:
                    continue
                other_review = tokenized_reviews[j]
                for k, sentence in enumerate(other_review):
                    if sentence == ' ' or sentence == '\t':
                        continue
                    row = ''
                    row += sentence.replace('\n', '[line_break_token]') + '\t'
                    row += 'O' + '\t'
                    row += 'O' + '\t'
                    row += 'Reply' + '\t'
                    row += str(review_pair_id)
                    if k != len(other_review)-1 or j != len(tokenized_reviews)-2 or i != len(tokenized_reviews)-1:
                        row += '\n'
                    output_file.write(row)
            if i != len(tokenized_reviews)-1:
                output_file.write('\n')
            review_pair_id += 1

# map (source_review_id, target_review_id, target_sentence id) -> B/I/O
# map (source_review_id, target_review_id, target_sentence id) -> [pair_labels]
def extract_mappings_from_results(result_path, n_reviews, review_lengths):
    label_mapping = {}
    pair_mapping = {}
    with open(result_path, 'r') as f:
        all_reviews = f.read().split('\n\n')
    current_entry_idx = 0
    for i in range(n_reviews):
        for j in ([i] + [x for x in range(n_reviews) if x != i]):
            for k, entry in enumerate(all_reviews[current_entry_idx].split('\n')):
                if entry == '':
                    continue
                entry_data = entry.split('\t')
                preimage = (i, j, k)
                label_mapping[str(preimage)] = entry_data[3].split('-')[0]
                raw_pairs = ast.literal_eval(entry_data[5])
                raw_head = raw_pairs[:review_lengths[i]]
                raw_pairs = raw_pairs[review_lengths[i]:]
                bound_idx = sum([x for x in review_lengths[:i]])
                raw_pairs[bound_idx:bound_idx] = raw_head
                pair_mapping[str(preimage)] = raw_pairs
        current_entry_idx += 1
    return label_mapping, pair_mapping

# map (source_review_id, target_review_id, target_sentence id) -> [pair_labels]
def extract_mappings_from_sentence_embeddings(tokenized_reviews, thresh=0.6, topk=None, model_name='paraphrase-distilroberta-base-v1'):
    #model = SentenceTransformer('paraphrase-distilroberta-base-v1')
    #model = SentenceTransformer('allenai/scibert_scivocab_uncased')
    #model = SentenceTransformer('allenai/specter')
    model = SentenceTransformer(model_name)

    embeddings = {} # map (review_id, sent_id) -> embedding
    for i, review in enumerate(tokenized_reviews):
        sentence_embeddings = model.encode(review)
        for j, e in enumerate(sentence_embeddings):
            embeddings[(i, j)] = e

    mapping = {}
    for ((i,j), e) in embeddings.items():
        #print([abs_cosine_distance(e, ep) for _,ep in sorted(embeddings.items())])
        if topk is None:
            labels = [1 if abs_cosine_distance(e, ep) < thresh else 0 for _,ep in sorted(embeddings.items())]
        else:
            scores = sorted([(abs_cosine_distance(e, ep[1]),i) for i,ep in enumerate(sorted(embeddings.items()))])
            labels = [0] * len(scores)
            for score in scores[:topk]:
                labels[score[1]] = 1
        #(review_match, sentence_match) = min([(abs_cosine_distance(e, ep), (x,y)) for (x,y),ep in embeddings.items() if (x,y) != (i,j)])[1]
        for k in range(len(tokenized_reviews)):
            mapping[str((k, i, j))] = labels

    return mapping

def map_word_to_sentence(sentences, word_idx):
    sentence = 0
    while word_idx >= 0:
        word_idx -= len(word_tokenize(sentences[sentence]))
        sentence += 1
    return sentence - 1

def extract_scico_results(tokenized_reviews):
    with open('pair_extraction/results/scico_ctrlsum.jsonl') as f:
        data = dict(json.loads(f.read()))

    sentence_level_data = []
    sentence_to_cluster = defaultdict(lambda: set())
    cluster_to_sentences = defaultdict(lambda: set())
    sentence_to_keywords = defaultdict(lambda: set())
    for mention in data['mentions']:
        review = tokenized_reviews[mention[0]]
        left = map_word_to_sentence(review, mention[1])
        right = map_word_to_sentence(review, mention[2])
        for j in range(left, right+1, 1):
            sentence_to_cluster[mention[0], j].add(mention[3])
            cluster_to_sentences[mention[3]].add((mention[0], j))
            for k in range(mention[1], mention[2]+1, 1):
                sentence_to_keywords[mention[0], j].add(data['tokens'][mention[0]][k])
            #sentence_level_data.append([mention[0], map_word_to_sentence(review, mention[1]), map_word_to_sentence(review, mention[2]), mention[3]])

    pair_mapping = {}
    label_mapping = {}
    for i1, review1 in enumerate(tokenized_reviews):
        for j1, sentence1 in enumerate(review):
            current_cluster = sentence_to_cluster[i1, j1]
            labels = []
            for i2, review2 in enumerate(tokenized_reviews):
                for j2, sentence2 in enumerate(review2):
                    if len(sentence_to_cluster[i2, j2].intersection(current_cluster)) > 0:
                        labels.append(1)
                    else:
                        labels.append(0)
            pair_mapping[str((i1, i1, j1))] = labels
            if len(sentence_to_keywords[i1, j1]) > 0:
                label_mapping[str((i1, i1, j1))] = ','.join(list(sentence_to_keywords[i1, j1]))
            else:
                label_mapping[str((i1, i1, j1))] = '-'
    return label_mapping, pair_mapping

def generate_bertscore_results(tokenized_reviews, topk=6):
    flattened_sentences = [item for sublist in tokenized_reviews for item in sublist]

    embeddings = {} # map (review_id, sent_id) -> embedding
    mapping = {}
    for i1, review1 in enumerate(tokenized_reviews):
        for j1, sentence1 in enumerate(review1):
            cand = [sentence1] * len(flattened_sentences)
            _, _, scores1 = bert_score(cand, flattened_sentences, lang='en')
            _, _, scores2 = bert_score(flattened_sentences, cand, lang='en')
            scores = [(a+b)/2 for (a,b) in zip(scores1, scores2)]
            scores = sorted([(s, i) for i,s in enumerate(scores)], reverse=True)
            labels = [0] * len(scores)
            for score in scores[:topk]:
                labels[score[1]] = 1
            for k in range(len(tokenized_reviews)):
                mapping[str((k, i1, j1))] = labels
            f = open("pair_extraction/results/bertscore_results.txt","w")
            f.write(str(mapping))
            f.close()

    return mapping

def extract_bertscore_mappings():
    with open("pair_extraction/results/bertscore_results.txt", 'r') as f:
        mappings = ast.literal_eval(f.read())
    return mappings

def extract_mappings_from_finetuned_embeddings(tokenized_reviews, thresh=0.6, topk=None, model_name='pair_extraction/model_files/parasci_bart_outputs'):
    model = SentenceTransformer('paraphrase-distilroberta-base-v1')
    model.load_state_dict(torch.load('pair_extraction/model_files/parasci_siamese.pt', map_location='cpu'))
    embeddings = {} # map (review_id, sent_id) -> embedding
    for i, review in enumerate(tokenized_reviews):
        sentence_embeddings = model.encode(review)
        for j, e in enumerate(sentence_embeddings):
            embeddings[(i, j)] = e

    mapping = {}
    for ((i,j), e) in embeddings.items():
        #print([abs_cosine_distance(e, ep) for _,ep in sorted(embeddings.items())])
        if topk is None:
            labels = [1 if abs_cosine_distance(e, ep) < thresh else 0 for _,ep in sorted(embeddings.items())]
        else:
            scores = sorted([(abs_cosine_distance(e, ep[1]),i) for i,ep in enumerate(sorted(embeddings.items()))])
            labels = [0] * len(scores)
            for score in scores[:topk]:
                labels[score[1]] = 1
        #(review_match, sentence_match) = min([(abs_cosine_distance(e, ep), (x,y)) for (x,y),ep in embeddings.items() if (x,y) != (i,j)])[1]
        for k in range(len(tokenized_reviews)):
            mapping[str((k, i, j))] = labels

    return mapping
    """model = Seq2SeqModel(encoder_decoder_type="bart", encoder_decoder_name=model_name)

    embeddings = {} # map (review_id, sent_id) -> embedding
    for i, review in enumerate(tokenized_reviews):
        encoded_input = model.encoder_tokenizer(review, truncation=True, padding="longest", return_tensors="pt")
        model.model = model.model.to(model.device)
        with torch.no_grad():
            outputs = model.model(encoded_input["input_ids"].to(model.device))
        sentence_embeddings = mean_pooling(outputs, encoded_input['attention_mask'].to(model.device)).cpu()
        for j, e in enumerate(sentence_embeddings):
            embeddings[(i, j)] = np.array(e)

    mapping = {}
    for ((i,j), e) in embeddings.items():
        #print([abs_cosine_distance(e, ep) for _,ep in sorted(embeddings.items())])
        if topk is None:
            labels = [1 if abs_cosine_distance(e, ep) < thresh else 0 for _,ep in sorted(embeddings.items())]
        else:
            scores = sorted([(abs_cosine_distance(e, ep[1]),i) for i,ep in enumerate(sorted(embeddings.items()))])
            labels = [0] * len(scores)
            for score in scores[:topk]:
                labels[score[1]] = 1
        #(review_match, sentence_match) = min([(abs_cosine_distance(e, ep), (x,y)) for (x,y),ep in embeddings.items() if (x,y) != (i,j)])[1]
        for k in range(len(tokenized_reviews)):
            mapping[str((k, i, j))] = labels

    return mapping"""

def generate_entailment_results(tokenized_reviews):
    predictor = Predictor.from_path("https://storage.googleapis.com/allennlp-public-models/decomposable-attention-elmo-2020.04.09.tar.gz", "textual_entailment")
    pair_mapping = {}
    total = sum([len(sentences) for sentences in tokenized_reviews])
    for i, review in enumerate(tokenized_reviews):
        for j, sentence in enumerate(review):
            if str((i, i, j)) not in pair_mapping.keys():
                pair_mapping[str((i, i, j))] = [0] * total
                if i > 0:
                    idx = sum([len(review) for review in tokenized_reviews[:i]]) + j
                else:
                    idx = j
                pair_mapping[str((i, i, j))][idx] = 1
    for i1, review1 in enumerate(tokenized_reviews):
        for j1, sentence1 in enumerate(review1):
            labels = []
            for i2, review2 in enumerate(tokenized_reviews):
                for j2, sentence2 in enumerate(review2):
                    result = predictor.predict(
                      hypothesis=sentence1,
                      premise=sentence2
                    )
                    if result['label'] == 'entailment':
                        labels.append(1)
                    else:
                        labels.append(0)
            pair_mapping[(i1, i1, j1)] = labels
            f = open("pair_extraction/results/entailment_results.txt","w")
            f.write(str(pair_mapping))
            f.close()

def extract_mappings_from_entail(tokenized_reviews):
    with open("pair_extraction/results/entailment_results.txt","r") as f:
        raw_mapping = ast.literal_eval(f.read())
    pair_mapping = {str(x):y for x,y in raw_mapping.items()}
    total = sum([len(sentences) for sentences in tokenized_reviews])
    for i, review in enumerate(tokenized_reviews):
        for j, sentence in enumerate(review):
            if str((i, i, j)) not in pair_mapping.keys():
                print(i, i, j, "not in entailment mapping")
            pair_mapping[str((i, i, j))] = [0]*total
            if i > 0:
                idx = sum([len(review) for review in tokenized_reviews[:i]]) + j
            else:
                idx = j
            pair_mapping[str((i, i, j))][idx] = 1
    return {str(x):y for x,y in raw_mapping.items()}

def generate_ner_results(tokenized_reviews, thresh=0.8):
    ner = pipeline("ner")
    entity_to_sent = defaultdict(lambda: set())
    sent_to_entity = defaultdict(lambda: set())
    for i, review in enumerate(tokenized_reviews):
        for j, sentence in enumerate(review):
            result = ner(sentence)
            entities = {r['word'] for r in result if r['score']>thresh}
            sent_to_entity[(i, j)] = sent_to_entity[(i, j)].union(entities)
            for ent in entities:
                entity_to_sent[ent].add((i, j))
    pair_mapping = {}
    label_mapping = {}
    for i1, review1 in enumerate(tokenized_reviews):
        for j1, sentence1 in enumerate(review1):
            labels = []
            all_common = set()
            for i2, review2 in enumerate(tokenized_reviews):
                for j2, sentence2 in enumerate(review2):
                    common = sent_to_entity[(i1, j1)].intersection(sent_to_entity[(i2, j2)])
                    all_common = all_common.union(common)
                    if len(common) > 0:
                        labels.append(1)
                    else:
                        labels.append(0)
            label_mapping[str((i1, i1, j1))] = str(all_common) if len(all_common) > 0 else '-'
            pair_mapping[(i1, i1, j1)] = labels
            f = open("pair_extraction/results/ner_pair_mapping.txt","w")
            f.write(str(pair_mapping))
            f.close()
            f = open("pair_extraction/results/ner_label_mapping.txt","w")
            f.write(str(label_mapping))
            f.close()

def extract_mappings_from_ner():
    with open("pair_extraction/results/ner_pair_mapping.txt","r") as f:
        raw_mapping = ast.literal_eval(f.read())
        pair_mapping = {str(x):y for x,y in raw_mapping.items()}
    with open("pair_extraction/results/ner_label_mapping.txt","r") as f:
        raw_mapping = ast.literal_eval(f.read())
        label_mapping = {str(x):y for x,y in raw_mapping.items()}
    return label_mapping, pair_mapping

def get_ids_from_span_idx(review_lengths, idx):
    review_id = 0
    while idx >= 0:
        for sentence_lengths in review_lengths:
            sentence_id = 0
            for x in sentence_lengths:
                if x >= idx+1:
                    return (review_id, sentence_id)
                else:
                    sentence_id += 1
                    idx -= x
            review_id += 1
    return (review_id, sentence_id)

def extract_mappings_from_coreference(tokenized_reviews):
    try:
        nlp = spacy.load('en')
        neuralcoref.add_to_pipe(nlp)
        flattened_reviews = []
        review_lengths = []
        for review in tokenized_reviews:
            flattened_reviews += review
            sentence_lengths = []
            for sentence in review:
                sentence_lengths.append(len(nlp(sentence)))
            review_lengths.append(sentence_lengths)
        flattened_reviews = ' '.join(flattened_reviews)
        doc = nlp(flattened_reviews)

        # construct mapping
        pair_mapping = {}
        label_mapping = {}
        for cluster in doc._.coref_clusters:
            pairs_from_clusters = []
            for mention in cluster.mentions:
                review_id, sentence_id = get_ids_from_span_idx(review_lengths, mention.start)
                pairs_from_clusters.append((review_id, sentence_id))
                label_mapping[str((review_id, review_id, sentence_id))] = str(mention)
            for a in pairs_from_clusters:
                labels = []
                for i, review in enumerate(tokenized_reviews):
                    for j, sentence in enumerate(review):
                        if (i,j) in pairs_from_clusters:
                            labels.append(1)
                        else:
                            labels.append(0)
                        if str((i, i, j)) not in label_mapping.keys():
                            label_mapping[str((i, i, j))] = '-'
                pair_mapping[str((a[0], a[0], a[1]))] = labels
        total = sum([len(sentences) for sentences in tokenized_reviews])
        for i, review in enumerate(tokenized_reviews):
            for j, sentence in enumerate(review):
                if str((i, i, j)) not in pair_mapping.keys():
                    pair_mapping[str((i, i, j))] = [0] * total
                    if i > 0:
                        idx = sum([len(review) for review in tokenized_reviews[:i]]) + j
                    else:
                        idx = j
                    pair_mapping[str((i, i, j))][idx] = 1
    except:
        print("Error with neural coreference!")
        pair_mapping = {}
        label_mapping = {}
        total = sum([len(review) for review in tokenized_reviews])
        for i, review in enumerate(tokenized_reviews):
            for j, sentence in enumerate(review):
                pair_mapping[str((i, i, j))] = [0] * total
                label_mapping[str((i, i, j))] = '-'

    return label_mapping, pair_mapping

def extract_groundtruth_mapping(tokenized_reviews):
    pair_mapping = {}
    with open("pair_extraction/results/ground_truth_raw.txt", 'r') as f:
        for line in f.readlines():
            pairs = line.replace(' ', '').replace('\n', '').split(';')
            pairs = {tuple([int(p) for p in x.split(',')]) for x in pairs}
            for a in pairs:
                labels = []
                for i, review in enumerate(tokenized_reviews):
                    for j, sentence in enumerate(review):
                        if (i,j) in pairs or (a[0], a[1]) == (i,j):
                            labels.append(1)
                        else:
                            labels.append(0)
                pair_mapping[str((a[0], a[0], a[1]))] = labels
    total = sum([len(review) for review in tokenized_reviews])
    for i, review in enumerate(tokenized_reviews):
        for j, sentence in enumerate(review):
            if str((i, i, j)) not in pair_mapping.keys():
                pair_mapping[str((i, i, j))] = [0] * total
                if i > 0:
                    idx = sum([len(review) for review in tokenized_reviews[:i]]) + j
                else:
                    idx = j
                pair_mapping[str((i, i, j))][idx] = 1
    return pair_mapping

def prep_scico_data(tokenized_reviews):
    final_dict = {}
    final_dict['flatten_tokens'] = []
    final_dict['flatten_mentions'] = []
    final_dict['tokens'] = []
    final_dict['doc_ids'] = []
    final_dict['metadata'] = []
    final_dict['sentences'] = []
    final_dict['mentions'] = []
    final_dict['relations'] = []
    final_dict['id'] = 0
    final_dict['hard_10'] = False
    final_dict['hard_20'] = False
    final_dict["source"] = "pwc"
    for i, review in enumerate(tokenized_reviews):
        for j, sentence in enumerate(review):
            tokenized_sentence = word_tokenize(sentence)
            for word in tokenized_sentence:
                final_dict['flatten_tokens'].append(word)
    flattened_tokens = []

def extract_groundtruth_from_df(df):
    pass
    """for paper_id in df['paper_id'].unique():
        pair_mapping = {}
        reviews = df[df['paper_id'] == paper_id]
        for i, row in df.iterrows():
            row[]"""


import pytest
from django.urls import reverse
from rest_framework.test import APIClient
import io
import csv
import json
from workflow.models import Products

@pytest.fixture
def api_client():  # An api client to make request
    return APIClient()

@pytest.fixture
def create_new_product_obj():  # A fixture for creating new products
    data = {
        "name": "John Edwards",
        "description": "Hello world",
        "sku": "movement-near",
        "active": True
    }
    product = Products(**data)
    product.save()
    return product

@pytest.mark.django_db
def test_create_product(api_client):  # Test for creating a product
    data = {
        "name": "Simeon Peter",
        "description": "It's a beautiful world",
        "sku": "move-here",
        "active": False
    }
    url = reverse('Product-List-View')
    client = api_client
    response = client.post(url, data, format='json')
    returned_data = json.loads(response.content)['data']
    assert response.status_code == 201
    assert returned_data['name'] == data['name']
    assert returned_data['description'] == data['description']
    assert returned_data['sku'] == data['sku']
    assert returned_data['active'] == data['active']

@pytest.mark.django_db
def test_get_all_products(api_client):  # Test for getting all products
    url = reverse('Product-List-View')
    client = api_client
    response = client.get(url)
    assert response.status_code == 200

@pytest.mark.django_db
def test_delete_all_products(api_client):  # Test for deleting all products
    url = reverse('Product-List-View')
    client = api_client
    response = client.delete(url)
    assert response.status_code == 204

@pytest.mark.django_db
def test_get_one_product(api_client, create_new_product_obj):  # Test for retrieving one product
    url = reverse('Product-Detail-View', kwargs={'product_id': create_new_product_obj.uuid})
    client = api_client
    response = client.get(url)
    assert response.status_code == 200

@pytest.mark.django_db
def test_update_one_product(api_client, create_new_product_obj):  # Test for updating a product
    data = {
        "name": "Christopher Columbus",
        "active": True
    }
    url = reverse('Product-Detail-View', kwargs={'product_id': create_new_product_obj.uuid})
    client = api_client
    response = client.patch(url, data, format='json')
    returned_data = json.loads(response.content)['data']
    assert response.status_code == 201
    assert returned_data['name'] == "Christopher Columbus"
    assert returned_data['active'] == True

@pytest.mark.django_db
def test_delete_one_product(api_client, create_new_product_obj):  # Test for deleting a product
    url = reverse('Product-Detail-View', kwargs={'product_id': create_new_product_obj.uuid})
    client = api_client
    response = client.delete(url, format='json')
    assert response.status_code == 204

@pytest.mark.django_db
def test_filter_product(api_client):  # Test search by `name`, `description`, `sku`
    url = reverse('Product-Filter-View')
    client = api_client
    response = client.get(url, {'search': "Chris"})
    assert response.status_code == 200

@pytest.mark.django_db
def test_filter_active_product(api_client):  # Test for filtering by active and inactive products
    url = reverse('Product-Status-Filter-View')
    client = api_client
    response = client.get(url, {'active': True})
    assert response.status_code == 200

@pytest.mark.django_db
def test_webhook(api_client):  # Test for webhook
    data = {
        "message": "Hello World"
    }
    url = reverse('Webhook-View')
    client = api_client
    response = client.post(url, data)
    assert response.status_code == 200

@pytest.mark.django_db
def test_bulk_upload(api_client):  # Test for bulk uploading of products
    """Creating CSV file to be uploaded"""
    csv_headers = ['name', 'description', 'sku', 'active']
    data = ['Raman', 'It works fine', 'hello-there', 'True']
    output = io.StringIO()
    writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC)
    writer.writerow(csv_headers)
    writer.writerow(data)
    content = output.getvalue()
    encoded_content = content.encode('utf-8')
    bio = io.BytesIO(encoded_content)
    bio.seek(0)

    data = {
        "file": bio
    }
    url = reverse('Product-BulkUpload-View')
    client = api_client
    response = client.post(url, data)
    assert response.status_code == 200


import pandas as pd
import numpy as np

def get_ratings(user_id):
    user_ratings = ratings[user_id]
    actual_ratings = user_ratings[~np.isnan(user_ratings)]
    return actual_ratings

def get_top_n(user_id, n):
    top_n = {}
    for rec, rec_name in zip(recs, recs_names):
        top_n_items = rec[user_id].argsort().sort_values()[:n].index.values
        top_n[rec_name] = top_n_items
    return top_n

def get_popular_items(n):
    pop_percentages = ratings.copy()
    pop_percentages['popularity'] = ratings.apply(lambda row: np.sum(~np.isnan(row))-1, axis=1)/len(ratings.columns[1::])
    pop_percentages = pop_percentages.sort_values(by = 'popularity', ascending=False)
    return pop_percentages.item.values[:n]

def get_rmse(user_id):
    user_ratings = get_ratings(user_id)
    rmse = {}
    for rec, rec_name in zip(recs, recs_names):
        predicted_ratings = rec.loc[user_ratings.index, user_id]
        temp = np.sqrt(np.average((predicted_ratings - user_ratings)**2))
        rmse[rec_name] = temp
    return rmse

def get_precision_at_n(user_id, n):
    top_n = get_top_n(user_id, n)
    user_ratings = get_ratings(user_id).index.values
    precisions = {}
    for rec, rec_name in zip(recs, recs_names):
        temp = np.sum(np.isin(top_n[rec_name], user_ratings))/n
        precisions[rec_name] = temp
    return precisions

# We will use the "FullCat" column in the items catalog to determine the product diversity in the recommendations.
# The recommender with a high number of distinct product categories in its recommendations is said to be product-diverse
def get_product_diversity(user_id, n):
    top_n = get_top_n(user_id, n)
    product_diversity = {}
    for rec_name in top_n:
        categories = items.loc[top_n[rec_name]][['FullCat']].values
        categories = set([item for sublist in categories for item in sublist])
        product_diversity[rec_name] = len(categories)
    return product_diversity

# We will use the "Price" column in the items catalog to determine cost diversity in the recommendations.
# The recommender with a high standard deviation in the cost across all its recommendations is said to be cost-diverse
def get_cost_diversity(user_id, n):
    top_n = get_top_n(user_id,n)
    cost_diversity = {}
    for rec_name in top_n:
        std_dev = np.std(items.loc[top_n[rec_name]][['Price']].values)
        cost_diversity[rec_name] = std_dev
    return cost_diversity

# We will use inverse popularity as a measure of serendipity.
# The recommender with least number of recommendations on the "most popular" list, will be called most serendipitous
def get_serendipity(user_id, n):
    top_n = get_top_n(user_id,n)
    popular_items = get_popular_items(20)
    serendipity = {}
    for rec, rec_name in zip(recs, recs_names):
        popularity = np.sum(np.isin(top_n[rec_name],popular_items))
        if int(popularity) == 0:
            serendipity[rec_name] = 1
        else:
            serendipity[rec_name] = 1/popularity
    return serendipity

avg_metrics = {}
for name in recs_names:
    avg_metrics[name] = {"rmse": [], "precision_at_n": [], "product_diversity": [], "cost_diversity": [], "serendipity": []}

for user_id in ratings.columns:
    if user_id == 'item':
        continue
    user_id = str(user_id)
    rmse = get_rmse(user_id)
    precision_at_n = get_precision_at_n(user_id, 10)
    product_diversity = get_product_diversity(user_id, 10)
    cost_diversity = get_cost_diversity(user_id, 10)
    serendipity = get_serendipity(user_id, 10)
    for key in avg_metrics:
        rec_name = avg_metrics[key]
        rec_name['rmse'].append(rmse[key])
        rec_name['precision_at_n'].append(precision_at_n[key])
        rec_name['product_diversity'].append(product_diversity[key])
        rec_name['cost_diversity'].append(cost_diversity[key])
        rec_name['serendipity'].append(serendipity[key])

# The Price for certain items is not available. Also rmse for certain users is turning out to be NaN.
# Ignoring nans in the average metric calculation for now. So basically narrowing down the evaluation to users who have
# rated atleast one item and items for which the price is known.
for key in avg_metrics:
    rec_name = avg_metrics[key]
    for metric in rec_name:
        temp = rec_name[metric]
        temp = [x for x in temp if not np.isnan(x)]
        rec_name[metric] = sum(temp) / len(temp)


from django.shortcuts import render, redirect
from django.views import View
from accounts.models import *
from django.contrib import messages
from django.db.models import Q
from django.contrib.auth import login, authenticate, logout
import random
from django.http.response import JsonResponse
import datetime
from django.utils import timezone

class ProjectListing(View):
    templates_name = 'project/project.html'

    def get(self, request):
        try:
            all_projects = Project.objects.filter(user_instance = User.objects.get(id = request.user.id)).order_by('-created_on')
            success_msg = request.GET.get('success_msg')

            messages.success(request, success_msg)
            return render(request, self.templates_name, locals())
        except:
            return render(request, self.templates_name, locals())

class AddProject(View):
    templates_name = 'project/add_project.html'

    def get(self, request):
        try:
            all_projects = Project.objects.filter(user_instance = User.objects.get(id = request.user.id))
            success_msg = request.GET.get('success_msg')

            messages.success(request, success_msg)
            return render(request, self.templates_name, locals())
        except:
            return render(request, self.templates_name, locals())

    def post(self, request, *args, **kwargs):

        ''' this function will hit while post request only and we can get every thing from request.POST parameter'''
        context = {}
        try:

            # getting title password and validate it

            title = request.POST.get('title')
            if not title:
                context['message'] = 'Error ! Please, Enter your title'
                context['status'] = 400
                return render(request, self.templates_name, context)
            # ends here title password and validate it

            # getting description password and validate it
            description = request.POST.get('description').strip().lower()
            if not description:
                context['message'] = 'Error ! Please, Enter your description'
                context['status'] = 400
                return render(request, self.templates_name, context)
            # ends here description password and validate it

            # getting timeframe timeframe and validate it
            timeframe = request.POST.get('timeframe')
            if not timeframe:
                context['message'] = 'Error ! Please, Enter your timeframe'
                context['status'] = 400
                return render(request, self.templates_name, context)
            # ends here timeframe timeframe and validate it

            document = request.FILES.get('document')

            obj = Project.objects.create(title=title, description=description, timeframe=timeframe,
                                           user_instance = User.objects.get(id = request.user.id))
            if document:
                obj.files = document
                obj.save()

            context['sucess_msg'] = 'Success ! Your project has been successfully registered !'
            all_projects = Project.objects.filter(user_instance = User.objects.get(id = request.user.id))
            context['all_projects'] = all_projects

            return render(request, self.templates_name, context)

        except Exception as e:
            print("e is", e)
            context['message'] = 'Something Going Wrong ! Please try again later or contact us !'
            context['status'] = 200
            return render(request, self.templates_name, context)

class ProjectView(View):
    templates_name = 'project/view_project.html'

    def get(self, request):
        context = {}
        try:
            idd = self.request.GET.get('id')
            success_msg = request.GET.get('success_msg')
            obj = Project.objects.get(id=idd, user_instance  = User.objects.get(id = request.user.id))
            all_assignee = ProjectBidders.objects.filter(project_instance = obj)
            one_assignee = ProjectBidders.objects.filter(project_instance = obj,is_selected = True).first()
            print(obj.files)
            order_detail = OrderProjected.objects.filter(project_instance = obj,is_cancelled = False).first()

            messages.success(request, success_msg)
            return render(request, self.templates_name, locals())
        except Exception as e:
            print(e)
            context['message'] = 'Something Going Wrong ! Please try again later or contact us !'
            context['status'] = 200

            return render(request, self.templates_name, context)

def MinuteHourAgo(time):

    '''Demonstrate docstring for informing that this Python View based function will give an exact time of message when this message was sent by user'''

    now = datetime.datetime.now(timezone.utc)
    diff = now - time
    second_diff = diff.seconds
    day_diff = diff.days
    if day_diff < 0:
        return 'dss'

    if day_diff == 0:
        if second_diff < 10:
            return "just now"
        if second_diff < 60:
            return str(int(second_diff)) + " seconds ago"
        if second_diff < 120:
            return "a minute ago"
        if second_diff < 3600:
            return str(int(second_diff / 60)) + " minutes ago"
        if second_diff < 7200:
            return "an hour ago"
        if second_diff < 86400:
            return str(int(second_diff / 3600)) + " hours ago"

    if day_diff == 1:
        return "Yesterday"
    if day_diff < 7:
        return str(int(day_diff)) + " days ago"
    if day_diff < 31:
        return str(int(day_diff / 7)) + " weeks ago"
    if day_diff < 365:
        return str(int(day_diff / 30)) + " months ago"

    return str(day_diff / 365) + " years ago"

class ProjectBid(View):

    templates_name = 'project/developer_project.html'

    def post(self,request):
        context = {}
        try:
            project_id = request.POST.get('id')
            timeframe = request.POST.get('timeframe')
            description = request.POST.get('description')

            user_instance = User.objects.get(id = request.user.id)
            project_instance = Project.objects.get(id = project_id)
            try:
                ProjectBidders.objects.get(user_instance = user_instance, project_instance = project_instance)
            except:
                ProjectBidders.objects.create(description = description,timeframe = timeframe,user_instance = user_instance, project_instance = project_instance)

            print(request.user.role_instance.role_name)
            all_projects = Project.objects.filter(is_active = True)
            sucess_msg = 'Success ! Offer has been sent to client !'
            return render(request, self.templates_name, locals())

        except Exception as e:
            print(e)
            context['message'] = 'Something Going Wrong ! Please try again later or contact us !'
            context['status'] = 200

            return render(request, self.templates_name, context)
    def get(self, request):
        try:
            print(request.user.role_instance.role_name)
            all_projects = Project.objects.filter(is_active = True)
            success_msg = request.GET.get('success_msg')

            messages.success(request, success_msg)
            return render(request, self.templates_name, locals())
        except:
            return render(request, self.templates_name, locals())

class Cancelled(View):

    templates_name = 'project/developer_project.html'

    def get(self,request):
        context = {}
        try:
            project_id = request.GET.get('id')
            user_instance = User.objects.get(id = request.GET.get('user_id'))

            project_instance = Project.objects.get(id = project_id)
            obj  =  ProjectBidders.objects.get(user_instance = user_instance, project_instance = project_instance)
            obj.is_selected = False
            obj.save()
            obj  =  OrderProjected.objects.get(user_instance = user_instance, project_instance = project_instance)
            obj.is_deliver = False
            obj.is_completed = False
            obj.is_cancelled = True
            obj.save()
            return redirect('/view_project/?id={}'.format(project_id))

        except Exception as e:
            print(e)
            context['message'] = 'Something Going Wrong ! Please try again later or contact us !'
            context['status'] = 200

            return redirect('/ReviewDelivery/?id={}'.format(obj.id))

class ApproveOrNot(View):

    templates_name = 'project/developer_project.html'

    def get(self,request):
        context = {}
        try:
            project_id = request.GET.get('id')
            obj  =  OrderProjected.objects.get(id = project_id)

            obj.is_completed = True
            obj.project_instance.is_active = False
            obj.save()
            return redirect('/ReviewDelivery/?id={}'.format(obj.id))

        except Exception as e:
            print(e)
            context['message'] = 'Something Going Wrong ! Please try again later or contact us !'
            context['status'] = 200
            return redirect('/ReviewDelivery/?id={}'.format(obj.id))

class ReviewDelivery(View):

    templates_name = 'project/review.html'

    def get(self,request):
        context = {}
        try:
            order_id = request.GET.get('id')
            obj  =  OrderProjected.objects.get(id = order_id)
            return render(request, self.templates_name, locals())

        except Exception as e:
            print(e)
            context['message'] = 'Something Going Wrong ! Please try again later or contact us !'
            context['status'] = 200

            return render(request, self.templates_name, context)

class ChatPage(View):
    templates_name = 'project/chat.html'

    def get(self, request):
        try:
            success_msg = request.GET.get('success_msg')
            me = request.user.id

            offer_id = request.GET.get('id')
            offer_instance = ProjectBidders.objects.get(id = offer_id)
            userid = offer_instance.user_instance.id
            project_instance = offer_instance.project_instance

            thread = ParticularThread.objects.filter(first__id = me, second__id = userid,project_instance = project_instance)
            thread_sec = ParticularThread.objects.filter(first__id = userid, second__id = me,project_instance = project_instance)
            thread_list = []
            message_me = []

            if thread:
                message_me = ChatMessages.objects.filter(thread = thread[0])
            if thread_sec:
                message_me = ChatMessages.objects.filter(thread = thread_sec[0])
            for one in message_me:
                dict_message = {}
                dict_message['sender'] = one.user
                dict_message['message'] = one.message
                saved_time = one.timestamp
                get_timestamp = MinuteHourAgo(saved_time)
                dict_message['get_timestamp'] = get_timestamp
                thread_list.append(dict_message)

            return render(request, self.templates_name, locals())
        except Exception as e:
            print("e",e)
            return render(request, self.templates_name, locals())

    def post(self,request):
        context = {}
        try:
            message = request.POST.get('message')
            offer_id = request.POST.get('offer_id')

            offer_instance = ProjectBidders.objects.get(id = offer_id)

            user_instance = User.objects.get(id = request.user.id)
            project_instance = offer_instance.project_instance
            developer_instance = offer_instance.user_instance

            try:
                obj = ParticularThread.objects.get(second = developer_instance,first = user_instance, project_instance = project_instance)
            except:
                obj = ParticularThread.objects.create(second = developer_instance,first = user_instance, project_instance = project_instance)

            ChatMessages.objects.create(thread = obj, user = user_instance, message =  message)

            context['status'] = 200
            return JsonResponse(context)

        except Exception as e:
            print(e)
            context['message'] = 'Something Going Wrong ! Please try again later or contact us !'
            context['status'] = 500
            return JsonResponse(context)


# --------------------------------------------------------
# 2020. 11. 25 Python Programming Self Test
# -------------------------------------------------------
# 파이썬 예제
if 0:
    # (1) 1~100사이 범위에서 짝수의 합을 반환하는 함수 만들기
    #     함  수  명 : evenSum
    #     매개  변수 : 없음
    #     반  환  값 : 합
    def evenSum() :
        sum = 0
        for i in range(1, 101, 1):
            if i % 2 == 0:
                sum += i

        return sum

    print(evenSum())

if 0:
    # (2) 입력 받은 구구단을 출력하는 프로그램 만들기
    #     입력 예 )  3
    #     결과 예 )  3 * 1 = 3    3 * 2 = 6    3 * 3 = 9
    #               3 * 4 = 12   3 * 5 = 15   3 * 6 = 18
    #               3 * 7 = 21   3 * 8 = 24   3 * 9 = 27
    xnum = int(input("정수를 입력하세요 : "))

    for i in range(1, 10):
        if i % 3 != 0:
            print(f"{xnum} * {i} = {xnum * i}\t", end='')
        else:
            print(f"{xnum} * {i} = {xnum * i}")

if 0:
    # (3) 가변인자를 입력받아서 더하기, 빼기, 곱하기, 최대값, 최소값 출력 함수 만들기
    #     함  수  명 : allFunc
    #     매개  변수 : 수치데이터 가변인자
    #     반  환  값 : 더하기, 빼기, 곱하기, 최대값, 최소값
    def allFunc(*params) :
        # 더하기
        add = 0
        for i in params:
            add += i

        # 빼기
        sub = 0
        for i in params:
            sub -= i

        # 곱하기
        mul = 1
        for i in params:
            mul *= i

        return add, sub, mul, max(params), min(params)

    print(allFunc(1, 2, 3, 4, 5))

if 0:
    # (4) 입력받은 문자열을 변환하는 함수 만들기
    #     함  수  명 : convertFunc
    #     매개  변수 : str데이터
    #     반  환  값 : 없음
    #     입  력  예 ) Good Luck
    #     실  행  예 ) gOOD lUCK
    #                 G*o*o*d* *L*u*c*K
    #                 'Good', 'Luck'
    def converFunc(param):
        for i in param:
            if ord('A') <= ord(i) <= ord('Z'):
                print(i.lower(), end='')
            elif ord('a') <= ord(i) <= ord('z'):
                print(i.upper(), end='')
            else:
                print(i, end='')
        print()
        for i in param:
            print(i, end='*')
        print()
        print(param.split())

    print(converFunc('Good Luck'))

if 0:
    # (5) 입력받은 점수에 따른 학점을 출력하세요.
    #     A+, A, B+, B, C+, C, D+, D, F
    x = int(input("점수를 입력하세요 : "))
    if  x >= 95:
        print('A+')
    elif 95 > x >= 90:
        print('A')
    elif 90 > x >= 85:
        print('B+')
    elif 85 > x >= 80:
        print('B')
    elif 80 > x >= 75:
        print('C+')
    elif 75 > x >= 70:
        print('C')
    elif 70 > x >= 65:
        print('D+')
    elif 65 > x >= 60:
        print('D')
    else:
        print("F")

if 0:
    # (6) 1~50사이 범위에서 소수를 출력하세요.
    #     소수(prime number): 1과 자기자신으로만 나누어지는 수
    #     값 : 2, 3, 5, 7, 11, 13, 17, 19, ......
    for i in range(2, 51):
        flag = True
        for j in range(2, i):
            if i % j == 0:
                flag = False
                break
        if flag:
            print(i, end=', ')

if 0:
    # (7) 계산기 클래스를 만들기
    #     클래스 명 :  Calc
    #     메 서 드 :  더하기, 곱하기, 나누기, 빼기, 종합(4가지 연산 한꺼번에 수행)
    #     동    작 : 메서드는 결과값 반환
    class Calc:

        def calFunc(param, param1):
            add = param + param1
            mul = param * param1
            div = round(param / param1, 2)
            sub = param - param1

            return add, mul, div, sub

    print(Calc.calFunc(2, 3))

if 0:
    # (8) 10명에 대한 비만도 정보 저장 기능 만들기
    #     기    능 :  이름, 키, 몸무게 입력 받아서 BMI 계산
    #                - 이름, BMI 정보 저장
    #                - 이름으로 BMI에 따른 저체중, 정상, 비만 정보 출력

    # 함수생성
    def bmiFunc(w, h):  # bmi 계산
        # BMI지수 = 몸무게(kg) / (신장(m) * 신장(m))
        h = h / 100
        x = round(w / (h * h), 3)
        # 비만의 정도를 측정
        if x >= 35:
            return '고도비만', x
        elif x >= 30:
            return '중정도 비만', x
        elif x >= 25:
            return '경도 비만', x
        elif x >= 23:
            return '과체중', x
        elif x >= 18.5:
            return '정상체중', x
        else:
            return '저체중', x

    # 입력
    cnt = 0
    file = open('bmiInfo.txt', mode='w', encoding='utf-8')
    while cnt < 2:
        try:
            name = input("이름을 입력하세요 : ")
            height = int(input("키를 입력하세요 : "))
            weight = int(input("몸무게를 입력하세요 :"))
            bmi = bmiFunc(weight, height)
            file.write(f"{cnt+1}. {name} : {bmi}\n")
        except Exception as e:
            print("[예외발생. 다시해주세요]")
            print('--------------------------------')
        else:
            print("[정상적으로 입력 되었습니다.]")
            print('--------------------------------')
            cnt += 1

    file.close()  # 파일 닫기

    # 출력
    file = open('bmiInfo.txt', mode='r', encoding='utf-8')
    alldata = file.read()
    print('--------------------------------')
    print(f"{alldata}")
    print('--------------------------------')
    file.close()

if 0:
    # (9) CSV 파일 기록 및 읽기
    #     기    능 :  입력 받은 사람 정보( 이름, 성별, 나이, 지역) CSV 파일에 기록
    #     파 일 명 : person.csv
    import csv

    with open('../../Desktop/PycharmProjects/person.csv', mode='w', encoding='utf-8', newline='') as pf:
        w = csv.writer(pf, delimiter=',', quotechar='"')
        perInfo = input('이름, 성별, 나이, 지역을 입력하세요 : ').split()
        w.writerow(perInfo)

    # 출력
    with open('../../Desktop/PycharmProjects/person.csv', mode='r', encoding='utf-8', newline='') as pf:
        r = csv.reader(pf, delimiter='\t')
        for i in r:
            print(i)

if 0:
    # (10) 입력받은 메시지를 파일에 기록하고 확인하는 프로그램 만들기
    #     파일명 :  message.txt
    #     구  성 : 파일 기록 함수  + 파일 읽기 함수
    #     동  작 : write & read 중 사용자 선택
    #             write => 기록 함수 실행
    #             read  => 읽기 함수 실행
    import time as t

    # 파일기록 함수
    def write():
        file = open('../../Desktop/PycharmProjects/message.txt', mode='w', encoding='utf-8')
        msg = input("메세지를 입력하세요 : ")
        time = t.ctime() + " : "
        file.write(time + msg)
        file.close()

    # 파일읽기 함수
    def read():
        file = open('../../Desktop/PycharmProjects/message.txt', mode='r', encoding='utf-8')
        x = file.read()
        print(f"{x}")
        file.close()

    # 함수실행
    write()
    read()


##
# Take melody data in format: time (s), pitch
# and convert to format: start (ms), duration (ms), pitch, midi
##

# Library dependancies
import csv
import json
import os
import sys
from midiutil.MidiFile import MIDIFile

# Config
FILE_NAME = sys.argv[1]
INPUT_FILE = 'data/raw/'+FILE_NAME+'.csv'
OUTPUT_JSON_FILE = 'data/interviews/'+FILE_NAME+'.json'
OUTPUT_MIDI_FILE = 'data/mid/'+FILE_NAME+'.mid'
FREQUENCIES_FILE = 'data/frequencies.json'
PRECISION = 3
BPM = 240
FREQUENCY_NOTE_THRESHOLD = 10
MIN_NOTE_DURATION = 100

# Init
sequence = []
duration = 0

# Get frequency table
json_data = open(FREQUENCIES_FILE)
frequencies = json.load(json_data)

# Mean of list
def mean(data):
	if iter(data) is data:
		data = list(data)
	n = len(data)
	if n < 1:
		return 0
	else:
		return sum(data)/n

def getPitchData(pitch):
	global frequencies
	data = frequencies[0]
	for i, f in enumerate(frequencies):
		hz = float(f['hz'])
		prev_hz = 0
		if i > 0:
			prev_hz = float(frequencies[i-1]['hz'])
		if pitch < hz:
			if prev_hz > 0 and abs(prev_hz-pitch) < abs(hz-pitch):
				data = frequencies[i-1]
			else:
				data = f
			break
	return data

# Add new sequence step
def addToSequence(ms, duration, pitch):
	global sequence
	global MIN_NOTE_DURATION
	if duration >= MIN_NOTE_DURATION:
		pd = getPitchData(pitch)
		sequence.append({
			"ms": int(ms),
			"dur": int(duration),
			"hz": round(pitch, PRECISION),
			"note": pd['note'],
			"mid": int(pd['midi'])}
		)

# Read frequencies from file
with open(INPUT_FILE, 'rb') as f:
	r = csv.reader(f, delimiter=',')
	pitch_queue = []
	start_ms = 0
	start_pitch = 0
	ms = 0
	for s, p in r:
		if s and p:
			ms = int(round(float(s) * 1000))
			pitch = round(float(p), PRECISION)
			# reached a pause, add previous note queue
			if pitch <= 0 and len(pitch_queue) > 0:
				addToSequence(start_ms, ms-start_ms, mean(pitch_queue))
				pitch_queue = []
			# reached a note threshold, add previous note queue
			elif pitch > 0 and abs(pitch-start_pitch) > FREQUENCY_NOTE_THRESHOLD and (ms-start_ms) > MIN_NOTE_DURATION and len(pitch_queue) > 0:
				addToSequence(start_ms, ms-start_ms, mean(pitch_queue))
				pitch_queue = []
			# add pitch to note queue
			elif pitch > 0:
				if len(pitch_queue) <= 0:
					start_ms = ms
					start_pitch = pitch
				pitch_queue.append(pitch)
	if len(pitch_queue) > 0:
		addToSequence(start_ms, ms-start_ms, mean(pitch_queue))

def msToBeats(ms):
	global BPM
	bpms = 1.0 * BPM / 60 / 1000
	return bpms * ms

# Write sequence to midi file
if len(sequence) > 0:
	duration = sequence[-1]["ms"] + sequence[-1]["dur"]

	# Create the MIDIFile Object with 1 track
	MyMIDI = MIDIFile(1)

	# Tracks are numbered from zero. Times are measured in beats.
	track = 0
	time = 0

	# Add track name and tempo.
	MyMIDI.addTrackName(track,time,"Track")
	MyMIDI.addTempo(track,time,BPM)

	# Add a note. addNote expects the following information:
	track = 0
	channel = 0
	pitch = 60
	time = 0
	duration = 1
	volume = 100

	# Now add the note.
	for step in sequence:
		time = msToBeats(step["ms"])
		duration = msToBeats(step["dur"])
		pitch = min([int(round(step["mid"])), 255])
		MyMIDI.addNote(track,channel,pitch,time,duration,volume)

	# And write it to disk.
	binfile = open(OUTPUT_MIDI_FILE, 'wb')
	MyMIDI.writeFile(binfile)
	binfile.close()
	print('Successfully wrote to file: ' + OUTPUT_MIDI_FILE)

# Write sequence to json file
if len(sequence) > 0:
	with open(OUTPUT_JSON_FILE, 'w') as outfile:
		json.dump(sequence, outfile)
		print('Successfully wrote to JSON file: '+OUTPUT_JSON_FILE)


# -*- coding: utf-8 -*-
"""
Created on Thu Jun  4 08:34:28 2020

@author: karlo
"""
import glob
import os
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

import DHMZ_auxiliary_functions as dhmz_functions

currentDirectory = os.getcwd()

"""
This is an example for extracting data from METEO stations.
IMPORTANT - select the DATA folder with the METEO stations data
The METEO and PRECIP stations data differ.
"""

list_files = os.listdir(currentDirectory + "./meteo_data") #folder where the data is located
os.chdir(currentDirectory + './meteo_data')
d_frames={file: dhmz_functions.df_editor_meteo(file) for file in list_files} #extract the dataframes from the meteo files
max_temps_dict = {station: dhmz_functions.extract_max_temp(station, d_frames[station]) for station in list_files}
humidity_dict = {station: dhmz_functions.extract_rel_hum(station, d_frames[station]) for station in list_files}

os.chdir(currentDirectory)

df_humidity = pd.concat(humidity_dict, axis=1)
humidity_stats = df_humidity.describe()

#ploting the data
sns.set()
df_humidity_monthly = df_humidity.resample('M').mean()
_ = df_humidity_monthly.plot(figsize = (10,5))
_ = plt.xlabel('Date')
_ = plt.ylabel('Rel. Humidity [-]')

"""
#example, string with the station name, dataframe of the station --> HOW TO RUN EXTRACTORS
bednja_min_temp = dhmz.extract_min_temp('bednja', d_frames_meteo['bednja'])
bednja_max_temp = dhmz.extract_max_temp('bednja', d_frames_meteo['bednja'])

bednja_oborine = dhmz.extract_meteo_precip ('bednja', d_frames_meteo['bednja'])

ludbreg_wind = dhmz.extract_wind_speed('ludbreg', d_frames_meteo['ludbreg'])

os.chdir(currentDirectory)

bednja_data = {'max_temp': bednja_max_temp}

df = pd.DataFrame(bednja_data)

"""

"""
This is an example for extracting data from PRECIPITATION stations.
IMPORTANT - select the DATA folder with the METEO stations data
"""
list_files = os.listdir(currentDirectory + "./precip_data") #folder where the data is located
os.chdir(currentDirectory + './precip_data')
d_frames={file: dhmz_functions.df_editor_precip(file) for file in list_files} #extract the dataframes from the meteo files
precip_dict = {station: dhmz_functions.extract_precip(station, d_frames[station]) for station in list_files}
os.chdir(currentDirectory)
df_precip = pd.concat(precip_dict, axis=1)

df_precip_stats = df_precip.describe()

#ploting the data
sns.set()
df_precip_monthly = df_precip.resample('M').sum() #sum of monthly precipitation
_ = df_precip_monthly.plot(figsize = (10,5))
_ = plt.xlabel('Date')
_ = plt.ylabel('Precipitation [mm]')