#!/usr/bin/python3
import sys
import unittest
import pep8
from models.state import State

class Test_StateModel(unittest.TestCase):
    """[test the State model]"""

    def testpep8(self):
        """ [testing codestyle] """
        pepstylecode = pep8.StyleGuide(quiet=True)
        user_path = 'models/state.py'
        result = pepstylecode.check_files([user_path])

    def setUp(self):
        self.model_test = State()
        self.model_test.save()

    def test_StateInstantiation(self):
        """[testing]"""
        self.assertIsInstance(self.model_test, State)
        self.assertTrue(hasattr(self.model_test, "name"))
        self.assertEqual(self.model_test.name, "")

if __name__ == '__main__':
    unittest.main()


from redis import Redis
from asyncspider import AsyncSpider
import json
import asyncio

rds = Redis(host='localhost',port=6379,db=0)
spr = AsyncSpider()

async def worker(i):
	import time
	print('worker:',i)
	while True:
		msg = rds.rpop('moxing_msg')
		if not msg:
			time.sleep(1)
			continue
		msg = msg.decode('utf-8')
		argues = msg.split('_')
		fid, page= argues[0],int(argues[1])
		keys = 'moxing_'+fid+'_'+str(page)

		start = time.time()
		print('start:',fid,page)

		res = rds.get(keys)
		if not res:
			infos = await spr.get_page_list(fid=fid,page=page,start_page=page,objs=[])
			rds.set(keys,json.dumps(infos).encode('utf-8'))
			rds.expire(keys,600)
			print('job done',msg,len(infos[0]))

		end = time.time()
		print('end:  ',fid,page,end-start,'s')

if __name__ == '__main__':
	import time
	for i in range(10):
		asyncio.ensure_future(worker(i))
	loop = asyncio.get_event_loop()
	try:
		loop.run_forever()
	except KeyboardInterrupt as e:
		print(asyncio.gather(*asyncio.Task.all_tasks()).cancel())
		loop.stop()
		loop.run_forever()
	finally:
		loop.close()


# File: yak.py
# Yak Message Model

from datetime import datetime
def parseTime(timeStr):
  format = '%Y-%m-%d %H:%M:%S'
  return datetime.strptime(timeStr, format)

class Yak:
  def __init__(self, raw, client):
    self.client = client
    self.posterID = raw['posterID']
    self.hidePin = bool(int(raw['hidePin']))
    self.messageID = raw['messageID']
    self.deliveryID = raw['deliveryID']
    self.longitude = raw['longitude']
    self.comments = int(raw['comments'])
    self.time = parseTime(raw['time'])
    self.latitude = raw['latitude']
    self.likes = int(raw['numberOfLikes'])
    self.message = raw['message']
    self.type = raw['type']
    self.liked = int(raw['liked'])
    self.reyaked = raw['reyaked']

    # yaks do not always have a handle
    try: self.handle = raw['handle']
    except KeyError: self.handle = None

    # for some reason this seems necessary
    self.messageID = self.messageID.replace('\\', '')

  def upvote(self):
    if self.liked == 0:
      self.liked += 1
      self.likes += 1

      # only triggers if not already voted on
      return self.client.upvoteYak(self.messageID)

  def downvote(self):
    if self.liked == 0:
      self.liked -= 1
      self.likes -= 1

      # only triggers if not already voted on
      return self.client.downvoteYak(self.messageID)

  def report(self):
    return self.client.reportYak(self.messageID)

  def delete(self):
    if self.posterID == self.client.ID:
      return self.client.deleteYak(self.messageID)

  def addComment(self, comment):
    return self.client.postComment(self.messageID, comment)

  def getComments(self):
    return self.client.getComments(self.messageID)


from django import forms
from django.shortcuts import render, redirect
from django.http import HttpResponse, HttpResponseRedirect
from django.contrib.auth import login,authenticate, logout
from .models import ServicesRequestParticipant, ServicesRequest, Awards, SR_LOGS, SRP_LOGS, ReBug_LOGS, Report_Bug
from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import User
from django.contrib import messages
from .forms import Admin_CuProForm, addSRForm, SRPForm, AwardsForm, Admin_CyProForm, Admin_SRForm, Admin_SRPForm, Admin_SR_LOGS, Admin_SRP_LOGS, ReportBugForm, ReBug_LOGSForm, Admin_ReportBugForm, Admin_Cus_LOGS, Admin_Cy_LOGS, Admin_SRPForm_Status
from users.models import customerProfile,cyProfile, Cus_LOGS, Cy_LOGS

# Create your views here.
@login_required(login_url="cylogin")
def Service_Requests(request):

    if request.user.last_name != 'cy':
        return redirect('index')

    SRQ= ServicesRequest.objects.all()
    SRP=ServicesRequestParticipant.objects.all()
    val=1
    context ={'SRQ':SRQ, 'SRP':SRP, 'val':val}
    return render(request, 'Service_Requests.html',context)

@login_required(login_url="cylogin")
def single(request, pk):

    if request.user.last_name != 'cy' and request.user.last_name != 'org':
        return redirect('index')

    obj= ServicesRequest.objects.get(SR_ID=pk)
    obj_SRP= ServicesRequestParticipant.objects.filter(SR_ID=obj.SR_ID)
    srp_logs=Admin_SRP_LOGS()

    form =SRPForm()
    page="cy"

    if request.method =='POST':
        form = SRPForm(request.POST, request.FILES)
        srp_logs=Admin_SRP_LOGS(request.POST)
        if form.is_valid():
            form = form.save(commit=False)

            srp_logs = srp_logs.save(commit=False)

            form.CyRes_ID=request.user.cyprofile
            #hform.SR_ID=obj.SR_Organization_Name
            form.SR_ID=ServicesRequest.objects.get(SR_ID=pk)

            LOCon=form.SRP_Loss_of_Confidentiality
            LOIntg=form.SRP_Loss_of_Integrity
            LOAva=form.SRP_Loss_of_Availability
            LOAcc=form.SRP_Loss_of_Accountability

            con=1
            intg=1
            ava=1
            acc=1

            if LOCon == 'Low':
                con = 2
            elif LOCon == 'Medium':
                con = 4
            elif LOCon == 'High':
                con = 7
            elif LOCon == 'Critical':
                con = 9

            if LOIntg == 'Low':
                intg = 2
            elif LOIntg == 'Medium':
                intg = 4
            elif LOIntg == 'High':
                intg = 7
            elif LOIntg == 'Critical':
                intg = 9

            if LOAva == 'Low':
                ava = 2
            elif LOAva == 'Medium':
                ava = 4
            elif LOAva == 'High':
                ava = 7
            elif LOAva == 'Critical':
                ava = 9

            if LOAcc == 'Low':
                acc = 2
            elif LOAcc == 'Medium':
                acc = 4
            elif LOAcc == 'High':
                acc = 7
            elif LOAcc == 'Critical':
                acc = 9

            tot=(con + intg + ava + acc) / 4
            danger=" "

            if tot < 3 :
                danger="Low"
            elif tot >= 3 and tot < 6:
                danger ="Medium"
            elif tot >=6 and tot < 8:
                danger="High"
            else:
                danger="Critical"

            form.SRP_Total = tot
            form.SRP_Danger = danger

            form.save()

            srp_logs.SRP_LOG_Status=form.SRP_Status
            srp_logs.SRP_ID=form
            srp_logs.SRP_LOG_User=request.user.username

            srp_logs.SRP_LOG_Loss_of_Confidentiality = form.SRP_Loss_of_Confidentiality
            srp_logs.SRP_LOG_Loss_of_Integrity = form.SRP_Loss_of_Integrity
            srp_logs.SRP_LOG_Loss_of_Availability =form.SRP_Loss_of_Availability
            srp_logs.SRP_LOG_Loss_of_Accountability =form.SRP_Loss_of_Accountability
            #srp_logs.SRP_LOG_Technical_Total = form.SRP_Total
            srp_logs.SRP_LOG_Technical_Total = tot
            srp_logs.SRP_LOG_Danger = danger

            srp_logs.save()

            messages.success(request,'Report submitted successfully')

            return redirect('index')

        else:
            messages.error(request,'An error happend!')

    val=0
    context ={'obj':obj,'form':form, 'page':page, 'obj_SRP':obj_SRP, 'val':val}
    return render(request, 'Single_Request.html',context)

@login_required(login_url="cylogin")
def Add_Service_Requests(request):

    if request.user.last_name != 'org':
        return redirect('index')

    page = '1'
    form =addSRForm(request.POST, request.FILES)

    if request.method =='POST':
        form = addSRForm(request.POST, request.FILES)
        if form.is_valid():
            hform = form.save(commit=False)
            hform.SR_Organization_Name=request.user.customerprofile.CuRes_Org_Name
            hform.CuRes_ID=request.user.customerprofile

            package= ""
            if request.POST['select'] == "1":
                package= "Basic Package"
            elif request.POST['select'] == "2":
                package= "Advance Package"
            else:
                package= "Legand Package"

            hform.SR_Package=package

            hform.save()

            messages.success(request,'Request created successfully')

            return redirect('cuRequests')

        else:
            messages.error(request,'An error happend!')

    context={'page':page,'form':form}

    return render(request, 'Add_Service_Requests.html', context)

@login_required(login_url="cylogin")
def payment(request, pk):
    # page = '1'
    # form =addSRForm()

    # if request.method =='POST':
    #     form = addSRForm(request.POST)
    #     if form.is_valid():
    #         hform = form.save(commit=False)
    #         hform.CuRes_ID=request.user.customerprofile
    #         hform.save()

    #         messages.success(request,'User account created')

    #         return redirect('profile')

    #     else:
    #         messages.error(request,'An error happend!')

    # context={'page':page,'form':form}

    return render(request, 'payment.html')

@login_required(login_url="cylogin")
def customerRequests(request):

    if request.user.last_name != 'org':
        return redirect('index')

    SRQ= ServicesRequest.objects.all().order_by('-SR_Created')
    context ={'SRQ':SRQ}
    return render(request, 'customer_requests.html',context)

@login_required(login_url="cylogin")
def AddAward(request, pk):

    if request.user.last_name != 'admin':
        return redirect('index')

    obj= ServicesRequestParticipant.objects.get(SRP_ID=pk)
    form =AwardsForm()
    page="cy"

    if request.method =='POST':
        form = AwardsForm(request.POST)
        if form.is_valid():
            form = form.save(commit=False)
            #hform.SR_ID=obj.SR_Organization_Name
            form.SRP_ID=ServicesRequestParticipant.objects.get(SRP_ID=pk)
            form.SRA_User_Created=request.user.username
            form.SRA_Danger_Of_Vuln=obj.SRP_Total

            prize=""
            if obj.SRP_Danger == 'Low':
                prize=obj.SR_ID.SR_Low_Award
            elif obj.SRP_Danger == 'Medium':
                prize=obj.SR_ID.SR_Medium_Award
            elif obj.SRP_Danger == 'High':
                prize=obj.SR_ID.SR_High_Award
            elif obj.SRP_Danger == 'Critical':
                prize=obj.SR_ID.SR_Critical_Award
            else:
                prize=0

            form.SRA_prize=prize
            obj.SRP_Status="Ended"

            form.save()
            obj.save()

            messages.success(request,'Award added successfully')

            return redirect('adminp')

        else:
            messages.error(request,'An error happend!')

    context ={'obj':obj,'form':form, 'page':page}
    return render(request, 'Add_Award.html',context)

@login_required(login_url="cylogin")
def AdminCy(request, pk):

    if request.user.last_name != 'admin':
        return redirect('index')

    obj= cyProfile.objects.get(CyRes_ID=pk)
    form =Admin_CyProForm(instance=obj)
    Cy_Logs=Admin_Cy_LOGS()
    Cy_Logs_obj=Cy_LOGS.objects.filter(CyRes_ID=pk).order_by('-Cy_Created')

    page="cy"

    if request.method =='POST':
        form = Admin_CyProForm(request.POST, request.FILES,  instance=obj)
        Cy_Logs=Admin_Cy_LOGS(request.POST)

        if form.is_valid():
            form = form.save(commit=False)
            Cy_Logs = Cy_Logs.save(commit=False)
            #form.CyRes_ID=cyProfile.objects.get(CyRes_ID=pk)

            Cy_Logs.CyRes_ID=form
            Cy_Logs.Cy_LOG_Status=form.CuRes_Status
            Cy_Logs.Cy_LOG_Sector=form.CyRes_Service_Type
            Cy_Logs.Cy_LOG_Note=form.CyRes_Note
            Cy_Logs.Cy_LOG_User=request.user.username

            form.save()
            Cy_Logs.save()

            messages.success(request,'cyber Researcher Changed successfully')

            return redirect('adminp')

        else:
            messages.error(request,'An error happend!')

    context ={'obj':obj,'form':form, 'page':page, 'Cy_Logs_obj':Cy_Logs_obj}
    return render(request, 'admin/admin_cyPro.html',context)

@login_required(login_url="cylogin")
def AdminCu(request, pk):

    if request.user.last_name != 'admin':
        return redirect('index')

    obj= customerProfile.objects.get(CuRes_ID=pk)
    form =Admin_CuProForm(instance=obj)
    Cus_Logs=Admin_Cus_LOGS()
    Cus_Logs_obj=Cus_LOGS.objects.filter(CuRes_ID=pk).order_by('-Cus_Created')

    page="org"

    if request.method =='POST':
        form = Admin_CuProForm(request.POST, request.FILES,  instance=obj)
        Cus_Logs=Admin_Cus_LOGS(request.POST)
        if form.is_valid():
            form = form.save(commit=False)
            Cus_Logs = Cus_Logs.save(commit=False)

           #form.CyRes_ID=cyProfile.objects.get(CyRes_ID=pk)
            Cus_Logs.CuRes_ID=form
            Cus_Logs.Cus_LOG_Status=form.CuRes_Status
            Cus_Logs.Cus_LOG_Sector=form.CuRes_organization_Sector
            Cus_Logs.Cus_LOG_Note=form.CuRes_Note
            Cus_Logs.Cus_LOG_User=request.user.username

            form.save()
            Cus_Logs.save()

            messages.success(request,'Customer Changed successfully')

            return redirect('adminp')

        else:
            messages.error(request,'An error happend!')

    context ={'obj':obj,'form':form, 'page':page, 'Cus_Logs_obj':Cus_Logs_obj}
    return render(request, 'admin/admin_CuPro.html',context)

@login_required(login_url="cylogin")
def AdminSR(request, pk):

    if request.user.last_name != 'admin':
        return redirect('index')

    obj= ServicesRequest.objects.get(SR_ID=pk)
    form =Admin_SRForm(instance=obj)
    sr_logs=Admin_SR_LOGS()
    SrLogs=SR_LOGS.objects.filter(SR_ID=pk).order_by('-SRA_Created')

    page="cy"

    if request.method =='POST':
        form = Admin_SRForm(request.POST, request.FILES,  instance=obj)
        sr_logs=Admin_SR_LOGS(request.POST)

        if form.is_valid():
            form = form.save(commit=False)
            sr_logs = sr_logs.save(commit=False)

            #form.CyRes_ID=cyProfile.objects.get(CyRes_ID=pk)
            sr_logs.SR_LOG_Status=form.SR_Approval_status
            sr_logs.SR_ID=form
            sr_logs.SR_LOG_User=request.user.username

            sr_logs.save()
            form.save()

            messages.success(request,'Service Request')

            return redirect('adminp')

        else:
            messages.error(request,'An error happend!')

    context ={'obj':obj,'form':form, 'page':page, 'sr_logs':sr_logs, 'SrLogs':SrLogs}
    return render(request, 'admin/admin_change.html',context)

@login_required(login_url="cylogin")
def AdminSRP(request, pk):

    if request.user.last_name != 'admin':
        return redirect('index')

    obj= ServicesRequestParticipant.objects.get(SRP_ID=pk)
    form =Admin_SRPForm(instance=obj)
    srp_logs=Admin_SRP_LOGS()
    SrpLogs=SRP_LOGS.objects.filter(SRP_ID=pk).order_by('-SRP_Created')

    page="cy"

    if request.method =='POST':
        form = Admin_SRPForm(request.POST, request.FILES,  instance=obj)
        srp_logs=Admin_SRP_LOGS(request.POST)

        if form.is_valid():
            form = form.save(commit=False)
            srp_logs = srp_logs.save(commit=False)

            #form.CyRes_ID=cyProfile.objects.get(CyRes_ID=pk)

            srp_logs.SRP_LOG_Status=form.SRP_Status
            srp_logs.SRP_ID=form
            srp_logs.SRP_LOG_User=request.user.username
            srp_logs.SRP_Note=form.SRP_Note

            srp_logs.SRP_LOG_Loss_of_Confidentiality = form.SRP_Loss_of_Confidentiality
            srp_logs.SRP_LOG_Loss_of_Integrity = form.SRP_Loss_of_Integrity
            srp_logs.SRP_LOG_Loss_of_Availability =form.SRP_Loss_of_Availability
            srp_logs.SRP_LOG_Loss_of_Accountability =form.SRP_Loss_of_Accountability

            LOCon=form.SRP_Loss_of_Confidentiality
            LOIntg=form.SRP_Loss_of_Integrity
            LOAva=form.SRP_Loss_of_Availability
            LOAcc=form.SRP_Loss_of_Accountability

            con=1
            intg=1
            ava=1
            acc=1

            if LOCon == 'Low':
                con = 2
            elif LOCon == 'Medium':
                con = 4
            elif LOCon == 'High':
                con = 7
            elif LOCon == 'Critical':
                con = 9

            if LOIntg == 'Low':
                intg = 2
            elif LOIntg == 'Medium':
                intg = 4
            elif LOIntg == 'High':
                intg = 7
            elif LOIntg == 'Critical':
                intg = 9

            if LOAva == 'Low':
                ava = 2
            elif LOAva == 'Medium':
                ava = 4
            elif LOAva == 'High':
                ava = 7
            elif LOAva == 'Critical':
                ava = 9

            if LOAcc == 'Low':
                acc = 2
            elif LOAcc == 'Medium':
                acc = 4
            elif LOAcc == 'High':
                acc = 7
            elif LOAcc == 'Critical':
                acc = 9

            tot=(con + intg + ava + acc) / 4

            danger=" "

            if tot < 3 :
                danger="Low"
            elif tot >= 3 and tot < 6:
                danger ="Medium"
            elif tot >=6 and tot < 8:
                danger="High"
            else:
                danger="Critical"

            form.SRP_Danger = danger
            srp_logs.SRP_LOG_Danger = danger

            form.SRP_Technical_Total = tot
            srp_logs.SRP_LOG_Technical_Total = tot

            srp_logs.save()
            form.save()

            messages.success(request,'Services Request Participant Changed Successfully')

            return redirect('adminp')

        else:
            messages.error(request,'An error happend!')

    context ={'obj':obj,'form':form, 'page':page, 'srp_logs':srp_logs, 'SrpLogs':SrpLogs}
    return render(request, 'admin/admin_change_SRP.html',context)

def ReportBug(request):

    form=ReportBugForm(request.POST)

    if request.method =='POST':
        form=ReportBugForm(request.POST)

        if form.is_valid():
            form = form.save(commit=False)
            Re_UserName=" "
            Re_Cyb=""
            if request.user.is_authenticated:
                Re_UserName="User Name: ("+request.user.username + ") In Form name: " + request.POST['Re_UserName']
                if request.user.last_name == "cy":
                    Re_Cyb=request.user.cyprofile
                    form.CyRes_ID=Re_Cyb

            else:
                Re_UserName = request.POST['Re_UserName'] + " (Not Registered)"

            Re_Phone = request.POST['Re_Phone']
            Re_Email = request.POST['Re_Email']
            Re_Title = request.POST['Re_Title']
            Re_Description = request.POST['Re_Description']
            Re_Report = request.POST['Re_Report']

            form.Re_UserName=Re_UserName
            form.Re_Phone=Re_Phone
            form.Re_Email=Re_Email
            form.Re_Title=Re_Title
            form.Re_Description=Re_Description
            form.Re_Report=Re_Report

            form.save()

            messages.success(request,'Report Submitted Successfully')

            return redirect('index')

        else:
            messages.error(request,'An error happend!')

    context ={'form':form}
    return render(request, 'report_bug.html',context)

@login_required(login_url="cylogin")
def AdminReBUg(request, pk):

    if request.user.last_name != 'admin':
        return redirect('index')

    obj= Report_Bug.objects.get(Re_ID=pk)
    form =Admin_ReportBugForm(instance=obj)
    Re_log=ReBug_LOGSForm()
    Relogs=ReBug_LOGS.objects.filter(Re_ID=pk).order_by('-ReBug_Created')

    if request.method =='POST':
        form = Admin_ReportBugForm(request.POST, request.FILES,  instance=obj)
        Re_log=ReBug_LOGS(request.POST)

        if form.is_valid():
            form = form.save(commit=False)

            Re_log.ReBug_LOG_Status=form.Re_Status
            Re_log.Re_ID=form
            Re_log.ReBug_LOG_User=request.user.username

            Re_log.save()
            form.save()

            messages.success(request,'Report Updated Successfully')

            return redirect('adminp')

        else:
            messages.error(request,'An error happend!')

    context ={'obj':obj,'form':form, 'Re_log':Re_log, 'Relogs':Relogs}
    return render(request, 'admin/admin_change_ReportBug.html',context)


from django.shortcuts import render
from .models import *
from .serializers import *
from rest_framework.decorators import api_view
from rest_framework.response import Response

@api_view(['GET'])
def animalFoods(request):
    animalFoods = AnimalFood.objects.all()
    serializer = AnimalFoodSerializer(animalFoods,many=True)
    return Response(serializer.data)

@api_view(["GET"])
def animalFood(request,pk):
    animalFood = AnimalFood.objects.get(pk=pk)
    serializer = AnimalFoodSerializer(animalFood,many=False)
    return Response(serializer.data)

@api_view(["POST"])
def createAnimalFood(request):
    data = request.data
    animalFood = AnimalFood.create(
        name = data['name'],
        brand = data['brand'],
        description = data['description'],
        category = data['category'],
        price = data['price'],
        image = data['image'],
        quantity = data['quantity'],
        provider = data['provider']

    )
    serializer = AnimalFoodSerializer(animalFood)

    return Response(serializer.data)


class CircularQueue:
    def __init__(self,size):
        self.size=size
        #can use self.queue=[None]*size
        self.queue=[None for i in range(size)]
        self.front=self.rear=-1
    def enqueue(self,data):
        #condition if queue is full
        if((self.rear+1)%self.size==self.front):#size 6 index from 0 to 5
            print("Queue is full\n")
        #condtion for empty queue
        elif (self.front==-1):
            self.front=0
            self.rear=0
            self.queue[self.rear]=data
            #always add at rear place
        else:
            #next position of rear
            self.rear=(self.rear+1)%self.size
            self.queue[self.rear]=data
    def dequeue(self):
        if(self.front==-1):
            #condition for empty space
            print("queue is empty")
            #condition for only one eleement
        elif (self.front==self.rear):
            temp=self.queue[self.front]
            self.front=-1
            self.rear=-1
            return temp
        else:
            temp=self.queue[self.front]
            self.front=(self.front+1)%self.size
            return temp
    def display(self):
        #condition for empty queue
        if(self.front==-1):
            print("queue is empty")
        elif (self.rear>=self.front):
            print("elements in the circular queue",end="  ")
            for i in range(self.front,self.rear+1):
                print(self.queue[i],end=" ")
            print()
        else:
            print("elements in circular queue are:",end=" ")
            for i in range(self.front,self.size):
                print(self.queue[i],end=" ")
            for i in range(0,self.rear+1):
                print(self.queue[i],end=" ")
            print()
        if ((self.rear+1)%self.size==self.front):
            print("queue is full")
obj=CircularQueue(5)
obj.enqueue(14)
obj.enqueue(22)
obj.enqueue(13)
obj.enqueue(-6)
obj.display()
print("deleted elmt:",obj.dequeue())
print("deleted elmt:",obj.dequeue())
obj.display()
obj.enqueue(10)
obj.enqueue(9)
obj.enqueue(8)
obj.display()
#it want to be inserted becoz queue is full
obj.enqueue(100)


from random import shuffle
from itertools import permutations

from utils import argv, read, write

PRECISION = 3

def gen(alph):
    key = list(alph)
    shuffle(key)
    return ''.join(key)

def handle_key(key, alph):
    try:
        key = int(key)
        key = alph[key:] + alph[:key]
    except ValueError:
        pass
    return key

def encrypt(msg, key, alph):
    key = handle_key(key, alph)
    return ''.join(map(lambda c: key[alph.index(c)] if c in alph else c, msg))

def decrypt(enc, key, alph):
    key = handle_key(key, alph)
    return ''.join(map(lambda c: alph[key.index(c)] if c in alph else c, enc))

def attack_shift(freq_true, freq_enc, alph, precision):
    keys = images(freq_true, freq_enc, precision)
    shifts = {}
    for key in keys:
        key = ''.join(sorted(key, key=lambda x: alph.index(freq_true[key.index(x)][0])))
        for idx, letter in enumerate(key):
            shift = (alph.index(letter) - idx) % len(alph)
            shifts[shift] = shifts.get(shift, 0) + 1
    # берем только самые большие по количеству смещения
    min_count, max_count = min(shifts.values()), max(shifts.values())
    border_value = min_count + (max_count - min_count) * 0.5
    shifts = {shift: count for shift, count in shifts.items() if count > border_value}
    shifts = list(sorted(shifts.keys(), key=lambda x: shifts[x], reverse=True))
    return [alph] + list(map(str, shifts[:3]))

def brute(groups, idx, cont):
    if idx == len(groups):
        yield cont
        return
    for perm in permutations(groups[idx]):
        cont.extend(perm)
        yield from brute(groups, idx + 1, cont)
        for _ in range(len(perm)):
            cont.pop()

def images(freq_true, freq_enc, precision):
    freq_true = ''.join(map(lambda pair: pair[0], freq_true))

    for idx, (letter, freq) in enumerate(freq_enc):
        freq_new = freq[:2] + freq[2:2 + precision + 1]
        freq_enc[idx] = (letter, freq_new)

    groups = []
    while len(freq_enc):
        groups.append([freq_enc[0][0]])
        value = freq_enc[0][1]
        freq_enc = freq_enc[1:]
        while len(freq_enc) and freq_enc[0][1] == value and float(value) > 0:
            groups[-1].append(freq_enc[0][0])
            freq_enc = freq_enc[1:]

    res = [''.join(map(lambda pair: pair[0], freq_true))]
    for key in brute(groups, 0, []):
        res.append(''.join(key))
        if len(res) > 1000:
            break
    return res

def exec_encrypt():
    msg = read(argv[2]).lower()
    alph, key = read(argv[3]).split('\n')
    enc = encrypt(msg, key, alph)
    write('enc.txt', enc)

def exec_decrypt():
    enc = read(argv[2]).lower()
    alph, key = read(argv[3]).split('\n')
    dec = decrypt(enc, key, alph)
    write('dec.txt', dec)


import gc

import imageio
import numpy as np
from tqdm import tqdm

import autoencoder

filename = "bladerunner2049.mp4"
reader = imageio.get_reader(filename, "ffmpeg")
fps = reader.get_meta_data()['fps']

frames = [val for val in tqdm(reader, desc="Reading video")]
del reader
gc.collect()
N = len(frames) - 1
print("Video has {} frames".format(N))

h, w, z = frames[0].shape
frames = np.stack(frames)
frames = frames.reshape((-1, h*w*z))

autoencode_and_generate = autoencoder.create_model(h, w, z)
gc.collect()
output_frames = autoencode_and_generate(frames, frames)

import ipdb
ipdb.set_trace()

writer = imageio.get_writer("output/" + filename, fps=fps)


import math
import numpy as np
from scipy.special import gamma, gammainc, gammaincc, gammainccinv, erfc, erfcx, erfcinv
from scipy.stats import norm

# from joblib import Parallel, delayed
# from multiprocessing import Pool

__doc__ = """
This file contains all level-spacing and width distributions.
"""

# =================================================================================================
#    Level-Spacing Probability Distributions
# =================================================================================================

def _gamma_ratio(x):
    'Gamma(x/2) / Gamma((x-1)/2)'
    rpii = 1.0 / math.sqrt(math.pi)
    if hasattr(x,'__iter__'):
        r = np.zeros(len(x))
        for idx, w in enumerate(x):
            q = rpii
            for i in range(3,int(w)):       q = (i-2) / (2*q)
            r[idx] = q
    else:
        r = rpii
        for i in range(3,int(x)+1):     r = (i-2) / (2*r)
    return r

def _high_order_variance(n):
    a = (n**2 + 5*n + 2)/2
    B =(_gamma_ratio(a+2) / (n+1))**2
    return (a+1)/(2*B) - (n+1)**2

def _high_order_level_spacing_pdf(X, n):
    """
    ...

    Source: https://journals.aps.org/pre/pdf/10.1103/PhysRevE.60.5371
    """
    if n <= 15: # Lower n --> Exact Calculation
        a = (n**2 + 5*n + 2)/2
        rB = _gamma_ratio(a+2) / (n+1)
        C = 2 * rB / gamma((a+1)/2)
        rBX = rB * X
        return C * rBX**a * np.exp(-rBX**2)
    else: # Higher n --> Gaussian Approximation:
        sig = np.sqrt(_high_order_variance(n))
        return norm.pdf(X, n+1, sig)

def level_spacing_pdf(X, Freq:float=1.0, w:float=1.0, pM:float=0.0, err:float=5e-3):
    """
    ...
    """
    if pM == 0.0:    # "True" Level-spacings
        if w == 1.0: # Wigner PDF
            c = (math.pi/4) * Freq*Freq
            Y = (2*c) * X * np.exp(-c * X*X)
        else:        # Brody PDF
            w1i = 1 / (w + 1)
            a = (w1i * gamma(w1i) * Freq) ** (w+1)
            Xw = X**w
            Y = (w+1) * a * Xw * np.exp(-a * Xw*X)
    else:            # Missing Level-spacings
        if w == 1.0:
            N_max = math.ceil(math.log(err, pM))
            Y = Freq * (1-pM) * np.sum([pM**k * _high_order_level_spacing_pdf(X*Freq,k) for k in range(N_max+1)], axis=0)
            Y /= 1 - pM**(N_max+1) # Renormalize to allieviate error
        else:
            raise NotImplementedError('Brody distribution with missing resonances has not been implemented yet')
    return Y

# =================================================================================================
#    Width Probability Distributions
# =================================================================================================

def Porter_Thomas_pdf(G, Gm, trunc:float=0.0, DOF:int=1):
    """
    ...
    """
    if trunc == 0.0:
        prob = DOF*(DOF*G/(2*Gm))**(DOF/2-1) * np.exp(-DOF*G/(2*Gm)) / (2 * Gm * gamma(DOF/2))
    else:
        prob = np.zeros(len(G))
        prob[G >  trunc] = DOF*(DOF*G[G > trunc]/(2*Gm))**(DOF/2-1) * np.exp(-DOF*G[G > trunc]/(2*Gm)) / (2 * Gm * gamma(DOF/2) * gammaincc(DOF/2, DOF*trunc/(2*Gm)))
        prob[G <= trunc] = 0.0
    return prob

def Porter_Thomas_cdf(G, Gnm:float=1.0, trunc:float=0.0, DOF:int=1):
    """
    ...
    """
    if trunc == 0.0:
        prob = gammainc(DOF/2, DOF*G/(2*Gnm))
    else:
        prob = np.zeros(len(G))
        prob[G >  trunc] = (gammainc(DOF/2, DOF*G[G>=trunc]/(2*Gnm)) - gammainc(DOF/2, DOF*trunc/(2*Gnm))) / gammaincc(DOF/2, DOF*trunc/(2*Gnm))
        prob[G <= trunc] = 0.0
    return prob

def FractionMissing(trunc:float, Gnm:float=1.0, DOF:int=1):
    'Gives the fraction of missing resonances due to the truncation in neutron width.'
    return gammainc(DOF/2, DOF*trunc/(2*Gnm))

def _high_order_level_spacing_parts(X, n):
    """
    ...

    Source: https://journals.aps.org/pre/pdf/10.1103/PhysRevE.60.5371
    """
    if n <= 15: # Lower n --> Exact Calculation
        a = (n**2 + 5*n + 2)/2
        rB = _gamma_ratio(a+2) / (n+1)
        C = 2 * rB / gamma((a+1)/2)
        rBX  = rB * X
        F0 = C * rBX**a * np.exp(-rBX**2)
        F1 = (C/2) * gammaincc((a+1)/2, rBX**2)
        F2 = (C/2) * gammaincc((a+2)/2, rBX**2)
    else: # Higher n --> Gaussian Approximation:
        sig = np.sqrt(_high_order_variance(n))
        F0 = norm.pdf(X, n+1, sig)
        F1 = (1/2) * erfc((X+n+1)/(sig*math.sqrt(2)))
        F2 = sig**2*F0 + (X+1)*F1 - 1
    return F0, F1, F2

def missing_pdf(X, Freq:float=1, pM:float=0, err:float=5e-3):
    """
    ...
    """
    N_max = math.ceil(math.log(err, pM))
    Y0, Y1, Y2 = 0, 0, 0
    for k in range(N_max+1):
        F0, F1, F2 = _high_order_level_spacing_parts(X*Freq, k)
        Y0 += pM**k * F0
        Y1 += pM**k * F1
        Y2 += pM**k * F2

    multfact = Freq * (1-pM) / (1 - pM**(N_max+1))
    Y0 *= multfact
    Y1 *= multfact
    Y2 *= multfact
    return Y0, Y1, Y2

def _gamma_ratio(x):
    'Gamma(x/2) / Gamma((x-1)/2)'
    rpii = 1.0 / math.sqrt(math.pi)
    if hasattr(x,'__iter__'):
        r = np.zeros(len(x))
        for idx, w in enumerate(x):
            q = rpii
            for i in range(3,int(w)):       q = (i-2) / (2*q)
            r[idx] = q
    else:
        r = rpii
        for i in range(3,int(x)+1):     r = (i-2) / (2*r)
    return r

def _high_order_variance(n):
    a = (n**2 + 5*n + 2)/2
    B =(_gamma_ratio(a+2) / (n+1))**2
    return (a+1)/(2*B) - (n+1)**2

def _high_order_level_spacing_parts(X, n, parts:tuple=(0,1,2)):
    """
    ...

    Source: https://journals.aps.org/pre/pdf/10.1103/PhysRevE.60.5371
    """
    out = []
    if n <= 15: # Lower n --> Exact Calculation
        a = (n**2 + 5*n + 2)/2
        rB = _gamma_ratio(a+2) / (n+1)
        C = 2 * rB / gamma((a+1)/2)
        rBX  = rB * X
        for part in parts:
            if   part == 0:     out.append(C * rBX**a * np.exp(-rBX**2))
            elif part == 1:     out.append((C/2) * gammaincc((a+1)/2, rBX**2))
            elif part == 2:     out.append((C/2) * gammaincc((a+2)/2, rBX**2))
    else: # Higher n --> Gaussian Approximation:
        sig = np.sqrt(_high_order_variance(n))
        for part in parts:
            if   part == 0:
                F0 = norm.pdf(X, n+1, sig)
                out.append(F0)
            elif part == 1:
                F1 = (1/2) * erfc((X+n+1)/(sig*math.sqrt(2)))
                out.append(F1)
            elif part == 2:
                if 0 not in parts:      F0 = norm.pdf(X, n+1, sig)
                if 1 not in parts:      F1 = (1/2) * erfc((X+n+1)/(sig*math.sqrt(2)))
                out.append(sig**2*F0 + (X+1)*F1 - 1)
    return tuple(out)

class Distribution:
    f"""
    {__name__} is a class for level-spacing distributions, their integrals and inverses.
    """
    def __init__(self, **kwargs):
        """
        ...
        """
        if 'f0' in kwargs:      self.f0 = kwargs['f0']
        if 'f1' in kwargs:      self.f1 = kwargs['f1']
        if 'f2' in kwargs:      self.f2 = kwargs['f2']

        if 'parts' in kwargs:
            self.parts = kwargs['parts']
        elif 'f0' in kwargs and 'f1' in kwargs and 'f2' in kwargs:
            def parts(*X):
                F0 = kwargs['f0'](*X)
                F1 = kwargs['f1'](*X)
                F2 = kwargs['f2'](*X)
                return F2, F0/F1, F1/F2
            self.parts = parts

        if 'if1' in kwargs:      self.if1 = kwargs['if1']
        if 'if2' in kwargs:      self.if2 = kwargs['if2']

    @property
    def pdf(self):
        return self.f0

    def sample(self, size=None, Boundary:bool=False):
        """
        Inverse CDF Sampling
        """
        if Boundary:    return self.if2(np.rand(size))
        else:           return self.if1(np.rand(size))

    @classmethod
    def wigner(cls, Freq:float=1.0):
        """
        ...
        """
        pid4  = math.pi/4
        coef = pid4*Freq**2
        root_coef = math.sqrt(coef)

        def get_f0(X):
            return (2*coef) * X * np.exp(-coef * X*X)
        def get_f1(X):
            return np.exp(-coef * X*X)
        def get_f2(X):
            return erfc(root_coef * X) / Freq
        def get_parts(X):
            fX = root_coef * X
            R1 = 2 * root_coef * fX
            R2 = Freq / erfcx(fX)       # "erfcx(x)" is "exp(x^2)*erfc(x)"
            F2 = np.exp(-fX*fX) / R2    # Getting the "erfc(x)" from "erfcx(x)"
            return F2, R1, R2
        def get_if1(X):
            return np.sqrt(-np.log(X)) / root_coef
        def get_if2(X):
            return erfcinv(Freq * X) / root_coef
        return cls(f0=get_f0, f1=get_f1, f2=get_f2, parts=get_parts, if1=get_if1, if2=get_if2)

    @classmethod
    def brody(cls, Freq:float=1.0, w:float=0.0):
        """
        ...
        """
        w1i = 1.0 / (w+1)
        a = (Freq*w1i*gamma(w1i))**(w+1)

        def get_f0(X):
            aXw = a*X**w
            return (w+1) * aXw * np.exp(-aXw*X)
        def get_f1(X):
            return np.exp(-a*X**(w+1))
        def get_f2(X):
            return (w1i*a**(-w1i)) * gammaincc(w1i, a*X**(w+1))
        def get_parts(X):
            aXw = a*X**w
            aXw1 = aXw*X
            R1 = (w+1)*aXw
            F2 = (w1i * a**(-w1i)) * gammaincc(w1i,aXw1)
            R2 = np.exp(-aXw1) / F2
            return F2, R1, R2
        def get_if1(X):
            return (-np.log(X) / a) ** w1i
        def get_if2(X):
            return gammainccinv(w1i, ((w+1)*a**w1i) * X)
        return cls(f0=get_f0, f1=get_f1, f2=get_f2, parts=get_parts, if1=get_if1, if2=get_if2)

    @classmethod
    def missing(cls, Freq:float=1.0, pM:float=0.0, err:float=5e-3):
        """
        ...
        """
        N_max = math.ceil(math.log(err, pM))
        coef = pM**np.arange(N_max+1)
        mult_fact = Freq * (1-pM) / (1 - pM**(N_max+1))
        def get_f0(X):
            func = lambda _n: _high_order_level_spacing_parts(X, _n, parts=(0,))
            Values = [func(n) for n in range(N_max+1)]
            return np.sum(coef * np.array([v[0] for v in Values]))
        def get_f1(X):
            func = lambda _n: _high_order_level_spacing_parts(X, _n, parts=(1,))
            Values = [func(n) for n in range(N_max+1)]
            return np.sum(coef * np.array([v[0] for v in Values]))
        def get_f2(X):
            func = lambda _n: _high_order_level_spacing_parts(X, _n, parts=(2,))
            Values = [func(n) for n in range(N_max+1)]
            return np.sum(coef * np.array([v[0] for v in Values]))
        def get_parts(X):
            func = lambda _n: _high_order_level_spacing_parts(X, _n, parts=(0,1,2))
            # Values = Parallel(n_jobs=-1, backend="threading")(map(delayed(func), range(N_max+1)))
            Values = [func(n) for n in range(N_max+1)]
            V0, V1, V2 = zip(*Values)
            F0 = np.sum(coef*V0)
            F1 = np.sum(coef*V1)
            F2 = np.sum(coef*V2)
            # F0 = np.sum(coef * np.array([v[0] for v in Values]))
            # F1 = np.sum(coef * np.array([v[1] for v in Values]))
            # F2 = np.sum(coef * np.array([v[2] for v in Values]))
            R1 = F0 / F1
            R2 = F1 / F2
            return F2, R1, R2
        def get_if1(X):
            raise NotImplemented
        def get_if2(X):
            raise NotImplemented
        return cls(f0=get_f0, f1=get_f1, f2=get_f2, parts=get_parts, if1=get_if1, if2=get_if2)

class Distributions:
    """
    ...
    """
    def __init__(self, *distributions:Distribution):
        'Initializing distributions'
        self.distributions = np.array(distributions)

    def f0(self, X):
        return np.array([distr.f0(X) for distr in self.distributions]).T
    def f1(self, X):
        return np.array([distr.f1(X) for distr in self.distributions]).T
    def f2(self, X):
        return np.array([distr.f2(X) for distr in self.distributions]).T
    def if1(self, X):
        return np.array([distr.if1(X) for distr in self.distributions]).T
    def if2(self, X):
        return np.array([distr.if2(X) for distr in self.distributions]).T
    def parts(self, X):
        return np.array([distr.parts(X) for distr in self.distributions]).T

    def __getitem__(self, key):
        if hasattr(key, '__iter__'):
            return self.__class__(*self.distributions[key])
        else:
            return self.distributions[key]

    @property
    def num(self):
        return len(self.distributions)

    @classmethod
    def wigner(cls, *args, **kwargs):
        if len(args) > 0:
            N = len(args[0])
        elif len(kwargs) > 0:
            N = len(kwargs.keys()[0])
        else:
            raise ValueError()
        distr = []
        for j in range(N):
            argsj   = [arg[j] for arg in args]
            kwargsj = {kwarg : val[j] for kwarg, val in kwargs.items()}
            distr.append(Distribution.wigner(*argsj, **kwargsj))
        return cls(*distr)
    @classmethod
    def brody(cls, *args, **kwargs):
        if len(args) > 0:
            N = len(args[0])
        elif len(kwargs) > 0:
            N = len(kwargs.keys()[0])
        else:
            raise ValueError()
        distr = []
        for j in range(N):
            argsj   = [arg[j] for arg in args]
            kwargsj = {kwarg : val[j] for kwarg, val in kwargs.items()}
            distr.append(Distribution.brody(*argsj, **kwargsj))
        return cls(*distr)
    @classmethod
    def missing(cls, *args, **kwargs):
        if len(args) > 0:
            N = len(args[0])
        elif len(kwargs) > 0:
            N = len(kwargs.keys()[0])
        else:
            raise ValueError()
        distr = []
        for j in range(N):
            argsj   = [arg[j] for arg in args]
            kwargsj = {kwarg : val[j] for kwarg, val in kwargs.items()}
            distr.append(Distribution.missing(*argsj, **kwargsj))
        return cls(*distr)


from typing import List

from django.dispatch import receiver

from classification.enums import ShareLevel
from classification.models import Classification, classification_flag_types, ImportedAlleleInfo, \
    ImportedAlleleInfoStatus
from flags.models import FlagType
from flags.models.flag_health_check import flag_chanced_since
from library.health_check import health_check_signal, \
    HealthCheckRequest, HealthCheckTotalAmount, HealthCheckRecentActivity, HealthCheckStat

"""
Reports information about classifications to the Slack health report
"""

@receiver(signal=health_check_signal)
def allele_info_health_check(sender, health_request: HealthCheckRequest, **kwargs):
    output: List[HealthCheckStat] = []

    for status in [ImportedAlleleInfoStatus.PROCESSING, ImportedAlleleInfoStatus.MATCHED_IMPORTED_BUILD]:
        not_complete = ImportedAlleleInfo.objects.filter(status=status).count()
        if not_complete:
            output.append(HealthCheckTotalAmount(
                emoji=":hourglass_flowing_sand:",
                amount=not_complete,
                name=f"Imported Allele Matching in status of \"{status.label}\""
            ))

    last_failures = ImportedAlleleInfo.objects.filter(
        latest_validation__created__gte=health_request.since,
        latest_validation__created__lte=health_request.now,
        latest_validation__include=False
    ).\
        select_related('imported_genome_build_patch_version')
    last_failures_count = last_failures.count()

    if last_failures_count:
        extra = None
        if last_failures_count <= 5:
            extra = ", ".join(f"{failure.imported_c_hgvs} {failure.imported_genome_build_patch_version}" for failure in last_failures)

        output.append(HealthCheckRecentActivity(
            emoji=":warning:",
            name="Variant Matching Issues",
            amount=last_failures_count,
            extra=extra,
            stand_alone=True  # doesn't make sense ot merge this with other activity numbers
        ))

    return output

@receiver(signal=health_check_signal)
def classifications_health_check_count(sender, health_request: HealthCheckRequest, **kwargs):
    total_classification_qs = Classification.objects.filter(lab__external=False, withdrawn=False, created__lte=health_request.now).exclude(lab__name__icontains='legacy')
    total = total_classification_qs.count()
    total_shared = total_classification_qs.filter(share_level__in=ShareLevel.DISCORDANT_LEVEL_KEYS).count()
    if total:
        percent_shared = 100.0 * (float(total_shared) / float(total))

        return HealthCheckTotalAmount(
            emoji=":blue_book:",
            amount=total,
            name="Classifications",
            extra=f"{int(percent_shared)}% shared"
        )

@receiver(signal=health_check_signal)
def classification_health_check_activity(sender, health_request: HealthCheckRequest, **kwargs):
    # TODO, also want to restrict this data based on now (not too much of an issue if now() is pretty much now)
    classifications_of_interest = Classification.dashboard_report_classifications_of_interest(since=health_request.since)
    new_classification_count = Classification.dashboard_report_new_classifications(since=health_request.since)

    return [
        HealthCheckRecentActivity(
            emoji=":blue_book:",
            amount=new_classification_count,
            name="Classifications",
            sub_type="Created"
        ),
        HealthCheckRecentActivity(
            emoji=":blue_book:",
            amount=len(classifications_of_interest),
            name="Classifications",
            sub_type="Of Interest"
        )
    ]

@receiver(signal=health_check_signal)
def classification_flag_health_check_activity(sender, health_request: HealthCheckRequest, **kwargs):
    flag_deltas = flag_chanced_since(
        health_request.since,
        flag_types=FlagType.objects.filter(context=classification_flag_types.classification_flag_context)
    )
    checks = []
    for flag_delta in flag_deltas:

        parts = []
        if flag_delta.added:
            parts.append(f"+{flag_delta.added}")
        if flag_delta.resolved:
            parts.append(f"-{flag_delta.resolved}")

        if parts:
            checks.append(
                HealthCheckRecentActivity(
                    emoji=":flags:",
                    name=f"Flag {flag_delta.flag_type}",
                    amount=" / ".join(parts)
                )
            )
    return checks


import tensorflow as tf

# 플레이스 홀더 정의
# 정수 자료형 3개를 가진 배열(을 입력받을 것이라고 명시)
a = tf.placeholder(tf.int32,[3])

# 배열의 모든 값을 2배(값*2)하는 연산 정의
b = tf.constant(2) # 상수 2
x2_op = a*b

# 세션 시작
sess = tf.Session()

# 플레이스 홀더에 값을 넣고 실행
# x2_op는 파이썬의 lambda 같은 개념
# 정수형 자료 3개 배열을 입력받을 것이라 명시한 a 에 각 값을 넣고
# x2_op 연산에 값을 매핑하여 run
r1 = sess.run(x2_op, feed_dict={a:[1,2,3]})
print(r1)
r2 = sess.run(x2_op,feed_dict={a:[10,20,10]})
print(r2)

03. AI/03. Deep Learning/2. Tensorflow/1. perception/4.1 placeholder.py


#es un tipo de dato que permite definir a partir de claves y valores

car={
"name":"book",
"quantity":3,
"price":4.99
}

person={
    "first_name":"ryan",
    "last_name":"ray"
}

print(type(person))

print(person.items())

#tambien se pueden definir a partir de una lista

products=[
    {"name":"book", "price":10.99},
    {"name":"laptop", "price":10.99}
]

print(products)


from FaceSoundLink import *
from random import randint

class Brain():
    def __init__(self):
        self.State = "neutral"
        self.Anim = NeutralFace()
        self.change_state = False

    def RandomState(self):
        newState = "neutral"
        dice = randint(0,7)
        if(dice == 0):
            newState = "happy"
            self.Anim = HappyFace()
        if(dice == 1):
            newState = "neutral"
            self.Anim = NeutralFace()
        if(dice == 2):
            newState = "sad"
            self.Anim = SadFace()
        if(dice == 3):
            newState = "angry"
            self.Anim = AngryFace()
        if(dice == 4):
            newState = "sleepy"
            self.Anim = SleepyFace()
        if(dice == 5):
            newState = "love"
            self.Anim = LoveFace()
        if(dice == 6):
            newState = "message"
            self.Anim = MessageFace()
        if(dice == 7):
            newState = "batterie"
            self.Anim = BatterieFace()

        if(not(newState == self.State)):
            self.State = newState
            self.change_state = True
        else:
            self.change_state = False

    def ChangeState(self, State_):

        if(not(State_ == self.State)):
            self.State = State_
            self.change_state = True
        else:
            self.change_state = False


# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import collections
import copy

import jsonschema

from rally.common import cfg
from rally.common import logging
from rally import consts
from rally import exceptions
from rally.task import scenario

LOG = logging.getLogger(__name__)

CONF = cfg.CONF

class TaskConfig(object):
    """Version-aware wrapper around task config."""

    def __init__(self, config):
        """TaskConfig constructor.

        Validates and represents different versions of task configuration in
        unified form.

        :param config: Dict with configuration of specified task
        :raises InvalidTaskException: in case of validation error
        :raises Exception: in case of some unexpected things
        """
        if config is None:
            raise exceptions.InvalidTaskException("It is empty")
        elif not isinstance(config, dict):
            raise exceptions.InvalidTaskException("It is not a dict")

        self.version = str(config.get("version", 1))

        processors = {}
        for name in dir(self):
            if not name.startswith("_process_"):
                continue
            method = getattr(self, name)
            if callable(method):
                version = name[9:].replace("_", ".")
                processors[version] = method

        if self.version not in processors:
            msg = ("Task configuration version %s is not supported. "
                   "Supported versions: %s" %
                   (self.version, ", ".join(processors)))
            raise exceptions.InvalidTaskException(msg)

        config = processors[self.version](config)

        self.title = config.get("title", "Task")
        self.tags = config.get("tags", [])
        self.description = config.get("description", "")

        self.subtasks = []
        for sconf in config["subtasks"]:
            sconf = copy.deepcopy(sconf)

            # fill all missed properties of a SubTask
            sconf.setdefault("tags", [])
            sconf.setdefault("description", "")

            # it is not supported feature yet, but the code expects this
            # variable
            sconf.setdefault("contexts", {})

            workloads = []
            for position, wconf in enumerate(sconf["workloads"]):
                # fill all missed properties of a Workload

                wconf["name"], wconf["args"] = list(
                    wconf["scenario"].items())[0]
                del wconf["scenario"]

                wconf["position"] = position

                if not wconf.get("description", ""):
                    try:
                        wconf["description"] = scenario.Scenario.get(
                            wconf["name"]).get_info()["title"]
                    except (exceptions.PluginNotFound,
                            exceptions.MultiplePluginsFound):
                        # let's fail an issue with loading plugin at a
                        # validation step
                        pass

                wconf.setdefault("contexts", {})

                if "runner" in wconf:
                    runner = list(wconf["runner"].items())[0]
                    wconf["runner_type"], wconf["runner"] = runner
                else:
                    wconf["runner_type"] = "serial"
                    wconf["runner"] = {}

                wconf.setdefault("sla", {"failure_rate": {"max": 0}})

                hooks = wconf.get("hooks", [])
                wconf["hooks"] = []
                for hook_cfg in hooks:
                    hook_cfg["action"] = list(hook_cfg["action"].items())[0]
                    hook_cfg["trigger"] = list(hook_cfg["trigger"].items())[0]
                    wconf["hooks"].append(hook_cfg)

                workloads.append(wconf)
            sconf["workloads"] = workloads
            self.subtasks.append(sconf)

    def to_dict(self):
        """Returns a valid task config dictionary of the latest version."""
        task = collections.OrderedDict({"version": 2})
        task["title"] = self.title
        task["description"] = self.description
        task["tags"] = self.tags
        task["subtasks"] = []
        for subtask in self.subtasks:
            subtask = copy.deepcopy(subtask)
            # we do not allow to setup this property yet
            del subtask["contexts"]
            for w in subtask["workloads"]:
                # it is inner field, hope we will remove it someday
                del w["position"]

                w["scenario"] = {w.pop("name"): w.pop("args")}
                w["runner"] = {w.pop("runner_type"): w["runner"]}
                w["hooks"] = [{"description": h.get("description", ""),
                               "action": dict([h["action"]]),
                               "trigger": dict([h["trigger"]])}
                              for h in w["hooks"]]
            task["subtasks"].append(subtask)
        return task

    CONFIG_SCHEMA_V1 = {
        "type": "object",
        "$schema": consts.JSON_SCHEMA,
        "patternProperties": {
            ".*": {
                "type": "array",
                "items": {
                    "type": "object",
                    "properties": {
                        "args": {"type": "object"},
                        "description": {
                            "type": "string"
                        },
                        "runner": {
                            "type": "object",
                            "properties": {"type": {"type": "string"}},
                            "required": ["type"]
                        },
                        "context": {"type": "object"},
                        "sla": {"type": "object"},
                        "hooks": {
                            "type": "array",
                            "items": {"$ref": "#/definitions/hook"},
                        }
                    },
                    "additionalProperties": False
                }
            }
        },
        "definitions": {
            "hook": {
                "type": "object",
                "properties": {
                    "name": {"type": "string"},
                    "description": {"type": "string"},
                    "args": {},
                    "trigger": {
                        "type": "object",
                        "properties": {
                            "name": {"type": "string"},
                            "args": {},
                        },
                        "required": ["name", "args"],
                        "additionalProperties": False,
                    }
                },
                "required": ["name", "args", "trigger"],
                "additionalProperties": False,
            }
        }
    }

    def _process_1(self, config):
        try:
            jsonschema.validate(config, self.CONFIG_SCHEMA_V1)
        except jsonschema.ValidationError as e:
            raise exceptions.InvalidTaskException(str(e))

        subtasks = []
        for name, v1_workloads in config.items():
            workloads = []
            for v1_workload in v1_workloads:
                v2_workload = copy.deepcopy(v1_workload)
                v2_workload["scenario"] = {name: v2_workload.pop("args", {})}
                v2_workload["contexts"] = v2_workload.pop("context", {})
                if "runner" in v2_workload:
                    runner_type = v2_workload["runner"].pop("type")
                    v2_workload["runner"] = {
                        runner_type: v2_workload["runner"]}
                if "hooks" in v2_workload:
                    hooks = v2_workload["hooks"]
                    v2_workload["hooks"] = []
                    for hook_cfg in hooks:
                        trigger_cfg = hook_cfg["trigger"]
                        v2_workload["hooks"].append(
                            {"description": hook_cfg.get("description"),
                             "action": {
                                 hook_cfg["name"]: hook_cfg["args"]},
                             "trigger": {
                                 trigger_cfg["name"]: trigger_cfg["args"]}}
                        )
                workloads.append(v2_workload)
            subtasks.append({
                "title": name,
                "workloads": workloads,
            })

        return {"title": "Task (adopted from task format v1)",
                "subtasks": subtasks}

    CONFIG_SCHEMA_V2_SINGLE_ENTITY = {
        "type": "object",
        "description": "An object with a single property.",
        "minProperties": 1,
        "maxProperties": 1,
        "patternProperties": {
            ".*": {"type": "object"}
        }
    }

    CONFIG_SCHEMA_V2_HOOK = {
        "type": "object",
        "properties": {
            "action": {
                "type": "object",
                "minProperties": 1,
                "maxProperties": 1,
                "patternProperties": {".*": {}}
            },
            "trigger": CONFIG_SCHEMA_V2_SINGLE_ENTITY,
            "description": {"type": "string"},
        },
        "required": ["action", "trigger"],
        "additionalProperties": False
    }

    CONFIG_SCHEMA_V2_SUBTASK_SIMPLE = {
        "type": "object",
        "$schema": consts.JSON_SCHEMA,
        "properties": {
            "title": {"type": "string", "maxLength": 128},
            "group": {"type": "string"},
            "description": {"type": "string"},
            "tags": {
                "type": "array",
                "items": {"type": "string", "maxLength": 255}
            },
            "scenario": CONFIG_SCHEMA_V2_SINGLE_ENTITY,
            "runner": CONFIG_SCHEMA_V2_SINGLE_ENTITY,
            "sla": {"type": "object"},
            "hooks": {
                "type": "array",
                "items": CONFIG_SCHEMA_V2_HOOK,
            },
            "contexts": {"type": "object"}
        },
        "additionalProperties": False,
        "required": ["title", "scenario"],
    }

    CONFIG_SCHEMA_V2_SUBTASK_COMPLEX = {
        "type": "object",
        "properties": {
            "title": {"type": "string"},
            "group": {"type": "string"},
            "description": {"type": "string"},
            "tags": {
                "type": "array",
                "items": {"type": "string", "maxLength": 255}
            },
            "run_in_parallel": {"type": "boolean"},
            "workloads": {
                "type": "array",
                "minItems": 1,
                "items": {
                    "type": "object",
                    "properties": {
                        "scenario": CONFIG_SCHEMA_V2_SINGLE_ENTITY,
                        "description": {"type": "string"},
                        "runner": CONFIG_SCHEMA_V2_SINGLE_ENTITY,
                        "sla": {"type": "object"},
                        "hooks": {
                            "type": "array",
                            "items": CONFIG_SCHEMA_V2_HOOK,
                        },
                        "contexts": {"type": "object"}
                    },
                    "additionalProperties": False,
                    "required": ["scenario"]
                }
            }
        },
        "additionalProperties": False,
        "required": ["title", "workloads"]
    }

    V2_TOP_ALLOWED_KEYS = [
        "title", "version", "description", "tags", "subtasks"]
    V2_TOP_REQUIRED_KEYS = ["title", "version", "subtasks"]

    @staticmethod
    def _check_title(title, identifier=None):
        identifier = " of %s" % identifier if identifier else ""
        if not isinstance(title, str):
            raise exceptions.InvalidTaskException(
                "Title%s should be a string, but '%s' is found." %
                (identifier, type(title).__name__))

        if len(title) > 255:
            raise exceptions.InvalidTaskException(
                "Title%s should not be longer then 254 char. Use 'description'"
                " field for longer text."
                % identifier)

    @staticmethod
    def _check_tags(tags, identifier=None):
        identifier = " of %s" % identifier if identifier else ""
        if not isinstance(tags, list):
            raise exceptions.InvalidTaskException(
                "Tags%s should be an array(list) of strings, but '%s' is "
                "found."
                % (identifier, type(tags).__name__))

        for tag in tags:
            if not isinstance(tag, str):
                raise exceptions.InvalidTaskException(
                    "Tag '%s'%s should be a string, but '%s' is found." %
                    (tag, identifier, type(tag).__name__))

            if len(tag) > 255:
                raise exceptions.InvalidTaskException(
                    "Tag '%s'%s should not be longer then 254 char."
                    % (tag, identifier))

    def _process_2(self, config):
        # task format v2 is quite complex. To increase UX we need to
        # validate it by steps

        top_keys = set(config.keys())
        missed = set(self.V2_TOP_REQUIRED_KEYS) - top_keys
        if missed:
            if len(missed) > 1:
                raise exceptions.InvalidTaskException(
                    "'%s' are required properties, but they are missed." %
                    "', '".join(sorted(missed)))

            raise exceptions.InvalidTaskException(
                "'%s' is a required property, but it is missed." % missed.pop()
            )

        redundant = top_keys - set(self.V2_TOP_ALLOWED_KEYS)
        if redundant:
            raise exceptions.InvalidTaskException(
                "Additional properties are not allowed ('%s' %s unexpected)." %
                ("', '".join(sorted(redundant)),
                 "were" if len(redundant) > 1 else "was"))

        self._check_title(config["title"])
        self._check_tags(config.get("tags", []))

        if not isinstance(config["subtasks"], list):
            raise exceptions.InvalidTaskException(
                "Property 'subtasks' should be an array(list), but '%s' is "
                "found." % type(config["subtasks"]).__name__)

        for i, subtask in enumerate(config["subtasks"]):
            try:
                if "workloads" not in subtask:
                    jsonschema.validate(
                        subtask, self.CONFIG_SCHEMA_V2_SUBTASK_SIMPLE)
                else:
                    jsonschema.validate(
                        subtask, self.CONFIG_SCHEMA_V2_SUBTASK_COMPLEX)
            except jsonschema.ValidationError as e:
                raise exceptions.InvalidTaskException(
                    "Subtask #%s. %s" % (i + 1, e))

            if "workloads" not in subtask:
                workload = copy.deepcopy(subtask)
                subtask = {"title": workload.pop("title"),
                           "description": workload.pop("description", ""),
                           "tags": workload.pop("tags", []),
                           "workloads": [workload]}
                config["subtasks"][i] = subtask
        return config


import sys
from collections import deque
from datetime import datetime
from heapq import heappop, heappush

input = sys.stdin.readline

# 6571
DP = [0] * 1001
DP[1] = 1
DP[2] = 2
for i in range(3, 1001, 1):
    DP[i] = DP[i-1] + DP[i-2]
while True:
    A, B = map(int, input().split())
    if A == 0 and B == 0:
        break
    ans = 0
    for i in range(1, 1001, 1):
        if A <= DP[i] and DP[i] <= B:
            ans += 1
    print(ans)


from art import logo
bidders = {}
end = False
print('Welcome to blind auction')
print(logo)
while not end:
    name = input("Type your name : ")
    bid = int(input("Your bid is $"))
    bidders[name] = bid
    print("Is there other bidder?. Type 'yes' or 'no'")
    des = input()
    if des == "no":
        end = True
winner = ''
bigbid = 0
for key in bidders:
    curbidder = key
    curbid = bidders[key]
    if bigbid<curbid:
        winner = curbidder
        bigbid = curbid
print(f'The winner is {winner} with bid of ${bigbid}')


from pathlib import Path
import sys
path = str(Path(Path(__file__).parent.absolute()).parent.absolute())

sys.path.insert(0, path)
sys.path.insert(0, path + '/source_community_detection/')
sys.path.insert(0, path + '/source_community_embedding/')
sys.path.insert(0, path + '/source_graph_util/')

import MOSpecG_partitioning_OV as SpecG
import SpecOV_partitioning_v2 as SpecOV
import spectral_functions as sf
import spectral_based as Spectral
import community_embedding as ComEmb
import SpecNMF as SpecNMF
import SpecRp as SpecRp
import metrics as metrics
from sklearn.metrics.pairwise import cosine_similarity,cosine_distances

from sklearn.datasets import load_iris
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.metrics import f1_score

import subprocess
import matplotlib.cm as cm
import os
import sys
import numpy as np
from numpy import linalg
import multiprocessing
import queue
import time
import pandas as pd
import shutil
import igraph
import networkx as nx

very_low_num = 10E-10

#cos_sim=cosine_similarity(A.reshape(1,-1),B.reshape(1,-1))
def wang_Rp_main( it_emb, MO, IT, p_leading, file_graph, file_expected, results_folder, file_com_out, time_folder, pareto_file, num_pareto, ensemble_thr, n_gen, n_pop, p_offs, bool_ov, ov_thr=0,op_com=1,dim_emb=2, char_split=' ',name='', op_var='pdimemb',  lambdav = 1e9,  alpha=0.5, beta=5,bool_link_pred=False,bool_proposed=True,TEST_DISJ_OV=False):
    it_spec=IT
    #it_emb=30
    #pos_neg=True
    pos_neg=False
    print("TEST_DISJ_OV=",TEST_DISJ_OV)
    #M: basis matrix (n x m);
    #U: representations of nodes n x m;
    #H: community indicator matrix (n x k);
    #C: representations of communities (k x m);
    print("-- op_var", op_var)
    start_time_g = time.perf_counter()
    time_g = (time.perf_counter() - start_time_g)
    g = sf.read_construct_g(file_graph, bool_link_pred)    #bool_link_pred==True, isolated also True
    print("vert = ", g.vcount(), "number of edges = ", g.ecount())

    #file_graph = sf.create_edgelist_i0(file_graph)
    sf.create_edgelist_from_pajek(file_graph)
    p_best=None

    #---------------------------- results
    if file_expected is not None:
        #---- Overlapping communities
        if TEST_DISJ_OV==False and bool_ov==True and file_expected is not None:
            com_expected,lin_expected  = sf.read_partition_expected_OV(file_expected, g, char_split="\t")
            p_expected=None
            k_expected = len(lin_expected)
        #---- Disjoint communities
        elif (TEST_DISJ_OV==True or bool_ov==False) and file_expected is not None:
            p_expected = sf.read_partition_expected(file_expected,char_split)
            k_expected = len(pd.unique(p_expected))

        print("* Number of expected communities = ", k_expected)
    else:
        p_expected = None

    #------------ Solve initial problems

    start_time_spec = time.perf_counter()
    #solve H problem
    gamma1=1
    gamma2=1
    gamma3=0
    if op_com=="SpecDecOV" or op_com=="SpecDecOV-Rp":
        nome_alg="SpecDecOV"
        #specOV
        q_best, Sov_best, g, eig_val, ri_vert_pos, ri_vert_neg, Rp, Rn, mod_mat,A,P = SpecOV.spec_ov_main(file_graph, it_spec, p_leading, gamma1, gamma2, gamma3, ov_thr, bool_ov, file_com_out, bool_verbose, out_verbose)
    else:
        if op_com=="SpecG" or op_com=="SpecG-Rp":
            nome_alg="SpecG"
            ##Genetic algorithm
            p_best, graph, params, eig, pop, offs, Sov_best, Rp,Rn,list_gen_Q,mod_mat, A, P = SpecG.SpecG_single_main(False, it_spec, p_leading, file_graph, results_folder, time_folder, pareto_file, num_pareto, ensemble_thr, n_gen, n_pop, p_offs, bool_ov,KNumber_pre=-1,pos_neg=pos_neg,bool_link_pred=bool_link_pred)

        p_best=p_best[0]
        Sov_best=Sov_best[0]
        Rp=Rp[0]
        Rn=Rn[0]
        ri_vert_pos = graph.ri_vert_pos
        ri_vert_neg = graph.ri_vert_neg
        g=graph.g
        A=A.astype(float)
    print("* dim_emb=",dim_emb)
    dim_p = min(dim_emb,Rp.shape[1])

    ##new: embedding 290921
    ri_vert_pos = ri_vert_pos[:,0:dim_p ]
    Rp = Rp[:,0:dim_p ]
    # print("Rp ", Rp.shape)
    # print("Rn ", Rn.shape)

    time_spec = (time.perf_counter() - start_time_spec) + time_g

    if "Rp" not in op_com:
        H = Sov_best
        #print(H)
        #print("here")
    else:
        H, _ = SpecRp.autoencoder_keras(ri_vert_pos,ri_vert_pos, k_expected)

    print("DIM Rp:",Rp.shape[1])
    if bool_proposed == True:
        #U, M, S = solve_wang_U(g,dim_emb=dim_p)
        U, M, S = solve_wang_U(g, p_expected,dim=dim_p)
        #U, M, S = read_attributes(name, g, p_expected, dim=dim_p)
        U0 = U.copy()

    else:
        U, M, S = solve_wang_U(g, p_expected,dim=dim_emb)
        U0 = U.copy()

    if bool_proposed == True:
        C = ( Rp - np.min(Rp) )/ (  np.max(Rp) - np.min(Rp)) #normalizar
    else:
        C = np.full((Rp.shape[0], Rp.shape[1]), 1)

    M0 = M
    S0 = S
    H0 = H.copy()
    C0 = C

    #print(U)
    #if op_com > 0:
    Sov_nonzero = Sov_best[:, np.where( np.all(Sov_best[..., :] == 0, axis=0) == False )[0]]
    Sov_nonzero.shape
        #SpecOV.plot_vertex_vectors(g, min_max_norm(U0), min_max_norm(U0), Rp, Rn, Sov_nonzero, op="emb")

    #WAIT input("init ok. Next?")
    for it in np.arange(0, it_emb):
        print("---------it:", it, "H_UCt")
        #update M
        M = normalize2( update_wang_M(M, S, U) )
		# update U
        U = normalize2(update_wang_U(S, U, M, C, H,alpha)    )
        #PLOT SpecOV.plot_vertex_vectors(g,min_max_norm(U), min_max_norm(U), Rp, Rn, H, op="emb")
		# Update C
        C = normalize2(update_wang_C(C, H, U) )
        #print(M)
        H, p_best = normalize2(update_H(P, H, A, U, C, alpha, beta, lambdav, op_var,g,  ri_vert_pos, ri_vert_neg, Rp, Rn, p_best) )
       # mod0, nmi0, mod, nmi, metric_1, metric_2, f1_macro_spec, f1_micro_spec = metrics_disjoint(H, H,H0, U, U0, ri_vert_pos,g, results_folder, nome_alg, name, file_com_out, file_expected,char_split);

        H_nonzero = H[:, np.where( np.all(H[..., :] == 0, axis=0) == False )[0]]

    H_nonzero = H[:, np.where( np.all(H[..., :] == 0, axis=0) == False )[0]]

    print("H shape")
    print(H.shape)

    # p_best = create_part_from_Sov(H_nonzero).astype(int)
    # plot_embedding(H_nonzero, C,p_best, file+"H_wang.png")
    # plot_embedding(U, C,p_best, file+"U_wang.png")
    # plot_embedding(U0, C,p_best, file+"U0_wang.png")

#---------------------------- results
    if file_expected is not None and bool_link_pred==False:
        #---- Overlapping communities
        if TEST_DISJ_OV ==False and bool_ov==True and file_expected is not None:
            #must save on lines format
            mod0, nmi0, mod, nmi, metric_1, metric_2, f1_macro_spec, f1_micro_spec = metrics.metrics_overlapping(H, H0, U, U0, ri_vert_pos,g, results_folder, nome_alg, name, file_com_out, file_expected );

        #---- Disjoint communities
        elif (TEST_DISJ_OV==True or bool_ov==False) and file_expected is not None:
            mod0, nmi0, mod, nmi, metric_1, metric_2, f1_macro_spec, f1_micro_spec = metrics.metrics_disjoint(H, H,H0, U, U0, ri_vert_pos,g, results_folder, nome_alg, name, file_com_out, file_expected,char_split);

        return H, U, M, S, C, mod0, mod, nmi0, nmi, f1_macro_spec, f1_micro_spec, metric_1, metric_2, time_spec

    else:
        return H, U, M, S, C, -1, -1, -1, -1, -1, -1, -1, -1, -1

#%% NMF - Embedding Wang - main

#cos_sim=cosine_similarity(A.reshape(1,-1),B.reshape(1,-1))
def wang_main( it_emb, MO, it_spec, p_leading, file_graph, file_expected, results_folder, file_com_out, time_folder, pareto_file, num_pareto, ensemble_thr, n_gen, n_pop, p_offs, bool_ov, ov_thr=0,op_com=1,dim_emb=2, char_split=' '):
#if 1==1:
    #M: basis matrix (n x m);
    #U: representations of nodes n x m;
    #H: community indicator matrix (n x k);
    #C: representations of communities (k x m);

    #TODO: calc p_leading to dim_emb

    start_time_g = time.perf_counter()

    g = sf.read_construct_g(file_graph)
    time_g = (time.perf_counter() - start_time_g)

    gamma1=1
    gamma2 =1
    mod_mat = sf.construct_modularity_generalized(g, gamma1, gamma2).transpose()
    A = np.array(g.get_adjacency(attribute="weight").data) #mat_adj
    P = np.array(sf.construct_null_model(g, gamma2)) #mat_null: P

    lambdav = 1e9
    alpha=0.5 #100
    beta=5 #10

    p_best=None
    #------------ Solve initial problems

    start_time_spec = time.perf_counter()
    #solve H problem
    if op_com==1:
        #specOV
        q_best, Sov_best, g, eig_val, ri_vert_pos, ri_vert_neg, Rp, Rn = SpecOV.spec_ov_main(file_graph, it_spec, p_leading, gamma1, gamma2, gamma3, ov_thr, bool_ov, file_com_out, bool_verbose, out_verbose)
    elif op_com==2:
        gamma1=1
        gamma2=1
        print("gammas", gamma1, ",", gamma2)
        ##Genetic algorithm
        p_best, graph, params, eig, pop, offs, Sov_best, Rp,Rn,list_gen_Q = SpecG.SpecG_single_main(False, it_spec, p_leading, file_graph, results_folder, time_folder, pareto_file, num_pareto, ensemble_thr, n_gen, n_pop, p_offs, bool_ov, mo_weight_Qin=gamma1, mo_weight_Qnull=gamma2,gamma3=gamma3, mod_orig=True)

        p_best=p_best[0]
        Sov_best=Sov_best[0]
        Rp=Rp[0]
        Rn=Rp
        ri_vert_pos = graph.ri_vert_pos
        g=graph.g
    elif op_com==3:
        ##Genetic algorithm
        #p_best, graph, params, eig, pop, offs, Sov_best, Rp,list_gen_Q = ensemble_MOSpecG_main(MO_bool, num_it, p_leading, file_graph, results_folder, time_folder, pareto_file, num_pareto, ensemble_thr, n_gen, n_pop, p_offs,  bool_ov, ov_thr)
        p_best, graph, params, eig, pop, offs, Sov_best, Rp, Rn,list_gen_Q = SpecG.SpecG_single_main(False, it_spec, p_leading, file_graph, results_folder, time_folder, pareto_file, num_pareto, ensemble_thr, n_gen, n_pop, p_offs, bool_ov, mo_weight_Qin=gamma1, mo_weight_Qnull=gamma2, mod_orig=False)

        p_best=p_best[0]
        Sov_best=Sov_best[0]
        Rp=Rp[0]
        Rn=Rp
        ri_vert_pos = graph.ri_vert_pos
        g=graph.g

    time_spec = (time.perf_counter() - start_time_spec) + time_g

       # plot_vertex_vectors(g, ri_vert_pos, ri_vert_neg, Rp, Rn, Sov, op="vectors")
    if op_com>0:
        H = Sov_best
        k_com = np.shape(Sov_best)[1]

        #280921 TESTE
        row_sums = H.sum(axis=1)
        H = H / row_sums[:, np.newaxis]

        #280921 TESTE
        row_sums = Rp.sum(axis=1)
        Rp = Rp / row_sums[:, np.newaxis]

    else: #random init
        print("NÃO USAR AQUi")
        #TODO: overlapping
        p_expected = sf.read_partition_expected(file_expected,char_split)
        #k_expected = len(np.unique(p_expected))
        k_com = np.unique(p_expected).shape[0]
        #H = np.random.rand(g.vcount(),k_com)
        #H = np.random.randint(2, size=(g.vcount(),k_com))
        H = np.zeros(( g.vcount(),k_com ))
        for i in np.arange(0,g.vcount()):
            ki = np.random.randint(k_com, size=(1))[0]
            H[i,ki]=1

        Rp=None
        Rn=None

    #TODO: non_zero
    print("NaN----------")
    print(np.isnan( Rp[np.isnan(Rp)==True ] ))

    dim_emb = Rp.shape[1] #160921
    U, M, S = solve_wang_U(g,dim_emb=dim_emb)

    C = np.full((k_com,dim_emb), 1)

    print("NaN----------")
    print(np.isnan( C[np.isnan(C)==True ] ))

    C = update_wang_C(C, H, U) #190921
    #print(C)

    #TODO: non_zero
    print("NaN----------")
    print(np.isnan( C[np.isnan(C)==True ] ))
    #010421:
    #C = Rp[:,0:dim_emb]
    #C must be positive
    #C = (C + abs(np.min(C)))
    #C= C / abs(np.max(C))

    U0 = U
    M0 = M
    S0 = S
    #H0 = H
    H0 = H.copy()
    C0 = C

    print("U NaN----------")
    #print(np.isnan( U[np.isnan(U)==True ] ))
    if op_com > 0:
        #idx = np.argwhere(np.all(Sov_best[..., :] == 0, axis=0))[0]
        #idx= np.where( np.all(Sov_best[..., :] == 0, axis=0) == False )[0]
        Sov_nonzero = Sov_best[:, np.where( np.all(Sov_best[..., :] == 0, axis=0) == False )[0]]
        Sov_nonzero.shape
        #SpecOV.plot_vertex_vectors(g, ri_vert_pos, ri_vert_neg, Rp, Rn, Sov_best, op="vectors") #070321
        #SpecOV.plot_vertex_vectors(g, ri_vert_pos, ri_vert_neg, Rp, Rn, Sov_nonzero, op="vectors") #070321
        #PLOT SpecOV.plot_vertex_vectors(g, ri_vert_pos, ri_vert_neg, Rp, Rn, Sov_nonzero, op="emb") #070321
        #SpecOV.plot_vertex_vectors(g, ri_vert_pos, ri_vert_neg, Rp, Rn, H0, op="emb")

        #SpecOV.plot_vertex_vectors(g, min_max_norm(U0), min_max_norm(U0), Rp, Rn, H0, op="emb")
        SpecOV.plot_vertex_vectors(g, min_max_norm(U0), min_max_norm(U0), Rp, Rn, Sov_nonzero, op="emb")
        #SpecOV.plot_vertex_vectors(g, C, C, Rp, Rn, Sov_best, op="emb")

    #WAIT input("init ok. Next?")
    for it in np.arange(0, it_emb):
        print("it:", it, "H_UCt")
        #update M
        M = update_wang_M(M, S, U)
		# update U
        U = update_wang_U(S, U, M, C, H,alpha)
        #print(U)
        #PLOT SpecOV.plot_vertex_vectors(g,min_max_norm(U), min_max_norm(U), Rp, Rn, H, op="emb")

		# Update C
        C = update_wang_C(C, H, U)

        #TODO FUNCTION
        PH = np.dot(P, H)
        HHH = np.dot(H , np.dot( H.transpose() , H ) )

        #delta =  (2*beta*mult_null_H)^2 + 16*lambda*HHH * (2*beta*mat_adj%*%H + 2*alpha*U%*%t(C) + (4*lambda - 2*alpha)*H )
        delta= np.power(2*beta*PH,2) + np.multiply( 16*lambdav*HHH , ( 2*beta*np.dot(A,H) + 2*alpha* np.dot(U,C.transpose()) + (4*lambdav - 2*alpha)*H  ) )
        H = np.multiply(H, ( np.sqrt( (-2*beta*PH + np.sqrt(delta)) /  np.maximum( very_low_num, (8*lambdav*HHH)) ) ) )

        #280921 TESTE
        row_sums = H.sum(axis=1)
        H = H / row_sums[:, np.newaxis]

        H_nonzero = H[:, np.where( np.all(H[..., :] == 0, axis=0) == False )[0]]
        #PLOT SpecOV.plot_vertex_vectors(g,min_max_norm(U), min_max_norm(U), Rp, Rn, H_nonzero, op="emb")
        #print(U)
        print("U NaN----------")
        print(np.isnan( U[np.isnan(U)==True ] ))

    #input("it " + str(it) + " ok. Next?")
    # SpecOV.plot_vertex_vectors(g, ri_vert_pos, ri_vert_neg, Rp, Rn, H, op="emb")
    H_nonzero = H[:, np.where( np.all(H[..., :] == 0, axis=0) == False )[0]]
    #PLOT

    if op_com > 0:
        SpecOV.plot_vertex_vectors(g,min_max_norm(U), min_max_norm(U), Rp, Rn, H, op="emb")

#---------------------------- results
    if file_expected is not None:

        #---- Overlapping communities
        if bool_ov==True and file_expected is not None:
            #must save on lines format
            #must calc oNMI (external code)]

            #print(graph.g.vs['Index'])
            H_geq_0 = np.where(H >0)
            H[H_geq_0] = 1

            print("\n---------\nCommunity detection:")
            base_folder_out = results_folder+""+nome_alg + "" + "/"+ name +"/"
            file_com_out = base_folder_out+"lines_net_" + name + ".com"

            SpecOV.print_com_lines_file(file_com_out, H) #231220

            shutil.copyfile(file_com_out, "f1.txt")
            shutil.copyfile(file_expected, "f2.txt")

            os.system('./mutual3/mutual "f1.txt" "f2.txt"')

            #print(graph.g.vcount())
            #com_expected,lin_expected  = sf.read_partition_expected_OV(file_expected,  graph.g.vcount(), char_split="\t")

            com_expected,lin_expected  = sf.read_partition_expected_OV(file_expected, g, char_split="\t")

            #print(lin_expected)
            #print(com_expected)
            jacc_val = metrics.score_overlapping( H0,com_expected, lin_expected, type_sim="jaccard")
            f1_val = metrics.score_overlapping( H0,com_expected, lin_expected, type_sim="f1score")
            print("jacc_score0",jacc_val)
            print("f1_score0",f1_val)

            jacc_val = metrics.score_overlapping( H,com_expected, lin_expected, type_sim="jaccard")
            f1_val = metrics.score_overlapping( H,com_expected, lin_expected, type_sim="f1score")
            print("jacc_score - Wang",jacc_val)
            print("f1_score -Wang",f1_val)
            mod0=0
            nmi0=0
            nmi=0
            mod=0

            metric_1 = f1_val
            metric_2 = jacc_val
            #print("\n---------\nClassification using ri_vert_pos")
            #print('\n---------\n')

            f1_macro_spec=0
            f1_micro_spec=0

        #---- Disjoint communities
        elif bool_ov==False and file_expected is not None:

            #--- node classification
            #TODO: FUNCTION?
            p_expected = sf.read_partition_expected(file_expected,char_split)
            #k_expected = len(np.unique(p_expected))

            #if p_best is None:
            p_best_0 = sf.create_part_from_Sov(H0)
            p_best = sf.create_part_from_Sov(H_nonzero)

            #TODO reportar NMI e mod de ambos os H

            print("\n---------\nCommunity detection:")
            print("Number of communities - H0", len(pd.unique(p_best_0)))

            mod0 = g.modularity(p_best_0, g.es['weight'])
            print("mod - H0=",mod0)

            nmi0 = sf.calc_nmi(p_best_0, p_expected)
            print("nmi- H0=",nmi0)

            print("Number of communities - H", len(pd.unique(p_best)))
            mod = g.modularity(p_best, g.es['weight'])
            print("mod=",mod)

            nmi = sf.calc_nmi(p_best, p_expected)
            print("nmi=",nmi)

            if op_com > 0:
                print('\n---------\n')
                print("Classification using ri_vert_pos")
                f1_macro_spec, f1_micro_spec = node_classification.node_classification(ri_vert_pos, p_expected)
            else:
                f1_macro_spec=-1
                f1_micro_spec=-1
            print("\nClassification using U")
            f1_macro_U, f1_micro_U = node_classification(U, p_expected)
            print('\n---------\n')

            print("Initial U: Classification using U0")
            f1_macro_U0, f1_micro_U = node_classification(U0, p_expected)
            print('\n---------\n')

            metric_1=f1_macro_U
            metric_2=f1_micro_U

    return H, U, M, S, C, mod0, mod, nmi0, nmi, f1_macro_spec, f1_micro_spec, f1_macro_U, f1_micro_U, time_spec

#%%% Updates
def update_wang_U(S, U, M, C, H, alpha=1):
    #aux_U = ( np.dot(S.transpose(), M) + (alpha * np.dot(H,C) ) )/ np.dot( U , ( np.dot( M.transpose(), M) + (alpha* np.dot( C.transpose(), C) ) )    )
    aux_U = ( np.dot(S.transpose(), M) + (alpha * np.dot(H,C) ) )/ (
            np.maximum( very_low_num,   np.dot( U , ( np.dot( M.transpose(), M) + (alpha* np.dot( C.transpose(), C) ) )    )       )
            )
    U = np.multiply(U, aux_U)

    #280921 TESTE
    #row_sums = U.sum(axis=1)
    #U = U / row_sums[:, np.newaxis]

    return U

def update_wang_M(M, S, U):
    #aux_M = np.dot(S, U) / np.dot( M , np.dot( U.transpose(), U))
    aux_M = np.dot(S, U) / np.maximum( very_low_num, np.dot( M , np.dot( U.transpose(), U))  )
    M = np.multiply(M, aux_M)
    # M = min_max_norm(M)

    #280921 TESTE
    #row_sums = M.sum(axis=1)
    #M = M / row_sums[:, np.newaxis]

    return M

def update_wang_C(C, H, U):
    #TODO: min>0
    #aux_C =  np.dot( H.transpose() , U) / (  np.dot( np.dot(C, U.transpose()) , U  )  )
    aux_C =  np.dot( H.transpose() , U) /  np.maximum( very_low_num, (  np.dot( np.dot(C, U.transpose()) , U  )  ) )
    C = np.multiply( C , aux_C)
    # C = min_max_norm(C)

    #280921 TESTE
   #row_sums = C.sum(axis=1)
    #C = C / row_sums[:, np.newaxis]
    #sys.float_info.min
    return C

def update_H (P, H, A, U, C, alpha, beta, lambdav, op_var, g,  ri_vert_pos, ri_vert_neg, Rp, Rn, p_best):
    print(op_var)
    if 'H' in op_var:
        print('H heuristic')
        Sov=H
        R_group_pos = Rp
        R_group_neg = Rn
        pcurr=p_best
        #H = H_update_heuristic(H, C, rp, Rp )
        Rp, Rn, p_best, H, valQ=H_update_heuristic(g,H, C, ri_vert_pos, ri_vert_neg, Rp, Rn, p_best, U)

    else:
        #TODO!! change H
        print('classical H')

        #*: np.dot
        #.*: np.multiply

        # print("H")1
        # print(H)
        # print("sum")
        # print(np.sum(H, axis=1)[0:3])
        PH = np.dot(P, H)
        HHH = np.dot(H , np.dot( H.transpose() , H ) )
        UC = np.dot(U,C.transpose())
        delta_comp1= np.square(2*beta*PH)
        delta_comp2 = np.multiply(
            16*lambdav*HHH ,
            (
                2*beta*np.dot(A,H)
                + 2*alpha* UC
                + (4*lambdav - 2*alpha)*H  )
            )
        delta = delta_comp1 + delta_comp2
        ##delta[ np.where(delta<0)] = 0 #300921

        aux = ( (-2*beta*PH + np.sqrt(delta)) /  np.maximum( very_low_num, (8*lambdav*HHH)) )
        aux[np.where(aux<0)]=0
        # print(H.shape)
        H = np.multiply(H, np.sqrt(aux) )
        # print(H.shape)
        # print("Hup")

        #print(H)
        # print("sum")
        # print(np.sum(H, axis=1)[0:3])

        #nao está somando 1

        #print(H==H0)
        #280921 TESTE
        #row_sums = H.sum(axis=1)
        #H = H / row_sums[:, np.newaxis]
        #print(H)
        # PH = np.dot(P, H)
        # HHH = np.dot(H , np.dot( H.transpose() , H ) )
        # delta= np.power(2*beta*PH,2) + np.multiply( 16*lambdav*HHH , ( 2*beta*np.dot(A,H) + 2*alpha* np.dot(U,C.transpose()) + (4*lambdav - 2*alpha)*H  ) )
        # H = np.multiply(H, ( np.sqrt( (-2*beta*PH + np.sqrt(delta)) /  np.maximum( very_low_num*100, (8*lambdav*HHH)) ) ) )

    return H,p_best

#%%% Embedding Wang - aux

def min_max_norm(x):
    return ( x - np.min(x) ) / (np.max(x) - np.min(x))
    #TEst
    #return x

#%%% Not used H heuristic

def calc_H_SpecNMF(R_group_pos, R_group_neg, divtotal,Sov, U,C, lambdav, alpha, beta):
  valQ=0
  # print("num_groups="+str(np.shape(R_group_pos)[0]))
  for s in range(0,np.shape(R_group_pos)[0]):
    valQ= valQ + np.inner( R_group_pos[s,:], R_group_pos[s,:] ) - np.inner( R_group_neg[s,:], R_group_neg[s,:] )

  valQ = valQ/divtotal

  H_UC = Sov -  np.dot(U, C.transpose() )
  #val_UC = alpha*np.power(linalg.norm(H_UC),2)
  val_UC = alpha*np.trace( np.dot( H_UC, H_UC.transpose()   ))

  valQ -= val_UC

  return valQ, val_UC

def calc_gain_H_SpecNMF(valHUC_old, R_group_pos, R_group_neg, t_new, t_old, ri_vert_pos, ri_vert_neg, v, Sov, U,C, lambdav, alpha, beta):
    val_H = np.inner( R_group_pos[t_new,:], ri_vert_pos[v,:] ) - np.inner( R_group_neg[t_new,:], ri_vert_neg[v,:] )
    #val -= np.inner( R_group_pos[t_old,:], ri_vert_pos[v,:] ) - np.inner( R_group_neg[t_old,:], ri_vert_neg[v,:] )     #wrong
    #251120
    val_H -= np.inner( R_group_pos[t_old,:]-ri_vert_pos[v,], ri_vert_pos[v,:] ) - np.inner( R_group_neg[t_old,:]-ri_vert_neg[v,], ri_vert_neg[v,:] )
    #print("new calc update")
    #???  val += np.inner(ri_vert_pos[v,:],ri_vert_pos[v,:]) -  np.inner(ri_vert_neg[v,:],ri_vert_neg[v,:])

    val_H *= beta

    #TODO: precisa simular o movimento aqui

    #TODO: testa e volta

    Sov[v,t_old]=0
    Sov[v,t_new]=1
    H_UC = Sov -  np.dot(U, C.transpose() )
    Sov[v,t_old]=1
    Sov[v,t_new]=0

    val_UC =alpha* np.trace( np.dot( H_UC, H_UC.transpose()   ))
    #np.trace( np.dot(Sov, Sov.transpose()) - 2* np.dot( np.dot(Sov, C), U.transpose()) + np.dot( np.dot( np.dot(U, C.transpose()) , C), U.transpose()) )
    #val_UC = alpha*np.power(linalg.norm(H_UC),2)

    val = val_H - (val_UC - valHUC_old)
    valHUC_delta = (val_UC - valHUC_old)
    return val,valHUC_delta

def H_update_heuristic(g,Sov, C, ri_vert_pos, ri_vert_neg, R_group_pos, R_group_neg, pcurr, U):

    #This is going to be part of the genetic algorithm..
    #def spec_local_search(g, ri_vert_pos, ri_vert_neg, eig_pos_ind, eig_neg_ind, comms_est, R_group_pos, R_group_neg, Sov, pcurr, it_max=10,bool_verbose=False):
    it_ls=1
    it_max=10
    bool_improve=True

    Q_ini, valHUC_old = calc_H_SpecNMF(R_group_pos, R_group_neg,  2*g.ecount(),Sov, U,C, lambdav, alpha, beta)
    print("Q_ini=",Q_ini)
    while bool_improve==True and it_ls < it_max:
      bool_improve=False

      allv = np.arange(0,g.vcount(),1)
      allv_sample = np.random.choice(allv, len(allv), replace=False )

      for v in allv_sample:
        #print("** Try to move vertex " + str(v))
        val_best = -100
        t_best = -1
        t_old = pcurr[v] #current community of v

        #find the best candidate community t_best to insert v into
        comms_cand = [c for c in np.unique(pcurr) if c>=0 and c!=t_old ]
        # print("candidates")
        # print(comms_cand)

        for t in comms_cand:
          # print("t_old"+str(t_old))
          # print("t="+str(t))
          # print("v="+str(v))

          #val = calc_gainQ_Rgroup_update(R_group_pos, R_group_neg, t, t_old, ri_vert_pos, ri_vert_neg, v)
          val,valHUC_delta = calc_gain_H_SpecNMF(valHUC_old,R_group_pos, R_group_neg, t, t_old, ri_vert_pos, ri_vert_neg, v, Sov, U,C, lambdav, alpha, beta)  #TODO
          #print(valHUC_delta)
          #print(valHUC_delta>0)
          #print(valHUC_delta > 0 (val>val_best or t_best==-1))
          #if val>0 and (val>val_best or t_best==-1):
          if valHUC_delta > 0 and (val>val_best or t_best==-1):
            #print(valHUC_delta)
            #print("*** found new best, Check " + str(v) + " to community " + str(t) + ", val=" + str(val), ", val_best" + str(val_best) + ", t_best=", t_best)
            val_best = val
            t_best = t

        if t_best>-1 and t_best!= t_old: #just checking again for t_best!=t_old
          #move vertex v to community t_best
          #if bool_verbose:
          ##print("Move " + str(v) + " to community " + str(t_best))

          pcurr[v] = t_best
          Sov[v,t_old]=0
          Sov[v,t_best]=1

          R_group_pos, R_group_neg = sf.move_vertex_R_group(R_group_pos, R_group_neg, t_best, t_old, ri_vert_pos, ri_vert_neg, v)
          bool_improve = True

        #all valid communities, disconsidering old_o
      it_ls+=1

    #print("it ls=",it_ls)
    #do not need
    #valQ = sf.calc_Q_Rgroup(R_group_pos, R_group_neg, 2*g.ecount())
    #valQ = calc_Q_H_SpecNMF(R_group_pos, R_group_neg, 2*g.ecount()) #TODO
    valQ = calc_H_SpecNMF(R_group_pos, R_group_neg,  2*g.ecount(),Sov, U,C, lambdav, alpha, beta)

    if bool_verbose==True:
      #print(".. LS, IT = "+str(it_ls))
      conf_Q_vec = g.modularity(pcurr, g.es['weight'])
      print("..Q-LS:"+ str(valQ)  + ", Q_igraph-LS="+str(conf_Q_vec))

    #%TODO: atualiza U?
    return R_group_pos, R_group_neg, pcurr, Sov, valQ

#pcurr, Sov, valQ = local_search(g, ri_vert_pos, ri_vert_neg, eig_pos_ind, eig_neg_ind, comms_est, R_group_pos, R_group_neg, Sov, pcurr)
#print(pcurr)
#print(valQ)

#%% Wang - aux

#%%% U

def solve_wang_U(g,p_expected,dim=2,gamma1=1,par_eta=5):

    adj = g.get_adjacency(attribute="weight")
    #S1 = gamma1*np.matrix(adj.data)
    S1 = gamma1*np.array(adj.data)
    S2 = np.zeros((g.vcount(), g.vcount()))

    print("Start cosine similarity")
    S2 = cosine_similarity(S1)
    print("End cosine similarity")

    #S = S1 + par_eta * S2
    S = S1 +par_eta*S2

    print("Run NMF")
    #||S-MU^T||^2

    #011021
    model = NMF(n_components=dim, init='nndsvdar') #, init='random') #b, init='random' random_state=0,
    M = model.fit_transform(S) #M
    U = np.transpose(model.components_) #U

    #f1_macro, f1_micro= node_classification(U0, p_expected, test_size=0.3)
    #U=U0
    #x_train = S

    #U1, _ = autoencoder_keras(x_train,x_train, dim)
    #f1_macro, f1_micro= node_classification(U1, p_expected, test_size=0.3)

    # M = ( np.random.rand(g.vcount(),dim) )
    # M = update_wang_M(M, S, U)

    # from karateclub import DANMF
    # G = nx.read_pajek(file_graph)
    # G = G.to_undirected()
    # zip_mapping = zip(G.nodes(), np.arange(0, len(G.nodes())) )
    # mapping = dict(zip_mapping)
    # G=nx.relabel_nodes(G, mapping)

    # model = DANMF(layers= [144, 99],)
    # model.fit(G)

    # embedding = model.get_embedding()
    #f1_macro, f1_micro= node_classification(decoded_img, p_expected, test_size=0.3)

    #x_train = np.zeros((g.vcount(), g.vcount(),1 ))

    #for i in np.arange(g.vcount()):
        #x_train[i][:,:] = S[i]
    #x_train[0,1,1] = S
    # x_train = S
    # A_enc, A_dec = autoencoder_keras(x_train, x_train)
    # U = A_dec[0]
    # print("********** AUTOENCODER")

    # f1_macro, f1_micro= node_classification(U, p_expected, test_size=0.3)
    # f1_macro, f1_micro= node_classification(U0, p_expected, test_size=0.3)

    #U = min_max_norm(U)
    #M = min_max_norm(M)
    #S = min_max_norm(S)

    #280921 TESTE
    #row_sums = M.sum(axis=1)
    #M = M / row_sums[:, np.newaxis]

    #280921 TESTE
    #row_sums = U.sum(axis=1)
    #U = U / row_sums[:, np.newaxis]

    #M = ( np.random.rand(g.vcount(),dim_p) )

    #------------
    #https://www.analyticsvidhya.com/blog/2021/06/complete-guide-on-how-to-use-autoencoders-in-python/

    print(S.shape)
    print(U.shape)

    return U, M, S

#%%% Score overlapping

def normalize(mat):
    #return mat/np.sum(mat)
    return ( mat - np.min(mat) )/ (  np.max(mat) - np.min(mat))
def normalize2(mat):
    return mat


import os

def _all_python_files():
    root_dir = os.path.dirname(__file__)
    names = []
    for filename in os.listdir(root_dir):
        if filename.endswith('.py') and filename != '__init__.py':
            name = filename[:-3]
            names.append(name)
    return names

for name in _all_python_files():
    try:
        __import__(name, globals(), locals(), level=1)
    except ModuleNotFoundError:  #pragma: no cover
        print("estimator {} not installed".format(name))