#!/usr/bin/env python3
'''
**********************************************************************
* Filename      : relay1.py
* Purpose       : To control mains power supply to dumb Dehumidfier
* Created       : Dec 2019 fully reworked in Jan 2021
* Author        : Derek
**********************************************************************
* 16 Nov 2020 - make config data reload if fileexternally changed.
*
**********************************************************************
Updates:
20 Jan 2021 - DHTsensor now powered from a GPIO pin. Hence new routine to
              reaset it. Previously done using spare relay in the
              2 channel module on a direct power line. RElay2 left in code
              but not used. Status set that on board LED is not lit.
22 Jan 2021 - Reworked to add LCD output dispaly.

24 Jan 2021 - Add network tests for local router and Internet access
'''
# needed to access my other Pyton std. modules
import sys, os
sys.path.append(os.path.abspath("/home/pi/dev/gpio"))

import RPi.GPIO as GPIO
from time import sleep
from  RPLCD_class import Mylcd

import sensors   as sensor
import mytimer   as T
import FileClass as FU
import os
import DU
import config   as CFG

class Params(object):
    def __init__(self, File):
        self.InputFile = File
        self.LoadParams(self.InputFile)
        self.wrte2logfile()

    def LoadParams(self,configfile):
        oCfg     = CFG.ReadConfig(configfile)

        self.TestTime      = int(oCfg.GetConfigValue('TESTTIME'))
        self.PollTime      = int(oCfg.GetConfigValue('POLLTIME'))
        self.RestTime      = int(oCfg.GetConfigValue('RESTTIME'))
        self.MaxConRunTime = int(oCfg.GetConfigValue('MAXCONRUNTIME'))
        self.IntNetwork    =     oCfg.GetConfigValue('INTNETWORK')
        self.ExtNetwork    =     oCfg.GetConfigValue('EXTNETWORK')

        self.OffThres = float(oCfg.GetConfigValue('OFFTHRES')) # % RH - Threshold to turn OFF power
        self.OnThres  = float(oCfg.GetConfigValue('ONTHRES'))  # % RH - Threshold to turn ON  power

    def wrte2logfile(self):
        ### write basic data to log file
        logger.write("\nBasic Control Data")
        logger.write("Int Network       : "+self.IntNetwork)
        logger.write("Ext Network       : "+self.ExtNetwork+"\n")

        logger.write("Test Time         : "+str(self.TestTime)+" secs")
        logger.write("Poll interval     : "+str(self.PollTime)+" secs")
        logger.write("Max con run time  : "+str(self.MaxConRunTime)+" secs")
        logger.write("Rest Time         : "+str(self.RestTime)+" secs")
        logger.write("")
        logger.write("OFF threshold     : "+str(self.OffThres)+" %RH")
        logger.write("ON  threshold     : "+str(self.OnThres) +" %RH")

    def ReLoadParams(self):
        self.LoadParams(self.InputFile)
        self.wrte2logfile()
# set Pi pin config to "Board" NOT Physical pin numbers
GPIO.setmode(GPIO.BCM)
GPIO.setwarnings(False)

##### Object Defintions
logger  = DU.c_logger("/home/pi/dev/logs","log1.txt")
logger2 = DU.c_logger("/home/pi/dev/logs","log2.txt")

lcd     = Mylcd(i2c_expander='PCF8574', address=0x27, cols=16, rows=2)

# Realy #1 switched mains voltage !| grep re

Relay1_pin = 17
oRL1 = sensor.Relay(Relay1_pin)

## Relay #2 NOT used as at 20 Jan 2021
#but still wired up.
Relay2_pin = 18
oRL2 = sensor.Relay(Relay2_pin)
oRL2.switchOFF() # needed to define state, hence switch relay #2 LCD OFF

DHT_SENSOR  = 22
DHT_PIN     = 4
DHT_PWR_PIN = 27

oHT = sensor.Humidity(DHT_SENSOR,DHT_PIN, DHT_PWR_PIN)

oT  = T.timer()

oP = Params("config.txt")     # run time parameters

oF = FU.FileMod(oP.InputFile) # check if file updated while prog is Live
#####

def main():
    print("\nStarting RELAY1.py ...........\n")
    # check and Display network status on LCD
    oTest = DU.Comms(logger)
    rc = oTest.checknetwork(oP.IntNetwork ,4)
    lcd.write2pos("Int. Net= "+str(rc),1,1)

    rc = oTest.checknetwork(oP.ExtNetwork ,4)
    lcd.write2pos("Ext. Net= "+str(rc),2,1)
    sleep(3)

    ### COUNTERS
    OFFcount = 0 # set in the intialisaton process
    ONcount  = 0 # ditto
    PollCount= 0 #
    errcnt   = 0 # count of errors reading the DHT22 sensor

    ### set initial state
    logger.write("***** INITIALISATION *****")
    logger.write("Start Up  SWITCHing  ON  for "+str(oP.TestTime)+" secs"+"\n\t\t\t\t\t  to Test Humidfier fires up!")

    oRL1.switchON()
    sleep(oP.TestTime)

    oRL1.switchOFF()
    ON_FLAG = False
    logger.write("STARTUP   State = SWITCHed  OFF")
    oT.resetstarttime()

    ##### Start Infinte Polling Loop
    lcd.clear()
    while True:

        # Check if config.txt has been updated while program is running?
        if oF.Changed():
            logger.write("")
            logger.write("*** START  - Run Data ReLoaded ***")
            oP.ReLoadParams()
            logger.write("*** FINISH - Run Data ReLoaded ***")
            logger.write("")
        else:
            pass

        # READ DHT22 sensor values
        hvalue, tvalue =  oHT.read_dht()
        etime = oT.secs2dhms(oT.elapsedtime())
        logger.write ("Humidity : "+str(hvalue)+", Temp: "+str(tvalue)+" Elapased= "+str(etime) )
        logger2.write("Humidity : "+str(hvalue)+", Temp: "+str(tvalue)+" Elapased= "+str(etime) )

        lcd.write2pos("RH%={0:0.1f}".format(hvalue),1,1)
        lcd.write2pos(" "+chr(0)+"C={0:0.1f}".format(tvalue), 1,9)

        lcd.write2pos("{0:0.1f}{1}{2}{3:0.1f}".format(oP.OffThres, chr(3), chr(126), oP.OnThres), 2,1)
        PollCount += 1
        logger.write("")
        logger.write("*** Poll # = "+str(PollCount)+"  Time frm Start : "+str(oT.secs2dhms(int(oT.timefromprogramstart()) )))

        if(hvalue == 999):
            errcnt +=1
            if(errcnt >1000):
                logger.write("ERROR: Maxiumm sensor read Errors/Resets exceeded - program halted")
                destroy()
                break
            else: # take corrective action and rseet DHT22 by repowering it
                logger.write("ERROR: Reading DHT22 values -- Power OFF DHT-22, count: "+str(errcnt))
                oHT.reset()
                logger.write("ERROR: Reading DHT22 values -- Power ON  DHT-22")

        if  hvalue > oP.OnThres:
            if not ON_FLAG: # 1st time in this section of code
                ON_FLAG = True
                ONcount += 1
                logger.write("Zone_ON   SWITCHing  ON, count= "+str(ONcount))
                oT.resetstarttime()
                oRL1.switchON()
                lcd.write2pos("ON  ",2,13)
            else:          # 2nd or subsequent time in this code section
                logger.write("Zone_ON   State = SWITCHed  ON")
                CheckMaxRunTime(oP.MaxConRunTime, oP.RestTime, oP.PollTime,  ON_FLAG,ONcount,OFFcount)

        elif hvalue <= oP.OffThres:
            if ON_FLAG : # ie 1st time in this code section
                ON_FLAG = False
                OFFcount += 1
                logger.write("Zone_OFF   SWITCHing OFF,count= "+str(OFFcount))
                oT.calccumruntime()
                oRL1.switchOFF()
                lcd.write2pos("OFF ",2,13)

            else:
                logger.write("Zone_OFF   State = SWITCHed OFF")
                lcd.write2pos("OFF ",2,13)

        else: # Between the Max and Min thresholds
            if ON_FLAG:
                logger.write("Zone_ONorOFF   State = SWITCHed ON")
                CheckMaxRunTime(oP.MaxConRunTime, oP.RestTime, oP.PollTime, ON_FLAG, ONcount,OFFcount )
            else:
                logger.write("Zone_ONorOFF   State = SWITCHed OFF")

        sleep(oP.PollTime)
        logger.write("*** END OF POLLING LOOP")

    ##### End of Polling Loop

def CheckMaxRunTime(MaxConRunTime, RestTime, PollTime, ON_FLAG,ONcount,OFFcount):
        # Purpose: Check that maximum continuous run time has not been exceeded and if it has
        #          been then invoke rest period.
        if  oT.elapsedtime() > (MaxConRunTime):

            logger.write("   MAX Con. running time exceeded. Value    : "+str(oT.secs2dhms(oT.elapsedtime())) )
            logger.write("   NOW Invoking rest time of                : "+str(oT.secs2dhms(RestTime)))

            OFFcount += 1
            logger.write("   SWITCHing  OFF, count: "+str(OFFcount))
            oRL1.switchOFF()

            oT.calccumruntime()
            ttot = oT.secs2dhms(oT.getcumruntime() )
            logger.write("   Cum. Continuous running time to date is  : "+str(ttot) )

            lcd.write2pos("REST",2,13)
            sleep(RestTime)
            oT.resetstarttime()
            ONcount +=1
            logger.write("   SWITCHing  ON. count: "+str(OFFcount))

            oRL1.switchON()
            lcd.write2pos("ON  ",2,13)
        else:
            logger.write("   Max run time NOT exceeded.  On count: "+str(ONcount)+", OFF count: "+str(OFFcount))

def destroy():
    oRL1.switchOFF()
    lcd.cleanup()
    GPIO.cleanup()
    print("\nTime since program start: "+str(oT.secs2dhms(int(oT.timefromprogramstart()) )))
    print(  "Total ON time           : "+str(oT.secs2dhms(oT.getcumruntime()) ))
    print("\nAll done in relay\n")

if __name__ == '__main__':

    try:
        main()
    except KeyboardInterrupt:
        destroy()


import torch
import torch.nn as nn

class ConvolutionBlock(nn.Module):

    def __init__(self, in_channels, out_channels): # h, w -> h/2, w/2 | # of kernel -> 2 * # of kernel
        self.in_channels = in_channels
        self.out_channels = out_channels

        super().__init__()

        self.layers = nn.Sequential(
            nn.Conv2d(in_channels, out_channels, (3, 3), padding=1),
            nn.ReLU(),
            nn.BatchNorm2d(out_channels), # 출력의 크기보다는 output chanel의 숫자를 넣어줘야 함
            nn.Conv2d(out_channels, out_channels, (3, 3), stride=2, padding=1),
            nn.ReLU(),
            nn.BatchNorm2d(out_channels),
        )

    def forward(self, x):
        # |x| = (batch_size, in_channels, h, w)

        y = self.layers(x)
        # |y| = (batch_size, out_channels, h, w)

        return y

class ConvolutionalClassifier(nn.Module):

    def __init__(self, output_size):
        self.output_size = output_size

        super().__init__()

        self.blocks = nn.Sequential( # |x| = (n, 1, 28, 28)
            ConvolutionBlock(1, 32), # (n, 32, 14, 14)
            ConvolutionBlock(32, 64), # (n, 64, 7, 7)
            ConvolutionBlock(64, 128), # (n, 128, 4, 4)
            ConvolutionBlock(128, 256), # (n, 256, 2, 2)
            ConvolutionBlock(256, 512), # (n, 512, 1, 1)
        )

        self.layers = nn.Sequential(
            nn.Linear(512, 50),
            nn.ReLU(),
            nn.BatchNorm1d(50),
            nn.Linear(50, output_size),
            nn.LogSoftmax(dim=-1),
        )

    def forward(self, x):
        assert x.dim() > 2 # flatten을 실수로 하지 않을 경우를 대비해서 assert를 걸어 놓음 fc 인 경우, (bs, 784) dim = 2

        if x.dim() == 3: # grayscale인 경우 - tensor 모양 변경
            # |x| = (batch_size, h, w)
            x = x.view(-1, 1, x.size(-2), x.size(-1))
            # |x| = (batch_size, 1, h, w)

        z = self.blocks(x)
        # |z| = (batch_size, 512, 1, 1)

        y = self.layers(z.squeeze())
        # |y| = (batch_size, output_size)

        return y


# 208. 实现 Trie (前缀树)
# Trie（发音类似 "try"）或者说 前缀树 是一种树形数据结构，用于高效地存储和检索字符串数据集中的键。这一数据结构有相当多的应用情景，例如自动补完和拼写检查。
# 请你实现 Trie 类：
# Trie() 初始化前缀树对象。
# void insert(String word) 向前缀树中插入字符串 word 。
# boolean search(String word) 如果字符串 word 在前缀树中，返回 true（即，在检索之前已经插入）；否则，返回 false 。
# boolean startsWith(String prefix) 如果之前已经插入的字符串word 的前缀之一为 prefix ，返回 true ；否则，返回 false 。
class Trie:
    class Node:
        def __init__(self, value='', isKey=False):
            self.value = value
            self.isKey = isKey
            self.map = {}
            self.numWords = 0

    def __init__(self):
        """
        Initialize your data structure here.
        """
        self.root = self.Node()
        self.size = 0

    def __len__(self):
        return self.getSize()

    def __subStartWith(self, x: Node, word: str, depth: int):
        if not x:
            return None
        if depth == len(word):
            return x
        char = word[depth]
        return self.__subStartWith(x.map.get(char), word, depth + 1)

    def __get(self, x: Node, word: str, depth: int):
        if not x:
            return None
        if depth == len(word):
            if x.isKey:
                return x
            else:
                return None
        char = word[depth]
        return self.__get(x.map.get(char), word, depth + 1)

    def __subTraverse(self, x: Node, result: str, keys: list):
        for char in x.map.keys():
            self.__subTraverse(x.map.get(char), result + char, keys)
        if x.isKey:
            keys.append(result)

    def insert(self, word: str) -> None:
        """
        Inserts a word into the trie.
        """
        current = self.root
        for char in word:
            current.numWords += 1
            if not current.map.get(char):
                current.map[char] = self.Node(char, False)
            current = current.map.get(char)
        current.isKey = True
        self.size += 1

    def search(self, word: str) -> bool:
        """
        Returns if the word is in the trie.
        """
        return bool(self.__get(self.root, word, 0))

    def startsWith(self, prefix: str) -> bool:
        """
        Returns if there is any word in the trie that starts with the given prefix.
        """
        x = self.__subStartWith(self.root, prefix, 0)
        return bool(x)

    def getSize(self):
        return self.size

    def clear(self):
        self.root = self.Node()
        self.size = 0

    def keysWithPerfix(self, perfix: str):
        x = self.__subStartWith(self.root, perfix, 0)
        if x:
            keys = []
            self.__subTraverse(x, perfix, keys)
            return keys
        return False

trie = Trie()
trie.insert('apple')
trie.search('apple')
trie.search('app')
trie.startsWith('app')
trie.insert('app')
trie.search('app')


import pandas as pd
import sys as sys
import os as os

def extract_rows(data, ant_names):
    '''A function to extract relevant data from dataframe'''
    new_data = data.loc[ant_names]
    return(new_data)

def get_names(namefile):
   '''Read in a file of names'''
   with open(namefile, 'r') as f:
       ant_names = f.read().splitlines()
   return(ant_names)

def check_dir(path):
	check=path.split('/')[0]
	os.listdir(check)
	if 'output' in os.listdir(check):
		return
	else:
		os.mkdir('%s/output' % check)

if __name__ == '__main__':
	data = pd.read_csv(sys.argv[1], index_col=0)
	''' ../../Data/Ants.csv '''
	namefile = sys.argv[2]
	''' '../../Data/AntTestData.txt' '''
	outputPath = check_dir(sys.argv[3])
	ant_names = get_names(namefile)
	new_dataframe = extract_rows(data, ant_names)
	new_dataframe.to_csv(sys.argv[3])
	''' ../../Data/processed_ants.csv '''


import os, uuid
from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient, ContentSettings

def upload(local_file_name: str):
    """
    Upload to Azure using FTP
    https://github.com/Azure/azure-sdk-for-python/blob/master/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py#L375
    """
    az_key: str = os.getenv("AZ_STORAGE_KEY")
    az_string: str = os.getenv("AZ_STORAGE_CONNECTION_STRING")
    container_name: str = '$web'

    try:
        blob_service_client: BlobServiceClient = BlobServiceClient.from_connection_string(az_string)
        blob_client: BlobClient = blob_service_client.get_blob_client(container=container_name, blob=local_file_name)

        print("\nUploading to Azure Storage as blob:\n\t" + local_file_name)
        my_content_settings: ContentSettings = ContentSettings(content_type='text/html')

        with open('populated.html', "rb") as data:
            blob_client.upload_blob(data, overwrite=True, content_settings=my_content_settings)

    except Exception as ex:
        print('Exception:')
        print(ex)


from PyQt5.QtWidgets import *
from PyQt5.QtCore import *
from PyQt5.QtGui import *

import os

def chooseFile(mainwin:QWidget,title:str,fileTypes:list,allFiles:bool=False,cwd:str=os.getcwd())->tuple:
    '''
    弹出Qt文件选择对话框
    :param mainwin: 主窗口
    :param title: 窗口标题
    :param fileTypes: 可选择类型
    :param allFiles: 是否显示“所有文件”选项
    :param cwd: 起始位置
    :return: (
        选择成功，取消为False,
        文件路径
        文件类型
    )
    '''
    typeStr = ""
    for t in fileTypes[:-1]:
        typeStr += t + ';;'
    typeStr += fileTypes[-1]
    if allFiles:
        typeStr += ";;All Files (*.*)"
    fileName_choose, filetype = QFileDialog.getOpenFileName(mainwin,
                                                            title,
                                                            cwd,  # 起始路径
                                                            typeStr)  # 设置文件扩展名过滤,用双分号间隔

    if fileName_choose == "":
        return (False,"","")

    return (True,fileName_choose,filetype)

def warningBox(win,title,text):
    QMessageBox.warning(win,title,text)


from django.shortcuts import render, redirect
from django.http import HttpResponse, JsonResponse
from django.template import loader, context
from django.contrib.auth.decorators import login_required
from django.contrib.auth import authenticate, login, logout
# http://code.djangoproject.com/browser/django/trunk/django/contrib/admin/views/decorators.py
#from django.contrib.admin.views.decorators import staff_member_required
from django.utils import timezone
from django.core.exceptions import PermissionDenied
from django.views.static import serve
# password reset things
from django.contrib import messages
from django.contrib.auth import update_session_auth_hash
from django.contrib.auth.forms import PasswordChangeForm
# non-django related things:
from threading import Thread
from . import urls
import os, time, datetime, json, random, string, shutil
#pdf2image is used to convert PDFs to images for the video
from pdf2image import convert_from_path
from PIL import Image # PIL for saving the picture
import tempfile
from pdf2image.exceptions import (
    PDFInfoNotInstalledError,
    PDFPageCountError,
    PDFSyntaxError
)
# ToDo: form with delete button externally, to accomplish the same thing as delete-apply
from zipfile import ZipFile

try:
    import subprocess
except:
    print("\'Subprocess\'~$ package isn't installed [ https://pymotw.com/2/subprocess/ ].\n\t please install: pip(3) install subprocess\n")

try:
    import schedule
except Exception as e:
    print("\'Scheduler\' package isn't installed [ https://github.com/dbader/schedule ].\n\t please install: ~$ pip(3) install scheduler")

# Database implementation:
# https://docs.djangoproject.com/en/2.2/intro/tutorial02/
from manager.models import Content, Show, Screen

# video lib:
from . import VideoMaker as VM

#_______________________________________________________________________________
# global variables for things:

Video_Compiling = False

#_______________________________________________________________________________
# primary interface stuffs:
# password reset form
@login_required
def passwd_change(request):
    # code from: https://simpleisbetterthancomplex.com/tips/2016/08/04/django-tip-9-password-change-form.html
    if request.method == 'POST':
        form = PasswordChangeForm(request.user, request.POST)
        if form.is_valid():
            user = form.save()
            update_session_auth_hash(request, user)  # Important!
            messages.success(request, 'Your password was successfully updated!')
            return redirect('password_change')
        else:
            messages.error(request, 'Please correct the error below.')
    else:
        form = PasswordChangeForm(request.user)

    return render(request, 'change_password.html', {'form': form})

#@login_required
###@staff_member_required
def index(request):
    resp = HttpResponse("
Error 909: Development Whilst Drunk Occured

apologies for any inconveniece

")
    return resp, redirect('/')

# home / main data interface
@login_required
###@staff_member_required
def home(request):
    #print("Home", str(request.META['REMOTE_ADDR']))
    # content is passed into render, and can be parsed by the template

    content = getContent(request)

    return render(request, 'index.html', content)

# documentation senction, over by the admin page link
def documentation(request):
    resp = HttpResponse("documentation here...")

    return render(request, 'documentation.html')
#_______________________________________________________________________________
# File dealings:
@login_required
###@staff_member_required

def upload_file(request):
    if request.method == 'POST':
        try: # to mitigate the no file found error
            print("file upload requested") # indform the log...
            g = request.FILES['fileupload'] # get the file the user sent
            print("Length of files: ", len(g)) # see if there is data
            handle_file(fdup_file = g) # do things with the file
        except Exception as e: # don't do a damn thing
            print("No file selected, not trying to handle it.")
            print(e)
    return redirect('.') # redirect to the top of the page (_self)

# file handling functionality:
def handle_file(fdup_file):
    # method to recieve and deal with files:
    # > recieve and store
    # > build database entry
    print(fdup_file)
    filename = str(fdup_file).replace(' ','_') # replace whitespace with underscores

    if ".zip" in str(fdup_file):
        print("We got a biggn, needs to be unzipped and analized")
        handle_zipped_file(filename, fdup_file)

    else: # not a ZIP file, so behave in a one-off manner:
        make_db_entry(filename, fdup_file)

def handle_zipped_file(filename, fdup_file):
    print("handling zipped file")
    # store the zip temporaraly
    temp_file = "./manager/static/content/%s"%(filename)
    temp_dir = "./manager/static/content/%s"%(filename).replace(".zip", "").replace("_", ' ')
    # rebuild the file from the chunks
    print(fdup_file)
    rebuild_fdup_data(loc=temp_file, chunky_file=fdup_file) # save the data to the system
    # extract the jeezless thing:
    with ZipFile(temp_file, 'r') as zipObj:
        zipObj.extractall(temp_dir) # extract the objects

    # see what we have:
    extracted = os.listdir(temp_dir)
    print('Extracted:', extracted)
    #chunksize = 256 # bytes per chunk
    for file in extracted:
        print('loop', temp_dir, file)
        pth = "%s/%s"%(temp_dir, file)
        print(pth)

        make_db_entry(file, single_file = False, temp_path = pth)

    # cleanup the directory
    os.remove(temp_file)
    shutil.rmtree(temp_dir)

def make_db_entry(filename, fdup_file = None, single_file = True, temp_path = None):
        # now build the entry:
        item = Content(
            imageName = "ThisWillBeFilled", # the 'c' tag needs to reflect doc type
            uploadDate = timezone.now(),
            startDate = timezone.now(),
            expireDate = timezone.now() + timezone.timedelta(days=7), # today + 7 days
            displayTime = 5, # seconds
            order = 0
            )
        item.save() # save the item to get an ID,
        dire = item.id # use the ID to make a directory
        print("Item number:", dire) # debug
        item.file = "./manager/static/content/c%i/%s"%(dire, filename) # update the file location
        item.save() # resave

        # make a directory:
        success = False
        counts = 0 # loop counts
        while (not success) and (counts < 10):
            # make the directory in static:
            try:
                os.mkdir("./manager/static/content/c%i/"%(dire))
                success = True # say something if directory was made
            except FileExistsError:
                print("#%i -- directory: \"./manager/static/content/%i/\" exists, Removing"%(counts,dire))
                os.remove("./manager/static/content/c%i/"%(dire))
                counts += 1

        # build file location
        loc = "./manager/static/content/c%s/%s"%(dire, filename)

        if single_file: # if it is a single file from the web interface do this:
            rebuild_fdup_data(loc, fdup_file)
        else:  # used for zip files, because the data is already reconstructed:
            #copy_data_from_temp_path()
            shutil.copy(temp_path, loc)

        # find how the video will be previewed:
        if (".mp4") in filename.lower():
            print("I found an MP4, getting a thumbnail...")

            thumbnail = get_mp4_thumbnail(file=item.file)
            print("I gots the thumbnail: ", thumbnail)
            item.imageName = "./manager/static/content/c%s/%s"%(dire, thumbnail)
            item.file = loc
            item.save()
            print("thumbnail saved")

        elif ".pdf" in filename.lower():
            print("Found a PDF, getting a JPG")
            try:
                jpg_path = "./manager/static/content/c%s/%s"%(dire, filename.replace('.pdf', '.jpg'))
            except:
                jpg_path = "./manager/static/content/c%s/%s"%(dire, filename.replace('.PDF', '.jpg'))

            image = convert_from_path(pdf_path=loc, dpi=200, thread_count=2, fmt='jpg', single_file=True)
            print(image[0])
            image[0].save(fp=jpg_path)

            # save the stuff to database
            item.imageName = jpg_path
            item.file = jpg_path
            item.save()

            # now delete the PDF to save space
            os.remove(loc)

        elif ".png" or ".jpg" or ".gif" in filename.lower():
            print("This must be an image...")
            item.imageName = loc
            item.file = loc
            item.save()
            print("Image saved")

def rebuild_fdup_data(loc, chunky_file):
    # rebuild data as the user said:
    with open(loc, 'wb+') as dest:
        for chunk in chunky_file.chunks():
            dest.write(chunk)
        dest.close()

#_______________________________________________________________________________
# Screen settings

# TV token authentication decoration:
def token_authorization(func):
    def function_wrapper(req): # wrapper for the request stuffs (I think)
        try:
            key = req.GET.getlist('auth_token')[0] # get the key
        except:
            key = "None Found"
        print("auth_token =", key)

        appscreen = Screen.objects.filter(key=key) # filtered screen list
        key_check = ''
        for s in appscreen:
            key_check = s.key
            print("Key check:", key_check)

        if key_check == key:
            return func(req)
        else:
            raise PermissionDenied
            print(datetime.datetime.now(), "Unauthroizes access prevented from:", req.get_host())

    return function_wrapper

@login_required
##@staff_member_required
def screens(request):
    print("SCREENS!")

    content = {
        'screens': [],
        'msg':'HI!'
    }

    # get the screens:
    s = Screen.objects.all().order_by('id')

    for item in s:
        content['screens'].append({
            'id' : item.id,
            'name' : item.tvName,
            'addr' : item.hostIP,
            'width': item.width,
            'height': item.height,
            'useSched':item.useSchedule,
            'starttime':item.startTime.strftime("%H:%M"),
            'endtime': item.endTime.strftime("%H:%M"),
            'key' : item.key
        })

    return render(request, 'screens.html', content)

def screenSettings(request):
    # used to update screen settings
    print("screen settings")

def screen_active_hours(request):
    print("active hours")

@login_required
##@staff_member_required
def newScreen(request):
    # save a new screen
    screen = request.POST
    name = screen.getlist('name')[0]
    addr = screen.getlist('addr')[0]
    width = screen.getlist('width')[0]
    height = screen.getlist('height')[0]

    #print("Recieved:", name, addr, width, height)

    new = Screen(tvName = name, hostIP = addr, width=width, height=height, key=make_tv_key())
    new.save()

    return redirect('/screens')

@login_required
##@staff_member_required
def deleteScreen(request):
    # Delete a screen db entry
    screenid = request.POST.getlist("Delete")[0]

    rem = Screen.objects.get(id__exact=screenid)
    rem.delete()
    print("delete:", screenid)

    return redirect("/screens")

def make_tv_key():
    len = 48 # length of the key
    ky = ''.join(random.choices(string.ascii_letters + string.digits, k=len))

    return ky

# public facing content delivery methods, using token auth.
@token_authorization
def get_tv_video(request):
    # counterpart for the get_video method in simpleviewer

    print("transmitting video file")
    fileToTXDir = os.path.dirname(Show.objects.all()[0].file)
    fileToTXName = os.path.basename(Show.objects.all()[0].file)
    # from: https://stackoverflow.com/questions/1156246/having-django-serve-downloadable-files
    file_to_Serve = serve(request, fileToTXName, fileToTXDir)

    return file_to_Serve

@token_authorization
def check_new_video(request):
    # check if there is a new video for the screens
    print(request.POST.getlist('auth_token'))
    curr_show = Show.objects.all()[0].name # get the current show name, Screens will compare theirs to it.
    return HttpResponse(curr_show) # send a response containing the name of the show

# screen quiry for settings:
@token_authorization
def newSettingsForMe(request):
    # ses
    tok = request.GET.getlist('auth_token')[0]

    tv = Screen.objects.filter(key__exact = tok)[0]

    print(tv)

    tvSettings = {
        'width' : tv.width,
        'height': tv.height,
        'useSched' : tv.useSchedule,
        'startTime': tv.startTime,
        'endTime' : tv.endTime
    }

    return JsonResponse(tvSettings)

# mkae changes to the tv settings
def tvSettingsChange(request):
    tvId = request.POST.getlist('tv.id')[0]
    tvAdd = request.POST.getlist('tv.address')[0]
    #tvWidth = request.POST.getlist('tv.width')[0]
    #tvHeight = request.POST.getlist('tv.height')[0]
    try: # boolean values don't exist if they aren't checked...
        tvUseSched = request.POST.getlist('tv.useSched')[0]
    except:
        tvUseSched = 0 # do a 0, it makes things easier
    tvStartTime = request.POST.getlist('tv.starttime')[0]
    tvEndTime = request.POST.getlist('tv.endtime')[0]
    # int(tvWidth), int(tvHeight),
    print(tvId, tvAdd, tvUseSched, tvStartTime, tvEndTime)

    # updae / add information:
    tv = Screen.objects.filter(id__exact=tvId)[0]
    tv.hostIP = tvAdd # set TV address
    #tv.width = int(tvWidth)
    #tv.height = int(tvHeight)
    if int(tvUseSched) > 0:
        tv.useSchedule = True # boolean Value, I don't know how this uses
    else:
        tv.useSchedule = False
    tv.startTime = datetime.datetime.strptime(tvStartTime, '%H:%M')
    tv.endTime = datetime.datetime.strptime(tvEndTime, '%H:%M')
    tv.save()

    return redirect('/screens')

#_______________________________________________________________________________
# Media Creation things

def get_mp4_thumbnail(file="./static/content/animals-Imgur.mp4"):
    #abspath = str(os.path.abspath(file)) #
    rel_path = file[1:file.rindex('/')] # get the path and file split
    file = file[file.rindex('/')+1:len(file)]

    #print(abs_pth, abspath.rindex('/')-len(abspath), file)
    thumbnail = VM.get_thumbnail(abs_path=rel_path, file_name=file)
    return thumbnail

@login_required
##@staff_member_required
def make_video(request):

    videoFilePath = make_video_worker()

    return HttpResponse("
Show accelerated video: %s

Go Back
"%videoFilePath) #redirect('.')

def make_video_worker():
    # reusable worker!
    print("Compile Video")
    # get your content hat is selected:
    slides = Content.objects.filter(useInShow=True)
    #print(slides)
    # add a show entry:

    # compile_video(show_items={}, output_path='./static/shows/', frame_rate=30)
    global Video_Compiling
    Video_Compiling = True # indicate the video is compiling

    print("video compiling:", Video_Compiling)
    videoFilePath, new_show_name = VM.compile_video(show_items=slides)
    new_show = Show(name = new_show_name, file = videoFilePath)
    new_show.save()

    Video_Compiling = False # indicate the video is done

    print("video compiling:", Video_Compiling)

    # purge old shows:
    shows = Show.objects.all()

    for item in shows: # for every old show:
        if item.name != new_show_name: # if it isn't the newest
            os.remove(item.file) # delete the video file
            item.delete() # get rid of the DB entry

    return videoFilePath # return a thing for use by routines...

def video_compile_status(request):
    # routine for showing the status of the video comnpile for the UI,
    # so the user doens't get frustrated
    print(request.method)
    status = HttpResponse("%s"%(Video_Compiling))

    return status

#_______________________________________________________________________________
# Settings Managemnt Routines:
@login_required
##@staff_member_required # Apply selection of images
def apply_changes(request):
    changes = request.POST
    #log_applied_changes(changes)
    cid = changes.getlist('cid')
    sdate = changes.getlist('sdate')
    edate = changes.getlist('edate')
    dispTime = changes.getlist('dispTime')
    gifIteration = changes.getlist('gifIteration')
    print("GIFs:", gifIteration)
    use = changes.getlist('use')
    delOnEnd = changes.getlist('deleteOnEnd')

    #print(cid)
    for i in range(0, len(cid)): # adjust dates and dt
        item = Content.objects.get(id=cid[i])
        item.startDate = sdate[i]
        item.expireDate = edate[i]
        try:
            dt = int(dispTime[i])
            if dt < 301:
                item.displayTime = dt
            else:
                item.displayTime = 300
        except:
            pass
        try:
            gi = int(gifIteration[i])
            if gi < 301:
                item.gifIteration = gi
            else:
                item.gifIteration = 300
        except:
            pass

        item.save() # save the entries

    # determine if in delOnEnd:
    if len(delOnEnd) > 0:
        for e in cid:
            item = Content.objects.get(id=e)
            if e in delOnEnd:
                item.deleteOnExpire = True
            else:
                item.deleteOnExpire = False
            item.save() # save the entry

    # determine if in use:
    if len(use) > 0:
        for e in cid:
            item = Content.objects.get(id=e)
            if e in use: # if the element is in Use:
                item.useInShow = True # show
            else:
                item.useInShow = False # don't show
            item.save() # save the entry

    try: # try to delete things,
        if len(request.POST.getlist('Delete')) > 0:
            print("Deleting Things")
            delete_content(request)
    except Exception as e: # otherwise don't delete things
        print("Not deleting things")
        print(e)

    # rather than changing the page, redirect to self,
    # must return something
    return redirect('.')

def log_applied_changes(changes):
    # log the post to a file for later analysis
    try:
        with open('./manager/static/post.json','a') as file:
            file.write(str(changes))
            file.close()
    except FileNotFoundError:
        with open('./manager/static/post.json', 'w') as file:
            file.write(str(changes))
            file.close()
        print("Whoops")

@login_required
##@staff_member_required
def getContent(request):
    #print(File.objects.all())
    rem_addr = request.META.get('REMOTE_ADDR')

    # content template:
    content = {
        'File':[],
        'os':time.time(),
        'you':rem_addr
    }

    # get list of images, build web content
    #gottenFiles = os.listdir('./manager/static/content')
    files = Content.objects.all().order_by('expireDate')
    for entry in files:
        #print(entry.id, ':', entry.imageName, entry.expireDate.date())
        content['File'].append({
            'cid':entry.id,
            # strip off some of the non-pertanant information in the files and images
            'path':entry.imageName.replace("./manager/static/content/", ""),
            'filePath':entry.file.replace("./manager/static/content/", ""),
            'use':entry.useInShow,
            'startDate':str(entry.startDate.date()),
            'endDate':str(entry.expireDate.date()),
            'gifIteration':entry.gifIteration,
            'deleteOnEnd':entry.deleteOnExpire,
            'displayTime':entry.displayTime
        })

    #print(content)
    return content

#_______________________________________________________________________________
# content upkeep routines, used to remove old files, and refresh the slideshow
def upkeep():
    print("executing upkeep...")
    #Entry.objects.filter(pub_date__lte='2006-01-01')
    # get the contents that are to be deleted on expire date
    files = Content.objects.filter(deleteOnExpire=True).order_by("expireDate")
    # get the current date for use later:
    currentTime = timezone.now()
    deletecount = 0
    if len(files) > 0:
        print("looking for expired content")
        for entry in files:
            #print(entry.id, ':', entry.imageName, 'Expire:', entry.expireDate, 'Now:', currentTime)
            if currentTime.date() >= entry.expireDate.date():
                print("Deleting expored content: c%d"%(entry.id))
                delete_worker(entry)
                deletecount += 1 # increment the number of deleted things

    if deletecount > 0:
        # then recompile video
        make_video_worker()

# when the delete button is pressed, this happens:
def delete_content(request):
    deletion = request.POST.getlist('Delete')
    #log_applied_changes(deletion)
    # get the content we want to delete
    for dele in deletion:
        remove = Content.objects.get(id__exact=dele)
        print("Delete:",remove.imageName)

        delete_worker(remove)

    # redirect to the same page
    return redirect('.')

# actual reusable worker
def delete_worker(dbEntry):
    # Pass DJango DB object into, and things go away
    # use OS remove_dir to delete the whole directory
    directory = dbEntry.imageName[0:dbEntry.imageName.rindex('/')+1]
    print(directory)
    # first delete the contents:
    try: # try delete image file
        os.remove(dbEntry.imageName)
    except Exception as e:
        print('\n\n%s\n\n'%e)

    try: # try delete file file, if no file file, the an error will arise
        print("\ndelete:%s"%dbEntry.file)
        os.remove(dbEntry.file)
    except Exception as e: # then handle it...
        print('\n%s\n\n'%e)

    # then delete the directory:
    os.rmdir(directory)

    # remove the database entry
    dbEntry.delete()

#_______________________________________________________________________________
# Initalize Scheduling with the Schedule module:

def schedule_check():
    # schedule upkeep to run every day at 03:30 in the morning
    schedule.every().day.at('01:30').do(upkeep).run() # remove run in production

    while(True):
        #with open("./threadcheck.txt", "w+") as f:
        #    f.write(str(time.time()))
        #    f.close()
        schedule.run_pending() # quirie the schedule
        time.sleep(1) # wait a bit, keep CPU load low

def schedule_thread():
    print("running schedule thread")
    thr = Thread(target=schedule_check, daemon=True)
    thr.start()

schedule_thread()


"""
Created on 2022-12-13 20:36:01.409428
@author: suvobanik
"""

from math import log

import numpy as np

def playouts(
    idx,
    depthlist,
    playoutdata,
    Scorelist,
    parameterlist,
    perturbate,
    evaluate,
    a,
    maxdepth,
    nplayouts=10,
):

    nodeID = idx

    for i in range(nplayouts):
        idx += 1
        playdata = perturbate(
            parameterlist[nodeID], depth=depthlist[nodeID], a=a, maxdepth=maxdepth
        )
        playdata_relaxed, playscore = evaluate(playdata)
        #        print("Node: {}, Playout: {} Score: {}".format(nodeID,i+1,playscore))

        Scorelist.append(playscore)
        parameterlist.append(playdata_relaxed)
        playoutdata[nodeID].append(idx)

    return idx, playoutdata, Scorelist, parameterlist

def expansion_simulation(
    parentID,
    idx,
    visits,
    childlist,
    parent,
    depthlist,
    playoutdata,
    Scorelist,
    parameterlist,
    perturbate,
    evaluate,
    a,
    maxdepth,
    nplayouts=10,
):

    playindexes = playoutdata[parentID]
    playscores = [Scorelist[i] for i in playindexes]
    bestplayindex = playindexes[playscores.index(min(playscores))]

    # =========== update =============

    idx += 1
    visits[parentID] += 1
    visits[idx] = 1
    try:
        childlist[parentID].append(idx)
    except:
        childlist[parentID] = [idx]

    parent[idx] = parentID
    depth = depthlist[parentID] + 1
    depthlist[idx] = depth
    playoutdata[idx] = []
    data = perturbate(parameterlist[bestplayindex], depth=depth, a=a, maxdepth=maxdepth)
    data_relaxed, score = evaluate(data)
    Scorelist.append(score)
    parameterlist.append(data)

    idx, playoutdata, Scorelist, parameterlist = playouts(
        idx,
        depthlist,
        playoutdata,
        Scorelist,
        parameterlist,
        perturbate,
        evaluate,
        a,
        maxdepth,
        nplayouts=nplayouts,
    )

    return (
        visits,
        childlist,
        parent,
        depthlist,
        playoutdata,
        Scorelist,
        parameterlist,
        idx,
    )

def backpropagation_selection(
    visits,
    childlist,
    parent,
    depthlist,
    playoutdata,
    Scorelist,
    parameterlist,
    maxdepth,
    exploreconstant=1,
):
    def get_linegae(node, childlist):
        total_list = []
        try:
            total_list += childlist[node]
        except:
            return total_list
        for child in childlist[node]:
            total_list += get_linegae(child, childlist)

        return total_list

    selectionScores = []
    nodes = list(parent.keys())

    for n in nodes:
        par = parent[n]
        depth = depthlist[n]
        if par is None or depth > maxdepth:
            selectionScores.append(-1e300)

        else:
            lineage = get_linegae(n, childlist)
            allindexes = []
            for l in [n] + lineage:
                allindexes += playoutdata[l]

            bestreward = min([Scorelist[i] for i in lineage + allindexes])

            UCB_score = -bestreward + exploreconstant * np.sqrt(
                log(visits[par]) / visits[n]
            )

            selectionScores.append(UCB_score)

    c = zip(selectionScores, nodes)
    c = sorted(c, key=lambda x: -x[0])
    selectionScores, nodes = zip(*c)

    return nodes[0]

def MCTS(
    rootdata,
    perturbate,
    evaluate,
    niterations=200,
    headexpand=10,
    nexpand=3,
    nsimulate=3,
    nplayouts=10,
    exploreconstant=1,
    maxdepth=12,
    a=3,
    selected_node=0,
):

    if selected_node == 0:
        visits = {0: 1}
        childlist = {0: []}
        parent = {0: None}
        depthlist = {0: 0}
        playoutdata = {0: []}
        data_relaxed, score = evaluate(rootdata)
        Scorelist = [score]
        parameterlist = [data_relaxed]
        idx = 0

        # =======run playouts for rootnode ==================

        idx, playoutdata, Scorelist, parameterlist = playouts(
            idx,
            depthlist,
            playoutdata,
            Scorelist,
            parameterlist,
            perturbate,
            evaluate,
            a,
            maxdepth,
            nplayouts=nplayouts,
        )

    # =======simulation and expansion==================

    for _ in range(niterations):

        for _ in range(nexpand):
            (
                visits,
                childlist,
                parent,
                depthlist,
                playoutdata,
                Scorelist,
                parameterlist,
                idx,
            ) = expansion_simulation(
                selected_node,
                idx,
                visits,
                childlist,
                parent,
                depthlist,
                playoutdata,
                Scorelist,
                parameterlist,
                perturbate,
                evaluate,
                a,
                maxdepth,
                nplayouts=nplayouts,
            )

            selected_node = backpropagation_selection(
                visits,
                childlist,
                parent,
                depthlist,
                playoutdata,
                Scorelist,
                parameterlist,
                maxdepth,
                exploreconstant=exploreconstant,
            )

        print(
            "total evaluations: {}, best score yet : {}".format(
                len(Scorelist), min(Scorelist)
            )
        )


import numpy as np
import random
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from utils import to_torch_sparse_tensor, to_list

class AutoGNN(nn.Module):

    def __init__(self, feat_dim, graph,
                 hidden_dim, step_num, sample_num, nonlinear, agg,
                 class_num, dropout):
        super(AutoGNN, self).__init__()

        self.feat_dim = feat_dim
        self.hidden_dim = hidden_dim
        self.sample_num = sample_num
        self.step_num = step_num
        self.nonlinear = nonlinear
        self.agg = agg
        self.class_num = class_num

        self.sampleGraph(graph)
        self.precompute = False

        self.W =  nn.ModuleList([nn.Linear(feat_dim, hidden_dim, bias=False)])
        for _ in range(step_num-2):
            self.W.append(nn.Linear(hidden_dim, hidden_dim, bias=False))
        self.lastW = nn.Linear(hidden_dim, class_num, bias=False)
        self.precomputeW = nn.Linear(feat_dim, class_num, bias=False)

        self.drop = dropout > 0 and dropout < 1
        if self.drop:
            self.Dropout = nn.Dropout(dropout)

        for w in self.W:
            nn.init.xavier_uniform_(w.weight.data)

    def sampleGraph(self, graph):
        if self.sample_num == -1:
            self.adj = to_torch_sparse_tensor(graph)
            return

        self.adj={}
        for i in range(len(graph)):
            self.adj[i] = []
        for node,neighbors in graph.items():
            if len(neighbors) == 0:
                continue
            if len(neighbors) > self.sample_num:
                neighbors = np.random.choice(neighbors, self.sample_num, replace=False)
            elif len(neighbors) < self.sample_num:
                neighbors = np.random.choice(neighbors, self.sample_num, replace=True)
            self.adj[node] = neighbors
        self.adj = to_torch_sparse_tensor(self.adj, self.agg)

    def initWeight(self, initW):
        for w in self.W:
            w.weight.data = initW
            for param in w.parameters():
                param.requires_grad = False

    def precomputeX(self, X):
        if self.nonlinear == False:
            for _ in range(self.step_num):
                X = torch.spmm(self.adj, X)
            self.precompX = X
            self.precompute = True

    def forward(self, X):
        if self.precompute:
            X = self.precomputeW(self.precompX)
        else:
            for w in self.W:
                X = torch.spmm(self.adj, X)
                X = w(X)
                if self.drop:
                    X = self.Dropout(X)
                if self.nonlinear:
                    X = F.relu(X)
            X = self.lastW(X)
        return X

def get_auto_model(feat_dim, graph,
                   hidden_dim, step_num, sample_num, nonlinear, agg,
                   class_num, dropout):
    model = AutoGNN(feat_dim, graph,
                    hidden_dim, step_num, sample_num, nonlinear, agg,
                    class_num, dropout)
    return model


# -*- coding: utf-8 -*-
"""
Created on Sun Aug  7 17:31:01 2022

@author: IIT
"""

#%% library import
import torch
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
from transformers import BertTokenizer, BertForSequenceClassification
from sklearn.model_selection import train_test_split

import pandas as pd
import numpy as np
from sklearn import preprocessing
from sklearn.model_selection import train_test_split

from transformers import DataCollatorWithPadding
from transformers import AutoModelForSequenceClassification, TrainingArguments, Trainer

#%% variables
#%%
num_of_epochs = 10
learning_rate = 5e-5
# %% defining custom dataset, training and testing class
import torch

class Dataset(torch.utils.data.Dataset):
    """
    Class to store the data as PyTorch Dataset
    """

    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        # an encoding can have keys such as input_ids and attention_mask
        # item is a dictionary which has the same keys as the encoding has
        # and the values are the idxth value of the corresponding key (in PyTorch's tensor format)
        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.labels)

print(Dataset.__doc__)

def train(dataloader, model, optimizer):
    """Method to train the model"""

    model.train()

    epoch_loss = 0
    size = len(dataloader.dataset)

    for i, batch in enumerate(dataloader):
        input_ids = batch['input_ids'].to(device)
        attention_mask = batch['attention_mask'].to(device)
        labels = batch['labels'].type(torch.LongTensor).to(device)

        outputs = model(input_ids, attention_mask=attention_mask, labels=labels)

        optimizer.zero_grad()
        loss = outputs.loss
        epoch_loss += loss.item()
        loss.backward()
        optimizer.step()

    print('Training loss: {:.3f}'.format(epoch_loss / size))

print(train.__doc__)

def test(dataloader, model):
    """Method to test the model's accuracy and loss on the validation set"""

    model.eval()

    size = len(dataloader.dataset)
    test_loss, accuracy = 0, 0

    with torch.no_grad():
        for batch in dataloader:
            X, y = batch['input_ids'].to(device), batch['labels'].type(torch.LongTensor).to(device)
            pred = model(X, labels=y)

            test_loss += pred.loss
            accuracy += (pred.logits.softmax(1).argmax(1) == y).type(torch.float).sum().item()

        test_loss /= size
        accuracy /= size

        print("Test loss: {:.3f}, accuracy: {:.3f}%".format(test_loss, accuracy * 100))

print(test.__doc__)

# %% reading data
data = pd.read_csv('../data/output/data_2019.csv')

allcols = ['description','attackVector','attackComplexity','privilegesRequired','userInteraction','scope',
        'confidentialityImpact','integrityImpact','availabilityImpact','baseScore','baseSeverity','exploitabilityScore','impactScore']
metc_clmn = ['attackVector','attackComplexity','privilegesRequired','userInteraction','scope',
        'confidentialityImpact','integrityImpact','availabilityImpact','baseScore','baseSeverity','exploitabilityScore','impactScore']
data = data[allcols].dropna()

#%% dataset splitiing for test, validate and train
import collections
X_main, X_test_res, y_main, y_test_res = train_test_split(data['description'], data[metc_clmn], test_size=0.2, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X_main, y_main, test_size=0.2, random_state=42)

#%% converting to list
X_train = X_train.values.tolist()
X_test = X_test.values.tolist()
X_test_res = X_test_res.values.tolist()

#%% tokenizing test data
tokenizer = BertTokenizer.from_pretrained('./model/bert_uncased_L-4_H-512_A-8',do_lower_case = True)

X_train = tokenizer(X_train, return_tensors="pt", padding="max_length", max_length=128, truncation=True)
X_test = tokenizer(X_test, return_tensors="pt", padding="max_length", max_length=128, truncation=True)
X_test_res = tokenizer(X_test_res, return_tensors="pt", padding="max_length", max_length=128, truncation=True)

# %% preparing label data

#%% creating dataSet and dataLoaders
from torch.utils.data import DataLoader

train_loader = DataLoader(Dataset(X_train, y_train), batch_size=64, shuffle=True)
test_loader = DataLoader(Dataset(X_test, y_test), batch_size=64, shuffle=True)
test_res_loader = DataLoader(Dataset(X_test_res, y_test_res), batch_size=64, shuffle=True)

# %%

from numpy import mean
from torch import nn

from transformers import AdamW

print('Created train & val datasets.')

# device (turn on GPU acceleration for faster execution)
device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')

# model
model = AutoModelForSequenceClassification.from_pretrained('./model/bert_uncased_L-4_H-512_A-8', num_labels=4)
model.to(device)

#optimizer
optimizer = AdamW(model.parameters(), lr=learning_rate)

#%%

def pri():
    '''doc'''
    number = 0
    print(number)

number = {'a':12,'v':432}
pri()
print(number)


#!/usr/bin/env python

import argparse
import re

def convert_gff_to_gtf(gff_path, gtf_path):
    t = re.compile(r'ID=[0-9]+_([0-9]+);')
    t2 = re.compile(r'ID=cds([0-9]+);')
    with open(gff_path, "r") as gff:
        with open(gtf_path, "w") as gtf:
            for line in gff:
                if line.startswith("#"):
                    continue
                toks = line.strip().split("\t")
                try:
                    orf = t.findall(toks[-1])[0]
                except:
                    try:
                        orf = t2.findall(toks[-1])[0]
                    except:
                        continue
                gene_id = toks[0] + "_" + orf
                toks[-1] = 'gene_id "%s"; %s' % (gene_id, toks[-1])
                gtf.write("\t".join(toks) + "\n")

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Converts GFF to GTF from Prodigal output.')

    # required
    parser.add_argument('gff', help='Path to input file.')
    parser.add_argument('gtf', help='Path to output file.')

    args = parser.parse_args()

    convert_gff_to_gtf(args.gff, args.gtf)


from collections import defaultdict
from typing import List

class GroupAnagrams:
    def groupAnagrams(self, strs: List[str]) -> List[List[str]]:
        group=defaultdict(list)
        for st in strs:
            group[tuple((sorted(st)))].append(st)
        return list(group.values())

if __name__ == '__main__':
    init = GroupAnagrams()
    print(init.groupAnagrams(["eat", "tea", "tan", "ate", "nat", "bat"]))


# coding:utf-8
"""
输入一个正整数 target ，输出所有和为 target 的连续正整数序列（至少含有两个数）。

序列内的数字由小到大排列，不同序列按照首个数字从小到大排列。

示例 1：

输入：target = 9
输出：[[2,3,4],[4,5]]
示例 2：

输入：target = 15
输出：[[1,2,3,4,5],[4,5,6],[7,8]]
"""

class Solution:
    def findContinuousSequence(self, target: int) -> list:
        left, right, sum = 1, 1, 0
        res = []
        while right <= ((target + 1) / 2):
            sum += right
            while sum > target:
                sum -= left
                left += 1
            if sum == target:
                tmp = [i for i in range(left, right+1)]
                res.append(tmp)
            right += 1
        return res

s1 = Solution()
print(s1.findContinuousSequence(9))


import pandas as pd
from django.core.management.base import BaseCommand, CommandError
from meteoapp.models import MeteoData, Site

class Command(BaseCommand):

    help =  """
            Permet d'insérer les données d'un fichier CSV modélisant le modèle MeteoData dans la base de données.
            Arguments:
                - Chemin vers le fichier csv
                - Nom du site auquels appartiennent les données
            """
    def add_arguments(self, parser):
        parser.add_argument('filename', nargs=1, type=str, help='Chemin vers le fichier csv contenant les données')
        parser.add_argument('sitename', nargs=1, type=str, help='Nom du site auxquel appartiennent les données')

    def handle(self, *args, **options):
        df = pd.read_csv(options['filename'][0], parse_dates=['Date Time'], dayfirst=True)
        new_columns = [field.name for field in MeteoData._meta.get_fields()]
        df.columns = new_columns[2:]

        site, created = Site.objects.get_or_create(name=options['sitename'][0])

        MeteoData.objects.bulk_create(MeteoData(site=site, **values) for values in df.to_dict('records'))


#o programa lê as coordenadas recebidas pelo usuário e marca um X na cédula correspondente do mapa

row1 = ["⬜","⬜","⬜"]
row2 = ["⬜","⬜","⬜"]
row3 = ["⬜","⬜","⬜"]
map = [row1, row2, row3]
print(f"{row1}\n{row2}\n{row3}")
position = input("Where do you want to put the treasure? ")

# o usuário entrará com 31, por exemplo. 3 é a coluna e 1 é a linha
row = int(position[1])

# a coluna corresponderá ao elemento das linhas
column = int(position[0])

# marcando um X no mapa
# porque contamos de a partir de 1, mas as listas começam a contar do 0, é preciso diminuir 1 unidade de cada índice
map[row - 1][column - 1] = 'X'

# alternativamente, para trabalharmos com o índice, poderia ser feito desta forma:
# horizontal = int(position[0])
# vertical = int(position[1])
# row = map[vertical - 1]
# row[horizontal - 1] = "X"

print(f"{row1}\n{row2}\n{row3}")


from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

DOCUMENTATION = r'''
---
module: zvm_startstop_user

short_description: z/VM start and stop guest/user Ansible Module

version_added: "0.0.1"

description:
    an Ansible module starting and stopping z/VM virtual machines.
    One option is to use SMAPI, but SMAPI does not provide an API to direct a
    virtual machine to IPL from a specific device, nor does it allow to pass
    a 'first command' to a virtual machine. the z/VM XAUTOLOG command does allow
    that however, so we support both methods.

options:
    name:
        description: this is the target user to be started or stopped
        required: true
        type: str
    loggedon:
        description: kinda like "exists" but for virtual machine instantiation
        required: true
        type: bool
    viaSMAPI:
        description: are we using SMAPI or CP XAUTOLOG|FORCE to start/stop the target user
        type: bool
    IPLdev:
        description: what device should the virtual machine IPL/Boot from once it logs on ?
        type: str
    CMD:
        description: pass a command to the operating system as it starts
        type: str
    stoptime:
        description:
            if we are logging a virtual machine off how long should we allow the OS to
            complete its internal shutdown. Time is in seconds.
        type: int

author:
    - Jay Brenneman (@rjbrenn)
'''

EXAMPLES = r'''
# start one up via SMAPI
- name: starting
  zvm_startstop_user:
    name: LINUX02
    loggedon: true
    viaSMAPI: true

# start one with XAUTOLOG and ask it to IPL the card reader
- name: start rdr
  zvm_startstop_user:
    name: LINUX02
    loggedon: true
    viaSMAPI: false
    IPLdev: 00c

# stop one with very little waiting
- name: force
  zvm_startstop_user:
    name: LINUX02
    loggedon: false
    viaSMAPI: false
    stoptime: 2

'''

RETURN = r'''
return_stdout:
    description: the stdout responses from smcli as a dict
    returned: always
    type: list
    elements: str
    sample: "...Done"
return_stderr:
    description: the stderr responses from smcli as a dict
    returned: always
    type: list
    elements: str
    sample: "HCPAUT053E LARRY not in CP directory"
return_code:
    description: The results of doing the thing
    type: int
    returned: always
    sample: 0
reason_code:
    description: The reason for the result
    type: int
    returned: always
    sample: 0
'''

from ansible.module_utils.basic import AnsibleModule

def run_module():
    module_args = dict(
        name=dict(type='str', required=True),
        loggedon=dict(type='bool', required=True),
        viaSMAPI=dict(type='bool', required=False),
        stoptime=dict(type='int', required=False),
        IPLdev=dict(type='str', required=False),
        CMD=dict(type='str', required=False)
    )

    result = dict(
        changed=False,
        return_stdout=[],
        return_stderr=[],
        return_code=-9,
        reason_code=-9
    )

    module = AnsibleModule(
        argument_spec=module_args,
        supports_check_mode=True
    )

    if module.check_mode:
        module.exit_json(**result)

    if module.params['loggedon']:
        # we are starting systems up
        # should we use SMAPI ?
        if module.params['viaSMAPI']:
            # we are starting systems using SMAPI
            smapi_up_cmd = []
            smapi_up_cmd.append("/opt/zthin/bin/smcli")
            smapi_up_cmd.append("Image_Activate")
            smapi_up_cmd.append("-T")
            smapi_up_cmd.append(module.params['name'])

            # check_rc=False because if we get rerun idempotently this may fail with 'already running'
            smcli_results = module.run_command(smapi_up_cmd, check_rc=False)
            result['return_stdout'] = smcli_results[1]
            result['return_stderr'] = smcli_results[2]
            result['return_code'] = smcli_results[0]
        else:
            # we are starting systems using XAUTOLOG
            xauto_up_cmd = []
            xauto_up_cmd.append("vmcp")
            xauto_up_cmd.append("xautolog")
            xauto_up_cmd.append(module.params['name'])
            if module.params['IPLdev'] is not None:
                xauto_up_cmd.append("IPL")
                xauto_up_cmd.append(module.params['IPLdev'])
            if module.params['CMD'] is not None:
                xauto_up_cmd.append("CMD")
                xauto_up_cmd.append(module.params['CMD'])

            # check_rc=False because if we get rerun idempotently this may fail with 'already running'
            vmcp_results = module.run_command(xauto_up_cmd, check_rc=False)
            result['return_stdout'] = vmcp_results[1]
            result['return_stderr'] = vmcp_results[2]
            result['return_code'] = vmcp_results[0]
    else:
        # we are shutting systems down
        # should we use SMAPI ?
        if module.params['viaSMAPI']:
            # we are shutting doen systems using SMAPI
            smapi_down_cmd = []
            smapi_down_cmd.append("/opt/zthin/bin/smcli")
            smapi_down_cmd.append("Image_Deactivate")
            smapi_down_cmd.append("-T")
            smapi_down_cmd.append(module.params['name'])
            if module.params['stoptime'] is not None:
                smapi_down_cmd.append("-f")
                smapi_down_cmd.append(str(module.params['stoptime']))

            # check_rc=False because if we get rerun idempotently this may fail with 'not running'
            smcli_results = module.run_command(smapi_down_cmd, check_rc=False)
            result['return_stdout'] = smcli_results[1]
            result['return_stderr'] = smcli_results[2]
            result['return_code'] = smcli_results[0]
        else:
            # we are shutting down systems using FORCE
            force_dn_cmd = []
            force_dn_cmd.append("vmcp")
            force_dn_cmd.append("force")
            force_dn_cmd.append(module.params['name'])
            if module.params['stoptime'] is not None:
                force_dn_cmd.append("within")
                force_dn_cmd.append(str(module.params['stoptime']))

            # check_rc=False because if we get rerun idempotently this may fail with 'not running'
            vmcp_results = module.run_command(force_dn_cmd, check_rc=False)
            result['return_stdout'] = vmcp_results[1]
            result['return_stderr'] = vmcp_results[2]
            result['return_code'] = vmcp_results[0]

    if result['return_code'] == 0:
        result['changed'] = True
        module.exit_json(**result)

    if result['return_code'] >= 1:
        notreally_errs = ["Image already active", "HCPUSO045E", "not logged on"]
        # notreally_errs is a list of VM error messages that are OK within this contect
        noterror_count = 0

        # an SMAPI error message comes back like this. The important parts are the
        # return code and the reason code
        #    "return_stdout": [
        #        "Defining thingone in the directory... ",
        #        "Failed",
        #        "  Return Code: 400",
        #        "  Reason Code: 8",
        #        "  Description: ULGSMC5400E Image or profile name already defined",
        #        "  API issued : Image_Create_DM"
        #    ]

        # this is gross but I looked at alternatives and they were less readable
        # scann return stdout and stderr for
        for j in notreally_errs:
            if result['return_stdout'].find(j) != -1:
                noterror_count += 1
        if noterror_count > 0:
            # optimistically guessing this is not an error condition
            # it might still actually be an error condition if there is a real error among the not-errors
            # will figure that out later if it ends up happening
            # exit no error
            result['return_stdout'] += ">>skipping an error because its probably OK in this situation<<"
            module.exit_json(**result)
        # here we are in error condition
        errormsg = "failing return code from smcli is: " + str(result['return_code'])
        module.fail_json(errormsg, **result)

    errormsg = "unknown return code from smcli: " + str(result['return_code'])
    result['return_stderr'] = ["how did we get here???"]
    module.fail_json(errormsg, **result)

def main():
    run_module()

if __name__ == '__main__':
    main()


from dataclasses import field
from pdb import post_mortem
from rest_framework import serializers
from user.models import User as UserModel
from .models import Post as PostModel, PostImg as PostImgModel

class PostSerializer(serializers.ModelSerializer):
    author = serializers.SerializerMethodField(read_only=True)
    author_id = serializers.IntegerField(write_only= True)
    postimg = serializers.CharField(write_only= True)
    def get_author(self, obj):
        return obj.author.username
    def create(self, validated_data):
        author_id = validated_data.pop('author_id')
        author = UserModel.objects.get(id = author_id)
        post_img = validated_data.pop('postimg')
        new_post= PostModel.objects.create(
            author = author,
            **validated_data
        )
        new_post.save()
        PostImgModel(post= new_post, img_url =post_img).save()
        return new_post
    def update(self, instance, validated_data):
        post_img = validated_data.pop('postimg')
        for key, value in validated_data.items():
            setattr(instance,key,value)
        instance.save()
        new_post_img = PostImgModel.objects.get(post = instance)
        new_post_img.img_url = post_img
        new_post_img.save()
        return instance
    class Meta:
        model = PostModel
        fields = ["author", "author_id", "title", "content", "postimg"]


from django.contrib import admin
from django.urls import path
from django.conf.urls import include, url

from rest_framework_swagger.views import get_swagger_view
from rest_framework_extensions.routers import ExtendedDefaultRouter

from general.views import *
from accounts.views import *

admin.site.site_title = admin.site.index_title = "Humble Rings Administration"

router = ExtendedDefaultRouter()

router.register(r'users', UserViewSet, basename='users')

workout_routes = router.register(
    r'workouts',
    WorkoutViewSet,
    basename='workouts'
)

exercise_routes = router.register(
    r'exercises',
    ExerciseViewSet,
    basename='exercises',
)

set_routes = workout_routes.register(
    r'sets',
    SetViewSet,
    basename='sets',
    parents_query_lookups=["workout"]
)

schema_view = get_swagger_view(title='Humble Rings API')

urlpatterns = [
    path('admin/', admin.site.urls),
    url(r'^api/', include(router.urls)),
    url(r'^auth/sign-up/', CustomRegisterView.as_view(), name='sign_up'),
    url(r'^auth/sign-in/', CustomLoginView.as_view(), name='sign_in'),
    path(r'swagger/', schema_view),
]


import unittest

class MorseTests(unittest.TestCase):
    def test_01_english_to_morse(self):
        click(Pattern("clear_english_text.png").similar(0.90))
        click(Pattern("english_tab.png").similar(0.80).targetOffset(0,25))
        type("test")
        click("english_to_morse.png")
        assert exists("test.png") is not None

    def test_02_morse_to_english(self):
        click(Pattern("clear_morse_text.png").similar(0.90))
        click(Pattern("morse_tab.png").targetOffset(0,25))
        type("--.- .-  / ")
        click("morse_to_english.png")
        assert exists("qa.png") is not None

suite = unittest.TestLoader().loadTestsFromTestCase(MorseTests)
unittest.TextTestRunner(verbosity=2).run(suite)


import os
import subprocess
import time
import socket
from psutil import cpu_percent, virtual_memory
from tkinter import *
from IPy import IP

def error_message(parent_window, msg):
    error_window = Toplevel(parent_window)
    error_window.configure(background='black')
    error_window.title("Error")

    error_label = Label(error_window, font=('arial', 25, 'bold'), fg='red', bg='black')
    error_label.pack()
    error_label.config(text=msg)
    error_button = Button(error_window, text="Aceptar", command=error_window.destroy)
    error_button.pack()

def reset_canary(comm_cut_queue, reset_iptables_queue):
    if not reset_iptables_queue.empty():
        subprocess.run("iptables -D INPUT -j DROP", shell=True)
    comm_cut_queue.put("Cortar")
    time.sleep(1)
    os.execl(sys.executable, sys.executable, *sys.argv)

def delete_ip(observer_window, ip_list, comm_cut_queue, reset_iptables_queue):
    def destroy_ip():
        try:
            bad_ip = str(white_list.get(white_list.curselection()))
            with open("ip.txt", "w") as f:
                for good_ip in ip_list:
                    if str(good_ip).strip("\n") != bad_ip:
                        f.write(str(good_ip) + "\n")
            delete_window.destroy()
            reset_canary(comm_cut_queue, reset_iptables_queue)
        except TclError:
            error_message(delete_window, "Selecciona una IP de la lista")

    delete_window = Toplevel(observer_window)
    delete_window.attributes("-fullscreen", True)
    delete_window.configure(background='black')
    delete_window.title("Canario")

    ip_delete_label = Label(delete_window, font=('arial', 25, 'bold'), fg='skyblue', bg='black')
    ip_delete_label.pack()
    ip_delete_label.config(text="Señale la IP que desea eliminar y, luego, pulse en el botón:")
    ip_delete_label.place(rely=0.1, relx=0.5, anchor=N)

    white_list = Listbox(delete_window, font=('arial', 20), width=50, selectmode=SINGLE)
    white_list.pack()
    white_list.place(rely=0.25, relx=0.5, anchor=N)

    delete_button = Button(delete_window, text="Eliminar", command=destroy_ip)
    delete_button.pack()
    delete_button.place(rely=0.8, relx=0.5, anchor=N)

    back_button = Button(delete_window, text="Regresar", command=delete_window.destroy)
    back_button.pack()
    back_button.place(rely=0.975, relx=0.01, anchor=SW)

    index = 0

    for ip in ip_list:
        white_list.insert(index, ip)
        index += 1

def add_ip(observer_window, ip_list, comm_cut_queue, reset_iptables_queue):
    def insert_ip():
        if ip_entry.get() == "":
            error_message(add_window, "El campo de texto no puede estar vacío")
        else:
            try:
                socket.inet_aton(ip_entry.get())
                if IP(ip_entry.get()) in ip_list:
                    error_message(add_window, "La IP ya está en la lista")
                else:
                    new_ip = str(ip_entry.get())
                    with open("ip.txt", "a") as f:
                        f.write(new_ip + "\n")
                    add_window.destroy()
                    reset_canary(comm_cut_queue, reset_iptables_queue)
            except socket.error:
                error_message(add_window, "La IP debe tener una estructura XXX.XXX.XXX.XXX (p.e. 194.153.205.26)")

    add_window = Toplevel(observer_window)
    add_window.attributes("-fullscreen", True)
    add_window.configure(background='black')
    add_window.title("Canario")

    ip_insert_label = Label(add_window, font=('arial', 25, 'bold'), fg='skyblue', bg='black')
    ip_insert_label.pack()
    ip_insert_label.place(rely=0.1, relx=0.5, anchor=N)
    ip_insert_label.config(text="Inserte la IP para no ser considerada una amenaza:")

    ip_entry = Entry(add_window, font=('arial', 45))
    ip_entry.pack()
    ip_entry.place(rely=0.3, relx=0.5, anchor=N, width=500, height=50)

    add_button = Button(add_window, text="Insertar", command=insert_ip)
    add_button.pack()
    add_button.place(rely=0.5, relx=0.5, anchor=N)

    back_button = Button(add_window, text="Regresar", command=add_window.destroy)
    back_button.pack()
    back_button.place(rely=0.975, relx=0.01, anchor=SW)

def observer(detection_queue, comm_cut_queue, reset_iptables_queue, ip_list):
    observer_window = Tk()
    observer_window.attributes("-fullscreen", True)
    observer_window.configure(background='black')
    observer_window.title("Canario")

    clock_label = Label(observer_window, font=('arial', 150, 'bold'), fg='yellow', bg='black')
    clock_label.pack()

    date_label = Label(observer_window, font=('arial', 75, 'bold'), fg='skyblue', bg='black')
    date_label.pack()

    cpu_label = Label(observer_window, font=('arial', 50, 'bold'), fg='skyblue', bg='black')
    cpu_label.pack()

    ram_label = Label(observer_window, font=('arial', 50, 'bold'), fg='skyblue', bg='black')
    ram_label.pack()

    ccc_var = IntVar()
    com_cut_check = Checkbutton(observer_window, font=('arial', 15), fg='white', bg='black', selectcolor='black',
                                text='Cortar las comunicaciones tras un ataque.',
                                variable=ccc_var, onvalue=1, offvalue=0)
    com_cut_check.pack()
    com_cut_check.place(rely=0.975, relx=0.01, anchor=SW)
    com_cut_check.select()

    reset_button = Button(observer_window, font=('arial', 15), text='Reiniciar',
                          command=lambda: reset_canary(comm_cut_queue, reset_iptables_queue))
    reset_button.pack()
    reset_button.place(rely=0.975, relx=0.99, anchor=SE)

    add_button = Button(observer_window, font=('arial', 15), text='Insertar IP',
                        command=lambda: add_ip(observer_window, ip_list, comm_cut_queue, reset_iptables_queue))
    add_button.pack()
    add_button.place(rely=0.025, relx=0.01, anchor=NW)

    delete_button = Button(observer_window, font=('arial', 15), text='Eliminar IP',
                           command=lambda: delete_ip(observer_window, ip_list, comm_cut_queue, reset_iptables_queue))
    delete_button.pack()
    delete_button.place(rely=0.1, relx=0.01, anchor=NW)

    def tick():
        current_time = time.strftime('%H:%M:%S')
        date_text = time.strftime('%d-%m-%Y')
        clock_label.config(text=current_time)
        date_label.config(text=date_text)
        cpu_label.config(text="CPU: " + str(cpu_percent(None)))
        ram_label.config(text="RAM: " + str(virtual_memory().percent))

        if not detection_queue.empty():
            com_cut_text = ""
            if ccc_var.get() == 1:
                time.sleep(1)
                comm_cut_queue.put("Cortar")
                com_cut_text = "Los puertos han sido cerrados para impedir el ataque."

            warning_window = Toplevel(observer_window)
            warning_window.configure(background='black')
            warning_window.title("Canario")

            warning_label = Label(warning_window, font=('arial', 150, 'bold'), fg='red', bg='black')
            warning_label.pack()
            warning_label.config(text="AVISO")

            detection_message_label = Label(warning_window, font=('arial', 25, 'bold'), fg='white', bg='black', pady=25)
            detection_message_label.pack()
            detection_message_label.config(text="El dispositivo ha detectado posibles amenazas a las " + current_time)

            threat_list = Listbox(warning_window, font='arial', width=100)
            threat_list.pack()

            communication_cut_label = Label(warning_window, font=('arial', 10, 'bold'), fg='white', bg='black', pady=25)
            communication_cut_label.pack()
            communication_cut_label.config(text=com_cut_text)

            index = 0

            while not detection_queue.empty():
                threat_list.insert(index, detection_queue.get())
                index += 1

        clock_label.after(200, tick)

    tick()
    observer_window.mainloop()


import numpy as np
import pandas as pd
import matplotlib as plt

dataset = pd.read_csv('C:/Users/SHUBHAM TOTLA/Desktop/Data analytics/Deep Learning/Self_Organizing_Maps/Credit_Card_Applications.csv')
x = dataset.iloc[:,:-1].values
y = dataset.iloc[:,-1].values#Customer approval 0 =not approved and  1 =approved

#Feature Scaling

from sklearn.preprocessing import MinMaxScaler
sc= MinMaxScaler(feature_range = (0,1))
x= sc.fit_transform(x)

#Training the SOM

#minisom.py is an inbuilt lib alreaddy built by another developer
from minisom import MiniSom
som = MiniSom(x=10, y=10, input_len=15, sigma=1, learning_rate=0.5)
#x and y are dimensions of grid  as observations are less we take 10x10 dim
#input_len = no of features in x
#sigma = radius default value is 1
som.random_weights_init(x)#Randomly initialzes weight
som.train_random(data=x,num_iteration = 100)

#Visulaizing the results

#mean to neuron distance (mid) is mean of distances of all neuron around the radius of winning node
#The higher the mid the more the winning node is outlier(potential frauds)

from pylab import bone, pcolor, colorbar, plot, show
bone()
#Put different winning nodes(closest) on map
#distance map gets mid of all winning nodes.

pcolor(som.distance_map().T)
colorbar()
#Therefore higher mid means white color and they are outliers(potential frauds)
markers= [ 'o', 's']
colors = [ 'r','g']
#i = indices of customers i.e., 0,1,...689
#j = 1st customer vextor then 2nd customer vector ..
for i,j in enumerate(x):
    #First will get winning node of j customer..
    w = som.winner(j)
    #On this winning node place marker on it i.e., whether this customer get approval or not
    plot(w[0]+0.5, w[1]+0.5, markers[y[i]], markeredgecolor = colors[y[i]], markerfacecolor = 'None', markersize = 10, markeredgewidth = 2)
    #0.5 to put it at center of square
    #markers[y[i]]  means if customer didn't get approval then y[i] =0 and markers[0] = o
    #markerfacecolor is inside color
show()
#Green means customer got approved
#Red means customer was not approved
#For outliers there is high risks of fraud and there are some customers who got approved and some not approved and we need to catch those who got approved

#Finding the Frauds
mapping =som.win_map(x)
#This gives all the customer associated to particular winning nodes i.e., 10 customers in (0,0) winning node
frauds = np.concatenate((mapping[(1,1)], mapping[(5,2)], mapping[(3,8)]),axis=0)#(8,1) are co-ordinates of first white box from the map
frauds = sc.inverse_transform(frauds)

#Going to Un-Supervised to Supervised to get the probability of customer to attempt fraud

#Create matrix of features that will help to predict probability of fraud
customers = dataset.iloc[:,1:].values

#Creating dependent variable i.e, 0 if no fraud and 1 if fraud
is_fraud = np.zeros(len(dataset))
for i in range(len(dataset)):
    # i is rows and 0 means customerid (means checking customerid is there in fraud)
    if dataset.iloc[i,0] in frauds:
        is_fraud[i] = 1

#Train ANN

# Feature Scaling
from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
customers = sc.fit_transform(customers)

# Part 2 - Now let's make the ANN!

# Importing the Keras libraries and packages
import keras
from keras.models import Sequential
from keras.layers import Dense

# Initialising the ANN
classifier = Sequential()

# Adding the input layer and the first hidden layer i.e., 6 hidden layers and 15 inputs
classifier.add(Dense(units = 2, kernel_initializer = 'uniform', activation = 'relu', input_dim = 15))

# Adding the output layer
classifier.add(Dense(units = 1, kernel_initializer = 'uniform', activation = 'sigmoid'))

# Compiling the ANN
classifier.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'])

# Fitting the ANN to the Training set
classifier.fit(customers, is_fraud, batch_size = 1, epochs = 2)

# Part 3 - Making predictions and evaluating the model

# Predicting the Test set results
y_pred = classifier.predict(customers)
y_pred = np.concatenate((dataset.iloc[:,0:1], y_pred),axis=1)# 1st column: customer id 2nd column probability of fraud
y_pred = y_pred[y_pred[:,1].argsort()]#Sorted predicted probabilities


from django.shortcuts import render
from django.http import HttpResponse
from .models import Order, OrderItem
from django.db import connection
from django.db.models import Sum
import operator
from django.views.decorators.csrf import csrf_exempt
from datetime import datetime
from django.db.models import Min, Max

@csrf_exempt
def period(request):

    if 'from' not in request.POST:
        date1 = Order.objects.aggregate(Min('created_date'))['created_date__min']
    else:
        date1 = datetime.strptime(request.POST['from'], "%Y-%m-%d %H:%M:%S")

    if 'to' not in request.POST:
        date2 = Order.objects.aggregate(Max('created_date'))['created_date__max']
    else:
        date2 = datetime.strptime(request.POST['to'], "%Y-%m-%d %H:%M:%S")

    orders = Order.objects.all().filter(created_date__gte=date1, created_date__lte=date2).order_by('created_date')
    result = []
    for order in orders:
        local={}
        local['created_date']=order.created_date
        local['number']=order.number
        orderItems = OrderItem.objects.filter(order_id=order.number)
        articles = ''
        sum = 0
        for orderItem in orderItems:
            articles += orderItem.product_name+'*'+str(orderItem.amount)+' '
            sum += orderItem.product_price*orderItem.amount
        local['articles'] = articles
        local['sum'] = sum
        result.append(local)
    queries = connection.queries
    dates_1 = Order.objects.values_list('created_date', flat=True)
    dates=[]
    for date in dates_1:
        dates.append(str(date)[:-6])
    context = {'orders': result, 'queries' : queries, 'dates':dates, 'date1':date1, 'date2':date2}
    return render(request, 'first_app/period.html', context)

def top100(request):

    orders = {}

    # Получение всех названий товаров
    items = OrderItem.objects.values_list('product_name', flat=True)

    # Исключение повторяющихся
    items = list(set(items))

    # Подсчет количества товаров
    for item in items:
        orders[item] = OrderItem.objects.filter(product_name=item).aggregate(Sum('amount'))['amount__sum']

    # Сортировка товаров по убыванию
    sorted_orders = sorted(orders.items(), key=operator.itemgetter(1), reverse=True)

    # Инициализация результирующего словаря
    result = []

    # Обход всех товаров по порядку
    for sorted_order in sorted_orders:

        # Инициализация словаря для одного товара
        local = {}

        # Запись названия товара в словарь товара
        local['product_name'] = sorted_order[0]

        # Определение всех номеров заказов по данному товару
        id_list = list(OrderItem.objects.filter(product_name=local['product_name']).values_list('order_id', flat=True))
        local_orders = Order.objects.filter(number__in=id_list)

        local['data'] = ''
        for local_order in local_orders:
            local_item = OrderItem.objects.get(order_id=local_order.number, product_name=item)
            local['data'] += f'Заказ {local_order.number} - Цена {local_item.product_price} - Дата {local_order.created_date}, '

        result.append(local)

    queries = connection.queries
    context = {'orders': result, 'queries' : queries}
    return render(request, 'first_app/top100.html', context)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Backend service response service class
"""
import json
import copy
from deploy.ctrl import ModuleCtrl, SessionCtrl
from deploy.utils import DeployError

MODULES = ['nlu', 'dst', 'policy', 'nlg']

class ServerCtrl(object):
    def __init__(self, **kwargs):
        self.net_conf = copy.deepcopy(kwargs['net'])
        self.module_conf = {
            'nlu': copy.deepcopy(kwargs['nlu']),
            'dst': copy.deepcopy(kwargs['dst']),
            'policy': copy.deepcopy(kwargs['policy']),
            'nlg': copy.deepcopy(kwargs['nlg'])
        }
        self.modules = {mdl: ModuleCtrl(mdl, self.module_conf[mdl]) for mdl in self.module_conf.keys()}
        self.sessions = SessionCtrl(expire_sec=self.net_conf['session_time_out'])

    def on_models(self):
        ret = {}
        for module_name in MODULES:
            ret[module_name] = {}
            for model_id in self.module_conf[module_name].keys():
                ret[module_name][model_id] = {key: self.module_conf[module_name][model_id][key] for key in
                                              ['class_path', 'data_set', 'ini_params', 'model_name']}
                ret[module_name][model_id]['ini_params'] = json.dumps(ret[module_name][model_id]['ini_params'])
        return ret

    def on_register(self, **kwargs):
        ret = {key: 0 for key in MODULES}
        try:
            for module_name in MODULES:
                model_id = kwargs.get(module_name, None)
                if isinstance(model_id, str):
                    ret[module_name] = self.modules[module_name].add_used_num(model_id)
        except Exception as e:
            for module_name in MODULES:
                model_id = kwargs.get(module_name, None)
                if isinstance(model_id, str) and ret[module_name] != 0:
                    self.modules[module_name].sub_used_num(model_id)
            raise e

        if ret['nlu'] == 0 and ret['dst'] == 0 and ret['policy'] == 0 and ret['nlg'] == 0:
            raise DeployError('At least one model needs to be started')

        token = self.sessions.new_session(*[kwargs.get(mn, None) for mn in MODULES])

        return {'token': token}

    def on_close(self, token):
        if not self.sessions.has_token(token):
            raise DeployError('No such token:\'%s\'' % token)

        session = self.sessions.pop_session(token)
        for module in MODULES:
            self.modules[module].sub_used_num(session['model_map'][module])
        return {'del_token': token}

    def on_clear_expire(self):
        expire_session = self.sessions.pop_expire_session()
        del_tokens = []
        for (token, session) in expire_session.items():
            del_tokens.append(token)
            for module in MODULES:
                self.modules[module].sub_used_num(session['model_map'][module])
        return {'del_tokens': del_tokens}

    def on_response(self, token, input_module, data, modified_output={}):
        if not self.sessions.has_token(token):
            raise DeployError('No such token:\'%s\'' % token)
        session = self.sessions.get_session(token)

        cur_turn = self._turn(last_turn=session['turns'][-1] if session['turns'] else None,
                              model_map=session['model_map'],
                              history=ServerCtrl._history_from_turns(session['turns']),
                              input_module=input_module,
                              data=data,
                              modified_output=modified_output)
        session['turns'].append(cur_turn)
        self.sessions.set_session(token, session)

        return ServerCtrl._response_from_session(session['turns'])

    def on_modify_last(self, token, modified_output):
        if not self.sessions.has_token(token):
            raise DeployError('No such token:\'%s\'' % token)
        session = self.sessions.get_session(token)

        if not session['turns']:
            raise DeployError('This is the first turn in this session.')

        last_turn = session['turns'][-1]
        session['turns'] = session['turns'][:-1]

        for (key, value) in modified_output.items():
            last_turn['modified_output'][key] = value

        cur_turn = self._turn(last_turn=session['turns'][-1] if session['turns'] else None,
                              model_map=session['model_map'],
                              history=ServerCtrl._history_from_turns(session['turns']),
                              input_module=last_turn['input_module'],
                              data=last_turn['data'],
                              modified_output=last_turn['modified_output'])
        session['turns'].append(cur_turn)
        self.sessions.set_session(token, session)

        return ServerCtrl._response_from_session(session['turns'])

    def on_rollback(self, token, back_turns=1):
        if not self.sessions.has_token(token):
            raise DeployError('No such token:\'%s\'' % token)
        session = self.sessions.get_session(token)

        session['turns'] = session['turns'][:-back_turns]
        self.sessions.set_session(token, session)

        return ServerCtrl._response_from_session(session['turns'])

    def get_module_models(self, model_map, cache):
        mods = ['nlu', 'dst', 'policy', 'nlg']
        models = []
        for mod in mods:
            if model_map[mod] is not None:
                model = self.modules[mod].models[model_map[mod]].get_model(cache[mod])
            else:
                model = None
            models.append(model)
        return models

    def _turn(self, last_turn, model_map, history, input_module, data, modified_output):
        # params
        modified_output = {mod: modified_output.get(mod, None) for mod in MODULES}
        cur_cache = last_turn['cache'] if last_turn is not None else {name: None for name in MODULES}

        # process
        new_cache = {name: None for name in MODULES}
        model_ret = {name: None for name in MODULES}
        temp_data = None

        # The following modifications to simulated PipilineAgent only functions in NL2NL and system agent mode.
        nlu_model, dst_model, policy_model, nlg_model = self.get_module_models(model_map, cur_cache)
        # if dst_model is not None:
        #     dst_model.state['history'].append(['usr', data])

        for mod in MODULES:
            if input_module == mod:
                temp_data = data

            if temp_data is not None and model_map[mod] is not None:
                # on response
                param = [temp_data]
                if mod == 'nlu' and dst_model is None:
                    context = [x[1] for x in dst_model.state['history'][:-1]]
                    param.append(context)
                # if mod == 'dst':
                #     dst_model.state['user_action'] = temp_data
                (model_ret[mod], new_cache[mod]) = self.modules[mod].run(model_map[mod],
                                                                         cur_cache[mod],
                                                                         last_turn is None,
                                                                         param,
                                                                         data if mod == 'dst' else None,
                                                                         temp_data if mod == 'dst' else None)
                # on modify last
                if modified_output[mod] is not None:
                    model_ret[mod] = modified_output[mod]

                # save intermediate result
                temp_data = model_ret[mod]
            elif mod == 'policy':
                temp_data = None

        if dst_model is not None:
            new_cache['dst']['history'].append(['sys', model_ret['nlg']])
            new_cache['dst']['system_action'] = model_ret['policy']

        # save cache
        cur_turn = {
            'data': data, 'input_module': input_module, 'modified_output': modified_output,
            'cache': new_cache,
            'context': {
                'usr': data if isinstance(data, str) and input_module == 'nlu' else '',
                'sys': model_ret['nlg'] if isinstance(model_ret['nlg'], str) else ''
            },
            'return': model_ret
        }

        return cur_turn

    @staticmethod
    def _history_from_turns(turns):
        history = []
        for turn in turns:
            history.append(['user', turn.get('context', {}).get('usr', '')])
            history.append(['system', turn.get('context', {}).get('sys', '')])
            # history.append(turn.get('context', {}).get('usr', ''))
            # history.append(turn.get('context', {}).get('sys', ''))
        return history

    @staticmethod
    def _response_from_session(turns):
        ret = {
            'nlu': turns[-1]['return']['nlu'] if turns else None,
            'dst': turns[-1]['return']['dst'] if turns else None,
            'policy': turns[-1]['return']['policy'] if turns else None,
            'nlg': turns[-1]['return']['nlg'] if turns else None,
            'modified_model': [mod for (mod, val) in turns[-1]['modified_output'].items() if val is not None] if turns else None,
            'history': ServerCtrl._history_from_turns(turns)
        }
        return ret

if __name__ == '__main__':
    pass


hour = int(input())

# Brute Force
count = 0
for h in range(hour + 1):
    for m in range(60):
        for s in range(60):
            if "3" in "".join(str(item) for item in [h, m, s]):
                count += 1
print(count)


# Вывести на экран коды и символы таблицы ASCII, начиная с символа под номером
# 32 и заканчивая 127-м включительно. Вывод выполнить в табличной форме:
# по десять пар «код-символ» в каждой строке.

row_len = 10
start_sym = 32
MAX_SYM = 128

while start_sym < MAX_SYM:
    result = ''
    if MAX_SYM - start_sym < 10:
        row_len = MAX_SYM - start_sym
    for i in range(row_len):
        result += f'№{start_sym + i}: {chr(start_sym + i)} '
    start_sym += row_len
    print(result)


import os
import requests
from bs4 import BeautifulSoup
from utils import Utils

class BaseDownloader:
    def __init__(self, min_th_version = 3, max_th_version = 15):
        self.base_url = 'https://www.clasher.us/clash-of-clans/layouts/'
        self.min_th_version = min_th_version
        self.max_th_version = max_th_version
        self.out_folder = 'layouts'
        self.create_folder(self.out_folder)
        self.current_th = None
        self.utils = Utils()

    def __format_params(self, params):
        string = []
        for each_key, value in params.items():
            string.append("{}={}".format(each_key, value))
        return "&".join(string)

    def get_request(self, url, params={"page": 94}):
        print('Getting Url {}?{}'.format(url, self.__format_params(params)))
        response = requests.get(url, params=params)
        return self.get_base_content(response.content)

    def create_folder(self, folder_path):
        if not os.path.exists(folder_path):
            os.makedirs(folder_path)

    def download_image(self, url):
        response = requests.get(url)
        if response.status_code == 200:
            with open('layouts/{}/{}'.format(self.current_th, os.path.basename(url)), 'wb') as fptr:
                fptr.write(response.content)

    def get_base_content(self, content):
        soup = BeautifulSoup(content, features="lxml")
        check = soup.find('div', {'class' : ['alert', 'alert-dismissible', 'alert-danger']})
        if check:
            return False
        images = soup.find_all('img', {'data-src' != None})
        invalid_images = 0
        image_array = []
        for image in images:
            try:
                if self.current_th in image['alt'].split(' ')[0].lower():
                    image_array.append(image['data-src'].replace('thumb', 'full'))
                else:
                    invalid_images += 1
            except:
                invalid_images += 1
        self.utils.run_parallel(self.download_image, 20, image_array)
        return (invalid_images != len(images))

    def solve_for_th(self, th_level):
        url_for_th = "{}town-hall-{}".format(self.base_url, th_level)
        params={"page": 1}
        self.current_th = 'th{}'.format(th_level)
        self.create_folder(os.path.join(self.out_folder, self.current_th))
        while self.get_request(url_for_th, params):
            params["page"] += 1

    def run_downloader(self):
        for i in range(self.min_th_version, self.max_th_version + 1):
            self.solve_for_th(i)

if __name__ == "__main__":
    base_downloader = BaseDownloader()
    base_downloader.run_downloader()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import socket
import os
import sys

sys.path.insert(0, os.path.dirname(os.getcwd()))

from test_utils import timeit as _timeit
from test_utils import separate_func as _separate_function

@_timeit
def be_client(sock):

    msg = b'This is a message for testing.. It will be repeatted'

    sock.sendall(msg)

    amount_received = 0
    amount_expected = len(msg)

    while amount_received < amount_expected:
        data = sock.recv(16)
        amount_received += len(data)
        print('received {!r}'.format(data))

if __name__ == '__main__':
    print("hello, test socket echo server")

    addr = ('localhost', 10000)

    try:
#       sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#       sock.connect(addr)
        sock = socket.create_connection(addr)

        be_client(sock)

    except socket.error as msg:
        print('socket Error: {}'.format(msg))

    finally:
        sock.close()