from flask_app.config.mysqlconnection import connectToMySQL
from flask_app.models import user
from flask_app.models import post
from flask import flash

class Comment:
    def __init__( self , data ):
        tempData = {
            'id': data['author_id']
        }
        self.id = data['id']
        self.comment = data['comment']
        self.author = user.User.get_by_id(tempData)
        tempData['id'] = data['post_id']
        self.post = post.Post.get_by_id(tempData)
        self.created_at = data['created_at']
        self.updated_at = data['updated_at']

    @classmethod
    def get_all(cls):
        query = "SELECT * FROM comments;"
        # make sure to call the connectToMySQL function with the schema you are targeting.
        results = connectToMySQL('blog').query_db(query)
        # Create an empty list to append our instances of comments
        comments = []
        # Iterate over the db results and create instances of comments with cls.
        for comment in results:
            comments.append( cls(comment) )
        return comments

    @classmethod
    def get_by_id(cls, data):
        query = "SELECT * FROM comments WHERE id = %(id)s;"
        # make sure to call the connectToMySQL function with the schema you are targeting.
        result = connectToMySQL('blog').query_db(query, data)
        if len(result) < 1 or result == False:
            return False

        return cls(result[0])

    @classmethod
    def get_by_post(cls, data):
        query = "SELECT * FROM comments WHERE post_id = %(id)s;"
        # make sure to call the connectToMySQL function with the schema you are targeting.
        results = connectToMySQL('blog').query_db(query, data)
        # Create an empty list to append our instances of comments
        comments = []
        if results == False:
            return comments

        # Iterate over the db results and create instances of comments with cls.
        for comment in results:
            comments.append( cls(comment) )
        return comments

    @classmethod
    def save(cls, data):
        query = "INSERT INTO comments ( post_id, author_id, comment, created_at, updated_at) VALUES ( %(post_id)s, %(author_id)s, %(comment)s, NOW(), NOW());"
        # data is a dictionary that will be passed into the save method from server.py
        return connectToMySQL('blog').query_db( query, data )

    @classmethod
    def delete(cls, data):
        query = "DELETE FROM comments WHERE id=%(id)s;"
        # data is a dictionary that will be passed into the save method from server.py
        return connectToMySQL('blog').query_db( query, data )


#import load_images_2_python
import pandas as pd
from collections import Counter
from sklearn.datasets import fetch_lfw_people
import matplotlib.pyplot as plt
import numpy as np

#%% Iris Data load
dataFrame_iris = pd.read_pickle("pythonDatabase")

counts = Counter(dataFrame_iris.label)
discardList = []
minNumOfImages = 10
for iris_name,value in counts.items():
    if value<=minNumOfImages:
        discardList.append(iris_name)
dataFrame_iris = dataFrame_iris[~dataFrame_iris['label'].isin(discardList)]
print("Classes ' with less than %.f images discarded in total are %.f : " % (minNumOfImages,len(discardList)),discardList)
print("Amount of classes now:", len(Counter(dataFrame_iris.label)))
temp_for_reshape = dataFrame_iris.image.values
img_dim = temp_for_reshape[1].shape
iris_imageVector = []

for i in temp_for_reshape:
    iris_imageVector.append(np.array(i.reshape(img_dim)))
iris_imageVector = np.asarray(iris_imageVector)

iris_label = dataFrame_iris.label.tolist() # The list coming from dataFrame is already in the correct format.

#%% Face data load

lfw_people = fetch_lfw_people(min_faces_per_person=10,
                              slice_ = (slice(61,189),slice(61,189)),
                              resize=0.5, color = True)

face_data = lfw_people.images
print("Original data shape: ", face_data.shape)
test_image = face_data[1]
plt.imshow(test_image.astype(np.uint8), interpolation='nearest')
plt.axis('off')

# access the class labels
face_label = lfw_people.target
print("Original label shape: ",face_label.shape)
dataFrame_face = pd.DataFrame({'image':list(face_data),'label':face_label})
#%%
unique_iris = set(iris_label)
unique_iris = list(unique_iris)
unique_face = set(face_label)
unique_face = list(unique_face)

#col_names =  ['iris', 'face', 'label']
#fusionDataframe = pd.DataFrame(columns=col_names)
fusionData = []
from matplotlib.pyplot import imshow, pause
for i in range(0,len(unique_iris)):
    current_label =  unique_iris[i]
    face_label = unique_face[i]
    #print('current_label: ',current_label)
    #print('i:',i)
    iris_temp = dataFrame_iris[dataFrame_iris['label']==unique_iris[i]]
    iris_temp = iris_temp.reset_index()
#    for iris in iris_temp.image:
#        plt.figure(figsize=[20,12])
#        plt.gca().set_title(current_label)
#        plt.imshow(iris, cmap='gray')

    face_temp =dataFrame_face[dataFrame_face['label']==unique_face[i]]
    face_temp = face_temp.reset_index()
#    for face in face_temp.image:
#        plt.figure()
#        plt.gca().set_title(face_label)
#        plt.imshow(face.astype(np.uint8))
    iris_temp_length = len(iris_temp)
    face_temp_length = len(face_temp)
    number_of_samples = min(iris_temp_length,face_temp_length)
    #print('n samples: ',number_of_samples)
    #label = np.empty(number_of_samples)
    #label.fill(i)
    for j in range(0,number_of_samples):
        #print('i that becomes class', i)
        #print('j', j)
        fusionData.append({'iris':iris_temp.image[j],'face':face_temp.image[j],'label':i,'iris_label':current_label,'face_label':face_label})
    #fusionDataframe.append(iris_temp[0:number_of_samples],face_temp[0:number_of_samples],label)

fusionDataframe = pd.DataFrame(fusionData)

'''
print("this is new")
for i in range(100,140):

    plt.figure()#figsize=[20,12])
    plt.subplot(1, 2, 1)
    plt.gca().set_title('Iris')
    plt.imshow(fusionDataframe.iris[i], cmap='gray')
    plt.subplot(1, 2, 2)
    plt.gca().set_title('Face')
    plt.imshow(fusionDataframe.face[i].astype(np.uint8))

    print(fusionDataframe.label[i], )
    #axs.imshow(test_image.astype(np.uint8), interpolation='nearest')
    #print(i)
'''

#temp = fusionDataframe.face[10]
#plt.imshow(temp.astype(np.uint8), interpolation='nearest')


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os
import re
import textwrap
import threading
import logging
import traceback
import envoy
import pyudev
import psutil
import uuid

from lxml import etree

from storeagent import store_exception
from storeagent import store_util as util
from storeagent import pools_pb2
from storeagent import store_util_pb2 as util_pb2
from storeagent import disk_manager
from storeagent import storeagentd

class DataStore(object):
    xml_template = textwrap.dedent("""
    <datastore type='%(type)s'>
        <name>%(name)s</name>
        <uuid>%(uuid)s</uuid>
        <naa>%(naa)s</naa>
        <portal>%(portal)s</portal>
        <target>%(target)s</target>
        <mountpoint>%(mountpoint)s</mountpoint>
    </datastore>
    """.strip())

    def __init__(self, uuid):
        self.__name = None
        self.__uuid = uuid
        self.__lock = threading.Lock()

    @property
    def lock(self):
        return self.__lock

    @property
    def name(self):
        return self.__name

    @name.setter
    def name(self, name):
        self.__name = name

    @property
    def uuid(self):
        return self.__uuid

    @uuid.setter
    def uuid(self, uuid):
        assert isinstance(uuid, str)
        self.__uuid = uuid

class DataStoreList(object):
    def __init__(self):
        self.__datastore_list = {}
        self.__lock = threading.Lock()

    @property
    def lock(self):
        return self.__lock

    def append(self, datastore):
        assert isinstance(datastore, DataStore)

        if not self.__lock.locked():
            raise store_exception.StoreLockException('the caller must hold DataStoreList lock')

        with datastore.lock:
            if datastore.uuid in self.__datastore_list:
                raise store_exception.StoreInvalidException('datastore(%s) already in datastore list' % datastore.uuid)
            else:
                self.__datastore_list[datastore.uuid] = datastore

    def get(self, uuid_str):
        if not self.__lock.locked():
            raise store_exception.StoreLockException('the caller must hold DataStoreList lock')

        if not (uuid_str in self.__datastore_list):
            raise store_exception.StoreInvalidException('invalid datastore uuid')

        return self.__datastore_list[uuid_str]

    def pop(self, uuid_str):
        if not self.__lock.locked():
            raise store_exception.StoreLockException('the caller must hold DataStoreList lock')

        if not (uuid_str in self.__datastore_list):
            raise store_exception.StoreInvalidException('invalid datastore uuid')

        return self.__datastore_list.pop(uuid_str)

    def exists(self, uuid_str):
        if not self.__lock.locked():
            raise store_exception.StoreLockException('the caller must hold DataStoreList lock')

        if not (uuid_str in self.__datastore_list):
            return False

        return True

    def __iter__(self):
        if not self.__lock.locked():
            raise store_exception.StoreLockException('the caller must hold DataStoreList lock')

        self.it = iter(self.__datastore_list)
        return self

    def __next__(self):
        if not self.__lock.locked():
            raise store_exception.StoreLockException('the caller must hold DataStoreList lock')

        return self.__datastore_list[next(self.it)]

DATASTORE_LIST = DataStoreList()

def CreateStoreCfg(xml_str, name):
    """ create Datastore conf xml
    :param request : xml_str
                     name the Datastore name
    :return : True create success
              False create fail
    """
    try:
        el = etree.fromstring(xml_str)
        tree = etree.ElementTree(el)
        tree.write('%s/%s.xml' % (util.STORE_XML_PATH, name), pretty_print=True,
                   xml_declaration=True, encoding='utf-8')
        return True
    except store_exception.StoreXmlException:
        logging.critical(traceback.format_exc())
        return False

def ReadXml(xml_file):
    """ read xml file
    :param request : xml file path
    :returns : etree._ElementTree
    """
    try:
        return etree.parse(xml_file)
    except OSError:
        logging.error(traceback.format_exc())
        raise store_exception.StoreOsErrorException(xml_file)
    except etree.LxmlError:
        logging.critical(traceback.format_exc())
        raise store_exception.StoreXmlException(xml_file)

def GetXmlElementByXpath(xml_tree, xpath):
    """ get Element from ElementTree by xpath
    :param request : xml tree
    :returns : etree._Element
    """
    try:
        return xml_tree.find(xpath)
    except OSError:
        logging.error(traceback.format_exc())
        raise store_exception.StoreOsErrorException(xml_tree)
    except etree.LxmlError:
        logging.critical(traceback.format_exc())
        raise store_exception.StoreXmlException(xml_tree)

def GetDeviceSizeInfo(device):
    """ get device size info
    :param request :device
    :returns : DataStoreSizeInfo {
                   totalsize
                   allocatedsize
                   availablesize
               }
    """
    SizeInfo = pools_pb2.DataStoreSizeInfo()

    total = envoy.run('%s %s  --output=size' % (util.DF, device))
    if total.status_code == 0:
        SizeInfo.totalsize = total.std_out.strip().split('\n')[1].lstrip()
    else:
        raise store_exception.StoreEnvoyException(total)

    allocation = envoy.run('%s %s --output=used' % (util.DF, device))
    if allocation.status_code == 0:
        SizeInfo.allocatedsize = allocation.std_out.strip().split('\n')[1].lstrip()
    else:
        raise store_exception.StoreEnvoyException(allocation)

    avail = envoy.run('%s %s  --output=avail' % (util.DF, device))
    if avail.status_code == 0:
        SizeInfo.availablesize = avail.std_out.strip().split('\n')[1].lstrip()
    else:
        raise store_exception.StoreEnvoyException(avail)

    return SizeInfo

def GetDeviceSizeInfoByName(store_name):
    """ get device size info by store_name
    :param request : store_name
    :returns : DataStoreSizeInfo {
                   totalsize
                   allocatedsize
                   availablesize
               }
    """
    SizeInfo = pools_pb2.DataStoreSizeInfo()
    s_name = store_name

    xml_file = ('%s/%s.xml' % (util.STORE_XML_PATH, s_name))
    xml_tree = ReadXml(xml_file)
    s_naa = GetXmlElementByXpath(xml_tree, 'naa').text
    s_device = GetDeviceByNaa(s_naa)
    SizeInfo = GetDeviceSizeInfo(s_device)

    return SizeInfo

def GetDataStoreInfoByName(store_name):
    '''
    get DataStore info
    request : store_name
    return:  pools_pb2. DataStoreInfo()
    '''
    _store_info = pools_pb2.DataStoreInfo()
    _store_name = store_name
    _size_info = pools_pb2.DataStoreSizeInfo()

    # check xml conf
    xml_file = ('%s/%s.xml' % (util.STORE_XML_PATH, _store_name))
    if os.path.exists(xml_file):
        logging.info('xml_file is %s.' % xml_file)
    else:
        raise store_exception.StoreInvalidException(_store_name)

    # read xml
    xml_tree = ReadXml(xml_file)
    if isinstance(xml_tree, etree._ElementTree):
        # get info
        _s_type = GetXmlElementByXpath(xml_tree, '[@type]').get('type')
        if not _s_type:
            raise store_exception.StoreXmlException(xml_tree)

        _s_naa = GetXmlElementByXpath(xml_tree, 'naa').text
        # block pool naa is none.
        if int(_s_type) != pools_pb2.ISCSI_BLOCK and int(_s_type) != pools_pb2.FC_BLOCK:
            if not _s_naa:
                raise store_exception.StoreXmlException(xml_tree)

        _s_portal = GetXmlElementByXpath(xml_tree, 'portal').text
        _s_target = GetXmlElementByXpath(xml_tree, 'target').text

        _s_mountpoint = GetXmlElementByXpath(xml_tree, 'mountpoint').text
        if not _s_mountpoint:
            raise store_exception.StoreXmlException(xml_tree)
    else:
        raise store_exception.StoreXmlException('read xml fail')

    #  get size info
    if int(_s_type) != pools_pb2.ISCSI_BLOCK and int(_s_type) != pools_pb2.FC_BLOCK:
        if store_name == util.LOCALPOOL:
            with storeagentd.DATASTORE_CONF.lock:
                _s_device = storeagentd.DATASTORE_CONF.GetValue(util.DEVICE, store_name)
        else:
            _s_device = GetDeviceByNaa(_s_naa)

        _size_info = GetDeviceSizeInfo(_s_device)
    else:
        # get block pool size
        _lun_list = []
        _lun_list = disk_manager.GetIscsiLunListByTarget(_s_portal, _s_target)
        for _lun in _lun_list:
            _size = 0
            _size = _lun['Size']
            _size_info.totalsize += _size
            _size_info.allocatedsize += _size
        _size_info.availablesize = 0

    # get state
    with storeagentd.DATASTORE_CONF.lock:
        _s_state = storeagentd.DATASTORE_CONF.GetValue(util.STATE, store_name)
        _s_title = storeagentd.DATASTORE_CONF.GetValue(util.N_TITLE, store_name)

    _store_info.name = _store_name
    _store_info.p_type = int(_s_type)
    _store_info.naa = _s_naa
    _store_info.portal = _s_portal
    _store_info.target = _s_target
    _store_info.totalsize = _size_info.totalsize
    _store_info.allocatedsize = _size_info.allocatedsize
    _store_info.availablesize = _size_info.availablesize
    _store_info.state = str(_s_state)
    _store_info.n_title = str(_s_title)
    _store_info.mountpoint = _s_mountpoint

    return _store_info

def StringToInt(string):
    """ string   to int
    :param request : string
    :returns : int default (KB)
    """
    ret = ''

    num_re = re.match(r'\d+', string)
    if None is num_re:
        logging.error('invalid string %s' % string)
        raise store_exception.StoreInvalidException(num_re)

    num = int(num_re.group())

    unit_re = re.search(r'[K,M,G,T]', string)
    if None is unit_re:
        unit = ''
    else:
        unit = unit_re.group()

    if unit == '':
        ret = num//1024

    if unit == 'K':
        ret = num

    if unit == 'M':
        ret = num*1024

    if unit == 'G':
        ret = num*1024*1024

    if unit == 'T':
        ret = num*1024*1024*1024

    return ret

def QemuCreateVol(json_data):
    """ create a vol with json_data
    :param request : json_data
    :returns : util_pb2.STORE_OK
    """
    s_sizeinfo = pools_pb2.DataStoreSizeInfo()
    q_img_ret = util_pb2.QemuImgReply()

    # load json_data
    j_data = {}
    j_data = json_data
    logging.info('the json_data is %s' % j_data)

    p_name = j_data['p_name']
    vol_path = j_data['vol_path']
    vol_type = j_data['vol_type']
    vol_size = int(j_data['vol_size'])
    preallocation = j_data['preallocation']

    # check filepath
    filepath = ('%s/%s/%s' % (util.STORE_MOUNT_PATH, p_name, vol_path))
    if os.path.exists(filepath):
        logging.error('the %s is already exists' % filepath)
        raise store_exception.StoreInvalidException(filepath)

    # check size
    s_sizeinfo = GetDeviceSizeInfoByName(p_name)
    avail_size = StringToInt(s_sizeinfo.availablesize)
    if avail_size < vol_size:
        logging.error('the DataStore not enough space. avail_size%(KB) vol_size%s(KB)'
                      % (avail_size, vol_size))
        raise store_exception.StoreInvalidException(vol_size)

    # command
    if vol_type == 'raw':
        command = ('create -q -f raw -o preallocation=%s %s %sK' %
                   (preallocation, filepath, vol_size))

    if vol_type == 'qcow2':
        compat = j_data['compat']
        backing_file = j_data['backing_file']
        backing_fmt = j_data['backing_fmt']
        encryption = j_data['encryption']
        cluster_size = int(j_data['cluster_size'])
        lazy_refcounts = j_data['lazy_refcounts']

        # check cluster_size(512 -- 2M)
        if cluster_size < 0.5 or cluster_size > 4096:
            logging.error('cluster_size is %s.' % cluster_size)
            raise store_exception.StoreInvalidException(cluster_size)

        if backing_file == 'null':
            if not backing_fmt == 'null':
                raise store_exception.StoreInvalidException(backing_fmt)
            command = ('create -q -f qcow2'
                       ' -o compat=%s'
                       ' -o cluster_size=%dK'
                       ' -o encryption=%s'
                       ' -o lazy_refcounts=%s'
                       ' -o preallocation=%s %s %s' % (compat,
                                                       cluster_size,
                                                       encryption,
                                                       lazy_refcounts,
                                                       preallocation,
                                                       filepath,
                                                       vol_size))
        else:
            if not os.path.exists(backing_file):
                raise store_exception.StoreInvalidException(backing_file)

            # check backing_file busy
            if util.CheckBusy(backing_file):
                raise store_exception.StoreFileBusyException(backing_file)

            command = ('create -q -f qcow2'
                       ' -o compat=%s'
                       ' -o backing_file=%s'
                       ' -o backing_fmt=%s'
                       ' -o cluster_size=%dK'
                       ' -o encryption=%s'
                       ' -o lazy_refcounts=%s'
                       ' -o preallocation=%s %s %s' % (compat,
                                                       backing_file,
                                                       backing_fmt,
                                                       cluster_size,
                                                       encryption,
                                                       lazy_refcounts,
                                                       preallocation,
                                                       filepath,
                                                       vol_size))

    # create
    logging.info('command is %s.' % command)
    q_img_ret = util.QemuImg(command)

    return q_img_ret.errno

def QemuCreateFullVol(opaque, job=None):
    """ create full vol
    """
    json_data = opaque['json_data']

    QemuCreateVol(json_data)

def GetLocalStoreSizeInfo():
    '''
    get LocalStore size info.
    return : totalsize  KB
             usedsize   KB
    '''
    _list_device = []
    _s_totalsize = 0
    _s_usedsize = 0

    _list_device = disk_manager.GetLocalSCSIResources()

    for item in _list_device:
        used = 0
        total = 0
        device = item['device']

        # device is cdrom skip
        device_type = disk_manager.GetDeviceAttribute(disk_manager.DiskUdevAttribute.ID_TYPE, ('/dev/' + device))
        if device_type == 'cd':
            continue

        # On virtual machine, _list_device  include system disk, need skip.
        std_out, std_err = util.StoreCmdRun('%s %s -o TYPE' % (util.LSBLK, ('/dev/' + device)))
        if re.search(r'part', std_out.strip()):
            continue

        std_out2, std_err2 = util.StoreCmdRun('%s %s --output=used' % (util.DF, ('/dev/' + device)))
        used = int(std_out2.strip().split('\n')[1].lstrip())
        total = int(item['Size'])

        _s_totalsize += total
        _s_usedsize += used

    _s_totalsize = int(_s_totalsize)/1024
    logging.info("GetLocalStoreSizeInfo  totalsize is %s, usedsize is %s" % (_s_totalsize, _s_usedsize))

    return str(_s_totalsize), str(_s_usedsize)

def GetDataStoreFileList(store_name):
    '''
    get DataStore file list
    '''
    _file_list = []

    xml_file = ('%s/%s.xml' % (util.STORE_XML_PATH, store_name))
    xml_tree = ReadXml(xml_file)
    _s_mountpoint = GetXmlElementByXpath(xml_tree, 'mountpoint').text

    _file_list = os.listdir(_s_mountpoint)

    return _file_list

def GetDataStoreNameList():
    '''
    get all DataStore name
    '''

    _name_list = os.listdir(util.STORE_MOUNT_PATH)

    return _name_list

def CheckPortalNeedLogout(portal, target_name):
    '''
        check one portal(x.x.x.x:port) if need logout. if one pool used the portal target_name, we don't logout.
        return True : logout
               False : not logout
    '''
    ret = False

    _xml_list = os.listdir(util.STORE_XML_PATH)
    for _xml in _xml_list:
        xmlfile = ''
        xmlfile = ('%s/%s' % (util.STORE_XML_PATH, _xml))
        _xml_tree = ReadXml(xmlfile)
        if isinstance(_xml_tree, etree._ElementTree):
            _type = GetXmlElementByXpath(_xml_tree, '[@type]').get('type')
            if int(_type) == pools_pb2.LOCAL:
                continue
            _str_portal = GetXmlElementByXpath(_xml_tree, 'portal').text
            _target = GetXmlElementByXpath(_xml_tree, 'target').text
            _portal_list = _str_portal.split(',')
            for _portal in _portal_list:
                if _portal == portal and _target == target_name:
                    ret = True
                    break
            if ret:
                break

    return ret

def GetMountpoint(device):
    '''
        get the device mountpoint, if not mountpoint, return null
    '''
    mountpoint = ''
    disks = psutil.disk_partitions()

    for disk in disks:
        if disk.device == device:
            mountpoint = disk.mountpoint
            break

    return mountpoint

def GetDeviceByNaa(naa):
    '''
        get device by naa,  localpool used GetDeviceByMountPoint, or datastore.cfg
    '''
    device = ''

    context = pyudev.Context()
    for dev in context.list_devices(subsystem='block', DEVTYPE='disk'):
        _naa = dev.get('ID_SERIAL')
        if _naa == naa:
            device = dev.get('DEVNAME')
            break

    return device

def CreateLocalStore(naa, name, n_title):
    '''
        create local Datastore
    '''
    global DATASTORE_LIST

    store_uuid = str(uuid.uuid3(uuid.NAMESPACE_DNS, '%s' % name))
    with DATASTORE_LIST.lock:
        if DATASTORE_LIST.exists(store_uuid):
            logging.error('the name %s is already uesd.' % name)
            raise store_exception.StoreInvalidException(name)
    _store_node = DataStore(store_uuid)

    # check xml
    xmlfile = ('%s/%s.xml' % (util.STORE_XML_PATH, name))
    if os.path.exists(xmlfile):
        raise store_exception.StoreInvalidException(xmlfile)

    # check device
    if name != util.LOCALPOOL:
        device = GetDeviceByNaa(naa)
    else:
        with storeagentd.DATASTORE_CONF.lock:
            device = storeagentd.DATASTORE_CONF.GetValue(util.DEVICE, util.LOCALPOOL)

    if not os.path.exists(device):
        logging.info('the device %s is not exists.' % device)
        raise store_exception.StoreInvalidException(device)

    _mountpoint = GetMountpoint(device)
    if len(_mountpoint):
        logging.info('the device %s is already mountting.' % device)
        raise store_exception.StoreInvalidException(device)
    _mountpoint = os.path.join(util.STORE_MOUNT_PATH, name)

    # CreateStorecfg
    xmlstr = _store_node.xml_template % {
            'type': pools_pb2.LOCAL,
            'name': name,
            'uuid': store_uuid,
            'naa': naa,
            'portal': None,
            'target': None,
            'mountpoint': _mountpoint
            }

    if CreateStoreCfg(xmlstr, name):
        logging.info('Create %s xml success.' % name)
    else:
        raise store_exception.StoreXmlException('Create %s xml failed.' % name)

    # add datastore to DATASTORE_LIST
    with DATASTORE_LIST.lock:
        DATASTORE_LIST.append(_store_node)

    # add section on  datastore.cfg
    with storeagentd.DATASTORE_CONF.lock:
        try:
            storeagentd.DATASTORE_CONF.SetValue(util.STATE, util.INACTIVE, name)
            storeagentd.DATASTORE_CONF.SetValue(util.N_TITLE, n_title, name)
            storeagentd.DATASTORE_CONF.SetValue(util.AUTO_MOUNT, util.TRUE, name)
        except store_exception.StoreDataConfigException:
            if name != util.LOCALPOOL:
                os.remove(xmlfile)
                with DATASTORE_LIST.lock:
                    DATASTORE_LIST.pop(store_uuid)
            raise store_exception.StoreDataConfigException('%s add status failed.' % name)

    logging.info('create datastore %s success.' % name)

def DeleteLocalStore(name):
    '''
        delete local Datastore
    '''
    # check uuid
    store_uuid = str(uuid.uuid3(uuid.NAMESPACE_DNS, '%s' % name))

    with DATASTORE_LIST.lock:
        if not DATASTORE_LIST.exists(store_uuid):
            logging.info('the %s datastore is not exist.' % name)
            raise store_exception.StoreInvalidException(name)

        DATASTORE_LIST.pop(store_uuid)

    # check xml cfg
    xmlfile = ('%s/%s.xml' % (util.STORE_XML_PATH, name))
    if not os.path.exists(xmlfile):
        logging.error('the %s xml conf is not exist.' % name)
        raise store_exception.StoreInvalidException(name)

    # delete datastore.cfg datastore
    with storeagentd.DATASTORE_CONF.lock:
        if name != util.LOCALPOOL:
            storeagentd.DATASTORE_CONF.DeleteStore(name)

    # delete xml cfg
    os.remove(xmlfile)

    logging.info('Delete %s datastore success.' % name)

def StartLocalStore(name):
    '''
        start local store (include  localpool)
    '''
    # check xml cfg
    xmlfile = ('%s/%s.xml' % (util.STORE_XML_PATH, name))
    if not os.path.exists(xmlfile):
        raise store_exception.StoreInvalidException(xmlfile)

    # check  mountpoint, device
    xml_tree = ReadXml(xmlfile)
    if isinstance(xml_tree, etree._ElementTree):
        mountpoint = GetXmlElementByXpath(xml_tree, 'mountpoint').text
        naa = GetXmlElementByXpath(xml_tree, 'naa').text
        device = GetDeviceByNaa(naa)
        logging.info('mountpoint is %s, device is %s' % (mountpoint, device))
        # check mountpoint /data/localpool.
        if mountpoint == util.LOCALPOOL_MOUNTPOINT:
            # get device by datastore.cfg.
            device = storeagentd.DATASTORE_CONF.GetValue(util.DEVICE, name)
    else:
        raise store_exception.StoreXmlException(xmlfile)

    # check auto_mount
    with storeagentd.DATASTORE_CONF.lock:
        _auto_mount = storeagentd.DATASTORE_CONF.GetValue(util.AUTO_MOUNT, name)
        if _auto_mount != util.TRUE:
            if not os.path.exists(mountpoint):
                os.mkdir(mountpoint)

            if util.CheckBusy(mountpoint):
                raise store_exception.StoreFileBusyException(mountpoint)

            if os.listdir(mountpoint):
                logging.error('mountpoint dir %s is not null.' % mountpoint)
                raise store_exception.StoreInvalidException(mountpoint)

            mount_ret = envoy.run('%s %s %s' % (util.MOUNT, device, mountpoint))
            if mount_ret.status_code == 0:
                logging.info('the datastore %s mountting success.' % name)
            else:
                raise store_exception.StoreEnvoyException('mount  %s %s failed.' % (device, mountpoint))

    # update status cfg
    with storeagentd.DATASTORE_CONF.lock:
        storeagentd.DATASTORE_CONF.SetValue(util.STATE, util.ACTIVE, name)

    logging.info('Start datastore %s success.' % name)

def StopLocalStore(name):
    '''
        stop local store
    '''
    # check xml cfg
    xmlfile = ('%s/%s.xml' % (util.STORE_XML_PATH, name))
    if not os.path.exists(xmlfile):
        raise store_exception.StoreInvalidException(xmlfile)

    # check mountpoint device
    xml_tree = ReadXml(xmlfile)
    if isinstance(xml_tree, etree._ElementTree):
        mountpoint = GetXmlElementByXpath(xml_tree, 'mountpoint').text
        if not mountpoint:
            raise store_exception.StoreInvalidException('mountpoint is null')

        # check mountpoint busy
        if util.CheckBusy(mountpoint):
            raise store_exception.StoreFileBusyException(mountpoint)

        # umount
        with storeagentd.DATASTORE_CONF.lock:
            _auto_mount = storeagentd.DATASTORE_CONF.GetValue(util.AUTO_MOUNT, name)
            if _auto_mount != util.TRUE:
                umount_ret = envoy.run('%s %s' % (util.UMOUNT, mountpoint))
                if umount_ret.status_code == 0:
                    logging.info('datastore %s umount %s success.' % (name, mountpoint))
                else:
                    raise store_exception.StoreEnvoyException('%s umount %s failed' % (name, mountpoint))

        # update datastroe.cfg
        with storeagentd.DATASTORE_CONF.lock:
            storeagentd.DATASTORE_CONF.SetValue(util.STATE, util.INACTIVE, name)
    else:
        raise store_exception.StoreXmlException('%s.xml exception' % name)

    logging.info('datastore %s is stop success.' % name)

def CreateBlockStore(name, portal, target, n_title):
    '''
        create Block DataStore
    '''
    _xml_list = []

    # check uuid
    store_uuid = str(uuid.uuid3(uuid.NAMESPACE_DNS, name))
    with DATASTORE_LIST.lock:
        if DATASTORE_LIST.exists(store_uuid):
            logging.error('the name %s is already used.' % name)
            raise store_exception.StoreInvalidException(name)
    _store_node = DataStore(store_uuid)

    # check xml cfg
    xmlfile = ('%s/%s.xml' % (util.STORE_XML_PATH, name))
    if os.path.exists(xmlfile):
        logging.error('the xml %s is already exists.' % xmlfile)
        raise store_exception.StoreInvalidException(name)

    # check device by portal, target
    # get xmlfile  list
    _xml_list = os.listdir(util.STORE_XML_PATH)
    for xml in _xml_list:
        _file = ''
        _file = ('%s/%s' % (util.STORE_XML_PATH, xml))
        _xml_tree = ReadXml(_file)
        if isinstance(_xml_tree, etree._ElementTree):
            _type = GetXmlElementByXpath(_xml_tree, '[@type]').get('type')
            if int(_type) == pools_pb2.LOCAL:
                continue
            _portal = GetXmlElementByXpath(_xml_tree, 'portal').text
            _target = GetXmlElementByXpath(_xml_tree, 'target').text
            if _portal == portal and _target == target:
                logging.error('the Datastroe %s is confict.' % xml)
                raise store_exception.StoreInvalidException('the portal %s, target %s is invalid.' % (portal, target))
        else:
            raise store_exception.StoreInvalidException(_file)

    # createxmlcfg
    _xmlstr = _store_node.xml_template % {
             'type': pools_pb2.ISCSI_BLOCK,
             'name': name,
             'uuid': store_uuid,
             'naa': None,
             'portal': portal,
             'target': target,
             'mountpoint': util.BLOCK_PATH
            }
    if CreateStoreCfg(_xmlstr, name):
        logging.info('Create %s xml success.' % name)
    else:
        raise store_exception.StoreXmlException('Create %s xml failed.' % name)

    # add to DATASTORE_LIST
    with DATASTORE_LIST.lock:
        DATASTORE_LIST.append(store_uuid)

    # add to DATASTORE_CONF
    with storeagentd.DATASTORE_CONF.lock:
        try:
            storeagentd.DATASTORE_CONF.SetValue(util.STATE, util.INACTIVE, name)
            storeagentd.DATASTORE_CONF.SetValue(util.STATE, util.N_TITLE, n_title)
        except store_exception.StoreDataConfigException:
            os.remove(xmlfile)
            with DATASTORE_LIST.lock:
                DATASTORE_LIST.pop(store_uuid)
            raise store_exception.StoreDataConfigException('%s update datastore.cfg failed.' % name)

    logging.info('create datasotre %s success.' % name)

def DeleteBlockStore(name):
    '''
        delete Block DataStore
    '''
    _state = ''

    xmlfile = ('%s/%s.xml' % (util.STORE_XML_PATH, name))
    if not os.path.exists(xmlfile):
        raise store_exception.StoreInvalidException(xmlfile)

    # get Store state, if inactive delete
    with storeagentd.DATASTORE_CONF.lock:
        _state = storeagentd.DATASTORE_CONF.GetValue(util.STATE, name)

    if _state == util.INACTIVE:
        # delete from DATASTORE_LIST
        store_uuid = uuid.uuid3(uuid.NAMESPACE_DNS, '%s' % name)
        with DATASTORE_LIST.lock:
            if not DATASTORE_LIST.exists(store_uuid):
                logging.error('the %s datastore is not exists.' % name)
                raise store_exception.StoreInvalidException(name)
            DATASTORE_LIST.pop(store_uuid)

        # delete from DATASTORE_CONF
        with storeagentd.DATASTORE_CONF.lock:
            storeagentd.DATASTORE_CONF.DeleteStore(name)

        # delete xml cfg
        os.remove(xmlfile)
    else:
        logging.info('the %s datastore is ACTIVE. please stop datastore.')
        raise store_exception.StoreInvalidException('the datastore is ACTIVE.')

    logging.info('delete datastore %s success.' % name)

def StartBlockStore(name):
    '''
        start block DataStore
    '''
    # check xml
    xmlfile = ('%s/%s.xml' % (util.STORE_XML_PATH, name))
    if not os.path.exists(xmlfile):
        raise store_exception.StoreInvalidException(xmlfile)

    # get portal and target
    _xml_tree = ReadXml(xmlfile)
    if isinstance(_xml_tree, etree._ElementTree):
        _portal = GetXmlElementByXpath(_xml_tree, 'portal').text
        _target = GetXmlElementByXpath(_xml_tree, 'target').text

        # login, if already login we again login has no affect.
        _portal_list = _portal.split(',')
        for item in _portal_list:
            _portal_address = item.split(':')[0]
            _portal_port = item.split(':')[1]
            util.IscsiadmLogin('%s:%s' % ((_portal_address, _portal_port), _target))

    # change DATASTORE_CONF
    with storeagentd.DATASTORE_CONF.lock:
        storeagentd.DATASTORE_CONF.SetValue(util.STATE, util.ACTIVE, name)

    logging.info('start %s datastore success.' % name)

def StopBlockStore(name):
    '''
        stop block DataStore
    '''
    lun_list = []
    _portal_list = []
    # check xml
    xmlfile = ('%s/%s.xml' % (util.STORE_XML_PATH, name))
    if not os.path.exists(xmlfile):
        raise store_exception.StoreInvalidException(name)

    # get portal and target
    _xml_tree = ReadXml(xmlfile)
    if isinstance(_xml_tree, etree._ElementTree):
        _portal = GetXmlElementByXpath(_xml_tree, 'portal').text
        _target = GetXmlElementByXpath(_xml_tree, 'target').text

    # check portal target, get lun by target. check lun if is busy.
    lun_list = disk_manager.GetIscsiLunListByTarget(_portal, _target)
    for lun in lun_list:
        # check busy
        _naa = lun['SCSI_ID']
        _device = GetDeviceByNaa(_naa)
        if util.CheckBusy(_device):
            logging.error('the device %s scsi_id %s is busy.' % (_device, _naa))
            raise store_exception.StoreFileBusyException(_device)

    # logout
    _portal_list = _portal.split(',')
    for item in _portal_list:
        _portal_address = item.split(':')[0]
        _portal_port = item.split(':')[1]
        util.IscsiadmLogout('%s:%s' % (_portal_address, _portal_port), _target)

    # change state to inactive, DATASTORE_CONF.
    with storeagentd.DATASTORE_CONF.lock:
        storeagentd.DATASTORE_CONF.SetValue(util.STAT, util.INACTIVE, name)

    logging.info('stop %s datastore success.' % name)


#!/bin/python3

def most_frequent(List):
    return max(set(List), key = List.count)

def findLeader(inputString):
    inputString = sorted(inputString)
    numbers = {}
    for number in inputString:
        if number in numbers:
            numbers[number] = numbers[number] + 1
        else:
            numbers[number] = 1

    numbers = sorted(numbers.items(), key=lambda x: x[1], reverse=True)
    return numbers

def printFirstTree(resultList):
    print(str(resultList[0][0]) + " " + str(resultList[0][1]))
    print(str(resultList[1][0]) + " " + str(resultList[1][1]))
    print(str(resultList[2][0]) + " " +  str(resultList[2][1]))

if __name__ == '__main__':

    arr = list(map(int, input().rstrip().split()))

    if 3 < len(arr) <= 10**4:
        resultList = findLeader(arr)

    printFirstTree(resultList)


import os
from urllib import request
import requests
import json
startpos = 0
urlsTotal = []
def main():
    term = input('Search term: \n').rstrip()
    startpos = 0
    z = recursiveGrab(term)
    if not z:
        print('No more links located upon search execution.')
    try:
        for x in z:
            print(x)
    except Exception:
        print(Exception)

def grablinks(term, startpos):
    ##returns a list of links on the page
    urlsTotal2 = []
    url = "https://google.com/search?q=" + term + '&start=' + str(startpos)
    resp = requests.get(url)
    txt = ''
    if resp.ok:
        txt = resp.text
    else:
        print('Error gathering')
        quit()

    if len(txt) > 0:
        urls = txt.split('url?q=')
        for x in urls:
            url2 = x.split('&amp')[0]
            if url2.startswith('https') and url2.find('google.') == -1:
                if url2 not in urlsTotal:
                    print(url2)
                    urlsTotal.append(url2)

                urlsTotal2.append(url2)
        return(urlsTotal2)

def recursiveGrab(term):
    global startpos
    global urlsTotal

    a = grablinks(term, startpos)
    if len(a) > 0:
        for x in a:
            if x not in urlsTotal:
                urlsTotal.append(x)

        startpos += 10
        recursiveGrab(term)
    else:
        return(urlsTotal)

if __name__ == '__main__':
    main()


from json import dump
from sys import argv
import sys

filename = 'hamlet.txt'
outfile = 'hamlet_10.txt'

with open(filename, 'rb') as fin:
        x = fin.read()

y = x*10

with open(outfile, 'wb') as fout:
    fout.write(y)


from d02 import parse_puzzle, solve_part_1, solve_part_2
from typing import List, Tuple

example_puzzle: str = """A Y
B X
C Z
"""

def test_puzzle_parser():
    """Just ensure our generator-baed parser works"""
    # we have to convert to a list to get the length
    throws: List[Tuple[int]] = list(parse_puzzle(example_puzzle.splitlines()))
    assert len(throws) == 3

def test_part_1_example():
    """Solve"""
    throws: List[Tuple[int]] = (parse_puzzle(example_puzzle.splitlines()))
    score = solve_part_1(throws)
    assert score == 15

def test_part_2_example():
    """Score by given outcome"""
    throws: List[Tuple[int]] = (parse_puzzle(example_puzzle.splitlines()))
    score = solve_part_2(throws)
    assert score == 12


from bs4 import BeautifulSoup
import requests
import os
import spotipy
from spotipy.oauth2 import SpotifyOAuth

sp = spotipy.Spotify(auth_manager=SpotifyOAuth(client_id=os.environ["CLIENT_ID"],
                                               client_secret=os.environ["CLIENT_SECRET"],
                                               redirect_uri="https://deveshnice2.com",
                                               scope="playlist-modify-private",
                                               show_dialog=True,
                                               cache_path=".cache"))

current_user_id = sp.current_user()["id"]
print(current_user_id)

ask_time = input("What year you would like to travel to? Type "
                 "the date in YYYY-MM-DD format")

response = requests.get(url=f"https://www.billboard.com/charts/hot-100/{ask_time}/")
data = response.text

soup = BeautifulSoup(data, 'html.parser')

songs_title = soup.find_all(name='h3', class_='a-no-trucate', id='title-of-a-story')

song_list = [song.getText().split('\n')[1] for song in songs_title]
year = ask_time.split("-")[0]

songs_uri_list = []
for song in song_list:
    results = sp.search(q=f"track: {song} year: {year}", type="track")

    try:
        song_uri = results["tracks"]["items"][0]["uri"]
        songs_uri_list.append(song_uri)
    except IndexError:
        print(f"{song} doesn't exist in Spoitfy. Skipped")

playlist = sp.user_playlist_create(user=current_user_id, name=f"{ask_time}BillBoard100", public=False)
print(playlist["id"])

sp.playlist_add_items(playlist_id=playlist["id"], items=songs_uri_list)


import random
import time
import keyboard
import pyautogui
from Xlib import X, display
import Xlib.XK
import Xlib.ext.xtest

times = 19.4

def keyS():
    pyautogui.mouseDown(button='left')
    time.sleep(0.1)
    pyautogui.keyDown('s')

def keyA():
    pyautogui.mouseDown(button='left')
    time.sleep(0.1)
    pyautogui.keyDown('a')

def keyW():
    time.sleep(0.01)
    pyautogui.keyDown('w')
    time.sleep(0.05)
    pyautogui.keyUp('w')
    time.sleep(0.01)
    pyautogui.keyDown('s')
    time.sleep(0.055)
    pyautogui.keyUp('s')
    time.sleep(0.05)

def resetup():
    pyautogui.mouseUp(button='left')
    pyautogui.keyUp('a')
    pyautogui.keyUp('s')

time.sleep(2)
disp = display.Display()
root = disp.screen().root
win_id = None
while not win_id:
    win_id = root.get_full_property(disp.intern_atom('_NET_ACTIVE_WINDOW'), X.AnyPropertyType).value[0]
window = disp.create_resource_object('window', win_id)
window.set_input_focus(X.RevertToParent, X.CurrentTime)
time.sleep(1)
pyautogui.typewrite(['esc'])

while not(keyboard.is_pressed('ctrl')):
    while not(keyboard.is_pressed('ctrl')):
        keyS()
        time.sleep(times)
        resetup()
        keyA()
        time.sleep(times)
        resetup()
    pyautogui.mouseUp(button='left')
    while not(keyboard.is_pressed('=')):
        print("waiting")


import logging
import sys
import requests
from threading import Thread

from core import AppConfig, ProcessHandler, Webhook
from model import Game, WebhookData

handler = logging.StreamHandler(sys.stdout)
logging.basicConfig(level=logging.INFO, handlers=[handler])
logger = logging.getLogger(__name__)

config = AppConfig.getInstance()

class AlreadyRunningException(Exception): pass
class NotRunningException(Exception): pass

class ServerService:
	_instance = None

	def __init__(self):
		self._thread : ProcessHandler = None

	@classmethod
	def getInstance(cls):
		if(cls._instance is None):
			cls._instance = ServerService()
		return cls._instance

	def is_running(self):
		if(self._thread == None):
			return False

		if(self._thread.proc == None):
			return False

		# check if process has terminated
		poll = self._thread.proc.poll()
		if(poll == None):
			return True
		else:
			return False

	def start(self, game: Game, callback_url=None):
		if(self.is_running()):
			raise AlreadyRunningException()

		logger.info(f"Starting the server... workingDirectory: '{game.path}', command: '{game.command}'")

		self._thread = ProcessHandler(
			cmd=game.command.split(),
			cwd=game.path,
			ready_log=game.ready_log,
			start_timeout=game.timeout,
		)

		# Sets the webhook events if present in the request
		if(callback_url is not None):
			self._thread.on_ready_events.append(lambda : Webhook.send(callback_url, WebhookData("Server is ready.")))
			self._thread.on_exit_events.append(lambda : Webhook.send(callback_url, WebhookData("Server has stopped.")))
			self._thread.on_reminder_events.append(lambda x: Webhook.send(callback_url, WebhookData(f"Reminder, the server will stop in {x} seconds.")))
			self._thread.timer_start_timeout.events.append(lambda : Webhook.send(callback_url, WebhookData("Server failed to start, timeout reached.")))

		self._thread.start()

	def stop(self):
		if(not self.is_running()):
			logger.info("Tried to stop the thread but no thread was running.")
			raise NotRunningException()

		self._thread.stop()

	def reset_timeout(self):
		if(not self.is_running()):
			logger.info("Tried to reset timeout but no thread was running.")
			raise NotRunningException()

		self._thread.reset_server_timeout()


from django.shortcuts import redirect, render
from dashboard.navforms import *
from .models import Sectionsothernavs as Sections
from .forms import section9Form , BlogNews
from .views import GetContentType
import json
from django.contrib import messages
from django.core.paginator import Paginator
from cflax.settings import sectionname
from .dashboardsettings import templateFolder
sections = [SubRegistrationContent,AboutRegistraionSubMenu,DocumentRequired,PackageIncluded,Procedure,Memorandum,CompanyRegisterRequirements,FAQ,Sainification,ourclients,BlogNews]
# adding multi objects form as a list in sectionsform
sectionsforms = [section0Form,section1Form,section2Form,[PackageIncludedForm],[section3Form],section4Form,section5Form,[section6Form],section7Form,[section8Form],[section9Form]]
editname = ['slider','AboutCA','News_nortification','DueDate_Reminder','Blog_News_nortification','New_Blogs']
def editothernavs(request,slug1,slug2):
    RegistrationSubMenuob = RegistrationSubMenu.objects.get(slug=slug2,title__slug=slug1)
    s = []
    for i in sections:
        s.append(i.objects.filter(reg_title = RegistrationSubMenuob) if i.objects.filter(reg_title = RegistrationSubMenuob).exists() else [None] )
    # s1 = sections[0].objects.filter(reg_title = RegistrationSubMenuob) if sections[0].objects.filter(reg_title = RegistrationSubMenuob).exists()  else [RegistrationSubMenuob]
    # s2 = sections[1].objects.filter(reg_title = RegistrationSubMenuob) if sections[1].objects.filter(reg_title = RegistrationSubMenuob).exists()  else [RegistrationSubMenuob]
    # s3 = sections[2].objects.filter(reg_title = RegistrationSubMenuob) if sections[2].objects.filter(reg_title = RegistrationSubMenuob).exists()  else [RegistrationSubMenuob]
    # s4 = sections[3].objects.filter(reg_title = RegistrationSubMenuob) if sections[3].objects.filter(reg_title = RegistrationSubMenuob).exists()  else [RegistrationSubMenuob]
    # s5 = sections[4].objects.filter(reg_title = RegistrationSubMenuob) if sections[4].objects.filter(reg_title = RegistrationSubMenuob).exists()  else [RegistrationSubMenuob]
    # s6 = sections[5].objects.filter(reg_title = RegistrationSubMenuob) if sections[5].objects.filter(reg_title = RegistrationSubMenuob).exists()  else [RegistrationSubMenuob]
    # s7 = sections[6].objects.filter(reg_title = RegistrationSubMenuob) if sections[6].objects.filter(reg_title = RegistrationSubMenuob).exists()  else [RegistrationSubMenuob]
    res = {}
    res['title'] = "Edit Registrations"
    li = []
    for i in range(0,len(sectionsforms)):
        if isinstance(sectionsforms[i],list):
            new = [sectionsforms[i][0](instance= j,initial={'reg_title': RegistrationSubMenuob}) for j in s[i]]
            new.append(sectionsforms[i][0](initial={'reg_title': RegistrationSubMenuob}))
            li.append(new)
        else:
            li.append(sectionsforms[i](instance= s[i][0],initial={'reg_title': RegistrationSubMenuob}))
    # li.append(section1Form(instance= s[0][0]))
    # li.append(section2Form(instance= s[1][0]))
    # li.append([section3Form(instance= i) for i in s[2]])
    # li.append(section4Form(instance= s[3][0]))
    # li.append(section5Form(instance= s[4][0]))
    # li.append([section6Form(instance= i) for i in s[5]])
    # li.append(section7Form(instance= s[6][0]))
    if request.method == "POST":
        page_number = int(request.GET.get('page') if request.GET.get('page') is not None else 1) - 1
        objectno = (request.GET.get('object') if request.GET.get('object') is not None else 1)
        if request.GET.get('object') is None:
            form = sectionsforms[page_number](request.POST,request.FILES,instance=s[page_number][0])
        else:
            if sectionsforms[page_number] is None or objectno == "new":
                form = sectionsforms[page_number][0](request.POST,request.FILES,instance = None)
            else:
                form = sectionsforms[page_number][0](request.POST,request.FILES,instance = s[page_number].filter(id=objectno)[0])
        if form.is_valid():
            form = form.save(commit=False)
            try:
                form.content_type =GetContentType(RegistrationSubMenuob)
                form.object_id = RegistrationSubMenuob.id
            except Exception as e:
                # print("exception +++++++",e)
                pass
            form.save()
            messages.success(request,"Information Is Added Successfully")
            path = request.get_full_path().rfind('/')
            if '&object' in request.get_full_path()[path:]:
                return redirect(request.get_full_path()[0:request.get_full_path().find('&object')])
            else:
                return redirect(request.get_full_path())
            return redirect(request.get_full_path()[0:request.get_full_path().find('&object')])
        else:
            messages.error  (request,"Plese Check Your Fields, Invalid Opration")
            li[page_number] = form
    # sec = Sections.objects.filter(reg_title = RegistrationSubMenuob)
    sec = sectionname
    paginator = Paginator(li, 1)
    page_number = request.GET.get('page')
    res['page_obj'] = paginator.get_page(page_number)
    if isinstance(res['page_obj'].object_list[0],list):
        paginator2 = Paginator(res['page_obj'].object_list[0], 1)
        card = request.GET.get('card')
        res['card_obj'] = paginator2.get_page(card)
        if 'icon' in res['card_obj'].object_list[0].fields:
            res['icon'] = icon.objects.all()
    res['forms'] = li
    res['slugs'] = [slug1,slug2,RegistrationSubMenuob.submenu]
    res['sec'] = [[i,sec[i-1]] for i in res['page_obj'].paginator.page_range ]
    try:
        return render(request,f'{templateFolder}/extra/editothernavs.html',res)
    except Exception:
        return render(request,'editothernavs.html',res)

def deleteothernavs(request):
    if request.method == "POST":
        formid = int(request.POST.get('page') if request.POST.get('page') is not None and request.POST.get('page')!=""  else 1) - 1
        delid = request.POST.get('id')
        ret = request.POST.get('return')
        if ret is not None:
            data = sections[formid].objects.get(id=delid)
            data.delete()
            messages.error(request,"successfully deleted")
            return redirect(ret)
    return redirect(request.get_full_path()[0:request.get_full_path().find('&object')])
def contact(request):
    res = {}
    res['title'] = "Registration Contacts"
    res['contacts'] = contacts.objects.all().order_by('-time')
    return render(request,'drcontacts.html',res)
def delcontacts(request):
    if request.method=="POST":
        delval = request.POST['delval'].split(',')
        for d in delval:
            d = contacts.objects.filter(id=d)
            if d.exists():
                d[0].delete()
        messages.error(request,"contact Deleted successfully")
    return redirect('drcontact')


import random
import string

HANGMAN_PICS = ['''
  3.   +---+
  4.       |
  5.       |
  6.       |
  7.      ===''', '''
  8.   +---+
  9.   O   |
 10.       |
 11.       |
 12.      ===''', '''
 13.   +---+
 14.   O   |
 15.   |   |
 16.       |
 17.      ===''', '''
 18.   +---+
 19.   O   |
 20.  /|   |
 21.       |
 22.      ===''', '''
 23.   +---+
 24.   O   |
 25.  /|\  |
 26.       |
 27.      ===''', '''
 28.   +---+
 29.   O   |
 30.  /|\  |
 31.  /    |
 32.      ===''', '''
 33.   +---+
 34.   O   |
 35.  /|\  |
 36.  / \  |
 37.      ===''']

words = {1: "ant", 2: "lion", 3: "cougar", 4: "horse", 5: "monkey", 6: "lama", 7: "tiger", 8: "panda", 9: "sheep"}
computer_word = list(words[random.randint(1, 9)])
print(computer_word)
mystery = []
for i in computer_word:
    mystery.append("_")

while "_" in mystery:
    def trying():
        letter_try = input("Guess the letter\n").lower()
        for position in range(len(computer_word)):
            letter = computer_word[position]
            if letter == letter_try:
                mystery[position] = letter
        if letter_try not in computer_word:
            print("Ops, wrong...\n\n")
            print(HANGMAN_PICS[0])
            HANGMAN_PICS.pop(0)

    print(f"{''.join(mystery)}\n\n")
    if not HANGMAN_PICS:
        print("You died")
        break

    trying()

if mystery == computer_word:
    print("Congrats, you won!!!!!")


from __future__ import absolute_import, division, print_function, unicode_literals

# stdlib
from cassandra.cluster import Cluster
from StringIO import StringIO
from unittest import TestCase

# Bunch
from bunch import Bunch

# mock
from mock import patch

# pysandra-unit
import pysandraunit

# Zato
from zato.apitest import util
from zato.apitest.steps import cassandra_

# ###############################################################################################################################

class EmbeddedCassandraTestCase(TestCase):

    def __init__(self, methodName='runTest'):
        super(EmbeddedCassandraTestCase, self).__init__(methodName)

    def setUp(self):
        # TODO: Cassandra tests are failing due to """PysandraUnitServerError:
        # Failed to execute command start: /127.0.0.1:7010 is in use by another process.
        # Change listen_address:storage_port in cassandra.yaml to values that do not conflict with other services"""
        return
        self.ctx = Bunch()
        self.ctx.zato = util.new_context(None, util.rand_string(), {})

        import time
        time.sleep(2)

        # setup initial Cassandra keyspace
        self.current_session_name = util.rand_string()
        self.columns = ['userid', 'fname', 'sname']
        self.values = util.rand_string(3)
        self.keyspace = util.rand_string()
        self.table = util.rand_string()
        data = (self.keyspace, self.table) + tuple(s for s in self.columns)
        keyspace_statement = (
            "CREATE KEYSPACE %s WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 3 }") % self.keyspace
        table_statement = ("CREATE TABLE %s.%s (%s text PRIMARY KEY, %s text, %s text)") % data

        self.embedded_cassandra = pysandraunit.PysandraUnit(native_transport_port=9042)
        self.embedded_cassandra.start()
        self.cluster = Cluster(protocol_version=1)
        self.session = self.cluster.connect()
        self.session.execute(keyspace_statement)
        self.session.execute(table_statement)
        self.cluster.shutdown()

        # setup Cassandra ctx
        cassandra_.given_cassandra_contact_points(self.ctx, 'localhost')
        cassandra_.given_cassandra_protocol_version(self.ctx, 1)
        cassandra_.given_cassandra_port(self.ctx, '9042')
        cassandra_.given_i_connect_to_keyspace_as_session(self.ctx, self.keyspace, self.current_session_name)

    def tearDown(self):
        # TODO: Cassandra tests are failing due to """PysandraUnitServerError:
        # Failed to execute command start: /127.0.0.1:7010 is in use by another process.
        # Change listen_address:storage_port in cassandra.yaml to values that do not conflict with other services"""
        return
        self.embedded_cassandra.stop()

class CassandraTestCase(EmbeddedCassandraTestCase):

    def test_given_i_connect_to_keyspace_as_conn_name(self):
        # TODO: Cassandra tests are failing due to """PysandraUnitServerError:
        # Failed to execute command start: /127.0.0.1:7010 is in use by another process.
        # Change listen_address:storage_port in cassandra.yaml to values that do not conflict with other services"""
        return
        current_session = self.ctx.zato.user_ctx[self.current_session_name]
        self.assertEquals(type(current_session), type(self.session))

    def test_then_i_insert_values_into_columns_of_cassandra_table(self):
        # TODO: Cassandra tests are failing due to """PysandraUnitServerError:
        # Failed to execute command start: /127.0.0.1:7010 is in use by another process.
        # Change listen_address:storage_port in cassandra.yaml to values that do not conflict with other services"""
        return
        cassandra_.then_i_insert_values_into_columns_of_cassandra_table(
            self.ctx, self.table, ', '.join(self.values), ', '.join(self.columns), self.current_session_name)

        statement = "SELECT * FROM %s" % self.table
        cassandra_.given_i_store_cql_query_result_under_name(self.ctx, statement, 'cql_result', self.current_session_name, 0)
        self.assertEquals(self.ctx.zato.user_ctx['cql_result'], ';'.join(self.values))

    def test_then_i_update_columns_of_cassandra_table_setting_values(self):
        # TODO: Cassandra tests are failing due to """PysandraUnitServerError:
        # Failed to execute command start: /127.0.0.1:7010 is in use by another process.
        # Change listen_address:storage_port in cassandra.yaml to values that do not conflict with other services"""
        return
        new_values = 'John, Doe'
        criterion = "WHERE %s='%s'" % (self.columns[0], self.values[0])
        cassandra_.then_i_update_columns_of_cassandra_table_setting_values(
            self.ctx, self.table, ', '.join(self.columns[1:]), new_values, self.current_session_name, criterion)

        statement = "SELECT * FROM %s" % self.table
        cassandra_.given_i_store_cql_query_result_under_name(self.ctx, statement, 'cql_result', self.current_session_name, 0)
        self.assertEquals(self.ctx.zato.user_ctx['cql_result'], '{};John;Doe'.format(self.values[0]))

    def test_then_i_delete_from_cassandra_table(self):
        # TODO: Cassandra tests are failing due to """PysandraUnitServerError:
        # Failed to execute command start: /127.0.0.1:7010 is in use by another process.
        # Change listen_address:storage_port in cassandra.yaml to values that do not conflict with other services"""
        return
        criterion = "WHERE %s='%s'" % (self.columns[0], self.values[0])
        cassandra_.then_i_delete_from_cassandra_table(self.ctx, self.table, self.current_session_name, criterion)

        statement = "SELECT * FROM %s" % self.table
        cassandra_.given_i_store_cql_query_result_under_name(self.ctx, statement, 'cql_result', self.current_session_name, 0)
        self.assertEquals(self.ctx.zato.user_ctx['cql_result'], '')

    @patch('__builtin__.open')
    def test_i_insert_data_from_csv_file_to_cassandra_table(self, open_mock):
        # TODO: Cassandra tests are failing due to """PysandraUnitServerError:
        # Failed to execute command start: /127.0.0.1:7010 is in use by another process.
        # Change listen_address:storage_port in cassandra.yaml to values that do not conflict with other services"""
        return
        values = (util.rand_string(), util.rand_string(), util.rand_string())
        fake_csv = 'userid, fname, sname\n%s, %s, %s' % values
        filename = util.rand_string()

        open_mock.return_value = StringIO(fake_csv)
        cassandra_.i_insert_data_from_csv_file_to_cassandra_table(self.ctx, filename, self.table, self.current_session_name)

        statement = "SELECT * FROM %s" % self.table
        cassandra_.given_i_store_cql_query_result_under_name(self.ctx, statement, 'cql_result', self.current_session_name, 0)
        self.assertEquals(self.ctx.zato.user_ctx['cql_result'], ';'.join(values))


#!/usr/bin/env python

f = open('../full.txt', 'r')
uniq = []
for l in f:
    parts = l.split('.')
    for p in parts[:-1]:
        if p not in uniq:
            uniq.append(p)
            print(p)


from bson.json_util import dumps
from django.views.decorators.csrf import csrf_exempt
from . import db, get_json, basic_success, basic_error

failure = dumps({"Failed"})
@csrf_exempt
def add_review(request):
    try:
        data = get_json(request)
        collection =db.test
        reviews = db.reviews
        try:
            result=reviews.distinct("uniq_id")
            data['uniq_id']=max(result)+1
        except:
            data['uniq_id']= 1
        reviews.insert(data)
        #collection.update({'uniq_id':data['f_uniq_id']},{ "review": 1 },{"upsert":True})
        return basic_success()
    except Exception as e:
        return basic_error(e)


#
# @lc app=leetcode.cn id=43 lang=python3
#
# [43] 字符串相乘
#

# @lc code=start
class Solution:
    def multiply(self, num1: str, num2: str) -> str:
        a = num1[::-1]
        b = num2[::-1]
        res = 0

        for i, x in enumerate(b):
            temp_res = 0
            for j, y in enumerate(a):
                temp_res += int(x) * int(y) * 10 ** j
            res += temp_res * 10 ** i

        return str(res)
# @lc code=end


from django.conf import settings
from django.contrib.auth.views import LogoutView
from django.urls import path
from . import views

urlpatterns = [
    path('auth/logout/', LogoutView.as_view(),
         {'next_page': settings.LOGOUT_REDIRECT_URL}, name='logout'),
    path('auth/signup/', views.SignUpView.as_view(), name='signup'),
    path('my_subscriptions/', views.follow_page, name='follow_page'),
    path('subscriptions', views.follow_author, name='subscription'),
    path('subscriptions/<int:id>', views.unfollow_author,
         name='unsubscription'),
    path('<username>/', views.user_profile, name='user_profile'),
]


"""
Fail fast

try/except
try -> tente executar o codigo
except -> ocorreu algum erro ao tentar executar
"""

numero_str = input('Vou dobrar o número que vc digitar: ')

try:
  numero_float = float(numero_str)
  print(f'O dobro de {numero_str} é {numero_float * 2:.2f}')
except:
  print('Não é um número')

# if numero_str.isdigit():
#   numero_float = float(numero_str)
#   print(f'O dobro de {numero_str} é {numero_float * 2:.2f}')
# else:
#   print('Não é um número')


import os
import numpy as np
from PIL import Image
import onnxruntime as rt
from django.utils import timezone
from .models import HushemPrediction
from rest_framework import viewsets, status
import torchvision.transforms as transforms
from rest_framework.decorators import action
from rest_framework.response import Response
from rest_framework.permissions import AllowAny
from .serializers import HushemPredictionSerializer

class HushemPredictionViewSet(viewsets.ViewSet):
    queryset = HushemPrediction.objects.all()
    permission_classes = [AllowAny]  # Permet l'accès sans authentification

    def get_queryset(self):
        # Renvoie le queryset par défaut pour cette vue
        return self.queryset

    @action(detail=False, methods=['post'])
    def predict(self, request):
        # Récupère la liste des images envoyées avec la requête
        images = request.FILES.getlist('images')
        predictions = []

        for image_file in images:
            # Ouvre l'image à partir du fichier
            image = Image.open(image_file)

            # Prétraitement de l'image
            preprocess = transforms.Compose([
                # Redimensionne l'image à une taille de 224x224 pixels
                transforms.Resize((224, 224)),
                # Convertit l'image en un tenseur PyTorch
                transforms.ToTensor(),
                # Normalise les valeurs des pixels de l'image en utilisant les moyennes et les écarts-types spécifiés
                transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
            ])
            # Applique les transformations de prétraitement à l'image et ajoute une dimension supplémentaire pour créer un lot d'images contenant une seule image
            image_tensor = preprocess(image).unsqueeze(0)

            # Charge le modèle ONNX à partir du fichier
            model_path = os.path.abspath(os.path.join(os.path.dirname(__file__), './hushem_model.onnx'))
            sess = rt.InferenceSession(model_path)

            # Préparer les données d'entrée pour le modèle
            input_name = sess.get_inputs()[0].name
            input_data = {input_name: image_tensor.expand(32, -1, -1, -1).numpy()}

            # Effectuer une prédiction avec le modèle
            outputs = sess.run(None, input_data)
            predicted = np.argmax(outputs[0])
            precision_array = outputs[0][predicted]
            precision = np.mean(precision_array) # Calcul de la moyenne de la précision

            # Récupérer la classe prédite à partir de l'indice de la prédiction
            class_names = ['normal', 'abnormal', 'healthy', 'unhealthy']
            predicted_class = class_names[predicted]

            # Enregistrer la date et le résultat de la prédiction dans la base de données
            prediction = HushemPrediction(date=timezone.now(), results=predicted_class, precision=precision)
            prediction.save()
            predictions.append(predicted_class)

        # Renvoie la liste des classes prédites en réponse à la requête
        return Response(predictions)

class HushemPredictionList(viewsets.ViewSet):
    # Renvoie un queryset contenant tous les objets HushemPrediction
    def get_queryset(self):
        return HushemPrediction.objects.all()

    # Gère les requêtes GET pour récupérer la liste des prédictions Hushem
    def list(self, request):
        # Récupère tous les objets HushemPrediction
        predictions = HushemPrediction.objects.all()
        # Sérialise les objets HushemPrediction en utilisant le HushemPredictionSerializer
        serializer = HushemPredictionSerializer(predictions, many=True)
        # Renvoie les données sérialisées en réponse à la requête
        return Response(serializer.data)

    # Gère les requêtes POST pour créer une nouvelle prédiction Hushem
    def create(self, request):
        # Valide les données de la requête en utilisant le HushemPredictionSerializer
        serializer = HushemPredictionSerializer(data=request.data)
        if serializer.is_valid():
            # Crée un nouvel objet HushemPrediction avec les données validées
            serializer.save()
            # Renvoie les données sérialisées du nouvel objet en réponse à la requête
            return Response(serializer.data, status=status.HTTP_201_CREATED)
        # Si les données ne sont pas valides, renvoie les erreurs de validation en réponse à la requête
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    # Gère également les requêtes POST pour créer une nouvelle prédiction Hushem
    def post(self, request):
        # Valide les données de la requête en utilisant le HushemPredictionSerializer
        serializer = HushemPredictionSerializer(data=request.data)
        if serializer.is_valid():
            # Crée un nouvel objet HushemPrediction avec les données validées
            serializer.save()
            # Renvoie les données sérialisées du nouvel objet en réponse à la requête
            return Response(serializer.data, status=status.HTTP_201_CREATED)
        # Si les données ne sont pas valides, renvoie les erreurs de validation en réponse à la requête
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)


import operator

from ... import opcodes as OperandDef
from ...utils import classproperty
from .core import DataFrameBinopUfunc
from .docstring import bin_arithmetic_doc

class DataFrameFloorDiv(DataFrameBinopUfunc):
    _op_type_ = OperandDef.FLOORDIV

    _func_name = "floordiv"
    _rfunc_name = "rfloordiv"

    @classproperty
    def _operator(self):
        return operator.floordiv

    @classproperty
    def tensor_op_type(self):
        from ...tensor.arithmetic import TensorFloorDiv

        return TensorFloorDiv

_floordiv_example = """
>>> a.floordiv(b, fill_value=0).execute()
a    1.0
b    NaN
c    NaN
d    0.0
e    NaN
dtype: float64
"""

@bin_arithmetic_doc("Integer division", equiv="//", series_example=_floordiv_example)
def floordiv(df, other, axis="columns", level=None, fill_value=None):
    op = DataFrameFloorDiv(
        axis=axis, level=level, fill_value=fill_value, lhs=df, rhs=other
    )
    return op(df, other)

@bin_arithmetic_doc("Integer division", equiv="//", series_example=_floordiv_example)
def rfloordiv(df, other, axis="columns", level=None, fill_value=None):
    op = DataFrameFloorDiv(
        axis=axis, level=level, fill_value=fill_value, lhs=other, rhs=df
    )
    return op.rcall(df, other)


import urllib.request, urllib.parse
import json
import urllib.parse, urllib.request
from harvest_records import harvest_records
from bs4 import BeautifulSoup
import write_error_to_logfile
import re
import find_sysnumbers_of_volumes

def create_publication_dicts(last_item_harvested_in_last_session, *other):
    publishers = {'1876': ['Athen', 'In Commission bei Karl Wilberg'],
                  '1886': ['Athen', 'Verlag Karl Wilberg'],
                  '1896': ['Athen', 'Barth & von Hirst'],
                  '1901': ['Athen', 'Beck und Barth'],
                  '1910': ['Athen', 'Eleutheroudakis und Barth'],
                  '1915': ['Athen', 'Archäologisches Institut'],
                  '1916': ['Berlin', 'Archäologisches Institut'],
                  '1926': ['Athen', 'Archäologisches Institut'],
                  '1938': ['Stuttgart', 'Verlag W. Kohlhammer'],
                  '1942': ['Berlin', 'Walter de Gruyter'],
                  '1953': ['Berlin', 'Verlag Gebr. Mann'],
                  '1997': ['Mainz', 'Philipp von Zabern'],
                  '2011': ['Berlin', 'Verlag Gebr. Mann']
                  }
    years_published_in = [int(year) for year in list(publishers.keys())]
    years_published_in.sort(reverse=True)
    titles_for_supplementing = ['Sitzungsprotocolle', 'Sitzungs-Protocolle', 'Litteratur und Funde',
                                'Ernennungen', 'Eingegangene Schriften', 'Miscellen', 'Litteratur',
                                'Berichtigungen', 'Epigraphische Miscellen', 'Funde', 'Sitzungsprotokolle',
                                'Ernennungen / Sitzungs-Protocolle', 'Register']
    publication_dicts = []
    items_harvested = []
    try:
        sysnumbers = find_sysnumbers_of_volumes.find_sysnumbers('000054834')
        sysnumbers['1883'] = '001101806'
        sysnumbers['1888'] = '001101811'
        basic_url = 'https://digi.ub.uni-heidelberg.de/diglit/am'
        record_nr = 0
        user_agent = 'Mozilla/5.0 (X11; Linux x86_64; rv:66.0)'
        values = {'name': 'Helena Nebel',
                  'location': 'Berlin',
                  'language': 'Python'}
        headers = {'User-Agent': user_agent}
        data = urllib.parse.urlencode(values)
        data = data.encode('ascii')
        req = urllib.request.Request(basic_url, data, headers)
        with urllib.request.urlopen(req) as response:
            yearbook_page = response.read()
        yearbook_page = yearbook_page.decode('utf-8')
        yearbook_page = BeautifulSoup(yearbook_page, 'html.parser')
        volume_years = [item.text for item in yearbook_page.find('table', class_='tabelle-baendeliste').find_all('span', class_='publ-daten-schwarz')]
        volume_urls = [item['href'] for item in yearbook_page.find('table', class_='tabelle-baendeliste').find_all('a')]
        for volume_url in volume_urls:
            #print(volume_url)
            volume_year=volume_years[record_nr].split('(')[0].split(', ')[-1]
            record_nr+=1
            mets_url=volume_url.split("?")[0]+'/mets'
            #print(mets_url)
            webFile = urllib.request.urlopen(mets_url)
            xml_soup=BeautifulSoup(webFile, 'xml')
            date_of_publication = xml_soup.find_all('mods:mods')[0].find('mods:originInfo').find('mods:dateIssued', keyDate='yes').text
            if '(' in date_of_publication:
                date_of_publication = date_of_publication.split('(')[1].replace(')')
            volume=xml_soup.find_all('mods:mods')[0].find('mods:number').text
            content_tags = [tag.find('mods:mods') for tag in xml_soup.find_all('mets:dmdSec')if re.findall(r'^dmd\d+$', tag['ID'])]
            for item in content_tags:
                current_item = int(volume_year)
                if current_item == 1888:
                    continue
                if not item.find('mods:identifier', type='doi'):
                    continue
                if current_item > last_item_harvested_in_last_session:
                    with open('publication_dict.json', 'r') as publication_dict_template:
                        publication_dict = json.load(publication_dict_template)
                    if item.find('mods:title').text in ['Maßstab/Farbkeil', 'Umschlag', 'Titelblatt', 'Inhalt']:
                        continue
                    if ('Heft' in item.find('mods:title').text) or ('Berichtigung' in item.find('mods:title').text):
                        continue
                    publication_dict['title_dict']['main_title'] = item.find('mods:title').text
                    if item.find('mods:subTitle'):
                        publication_dict['title_dict']['sub_title'] = item.find('mods:subTitle').text
                    if 'Beilage' in publication_dict['title_dict']['main_title']:
                        titles_for_supplementing.append(publication_dict['title_dict']['main_title'])
                    if publication_dict['title_dict']['main_title'] in titles_for_supplementing:
                        publication_dict['title_dict']['sub_title'] = 'Deutsches Archaäologisches Institut, Athenische Abteilung, ' + volume_year
                    publication_dict['rdacontent'] = 'txt'
                    publication_dict['rdamedia'] = 'n'
                    publication_dict['rdacarrier'] = 'nc'
                    for author in item.find_all('mods:name'):
                        author_name = ""
                        if author.find_all('mods:roleTerm'):
                            if author_name==author.find('mods:displayForm').text:
                                continue
                            author_name = author.find('mods:displayForm').text
                            publication_dict['authors_list'].append(author_name)
                    publication_dict['field_007'] = 'ta'
                    publication_dict['field_008_18-34'] = 'ar p|| ||||0   b|'
                    if item.find('mods:physicalDescription'):
                        physical_descriptions = item.find('mods:physicalDescription')
                        publication_dict['field_300'] = 'p. ' + physical_descriptions.find('mods:extent', unit='pages').text if physical_descriptions.find('mods:extent', unit='pages') else ''
                    publication_dict['LDR_06_07'] = 'ab'
                    publication_dict['default_language'] = item.find('mods:language').find('mods:languageTerm').text if item.find('mods:language') else 'ger'
                    publication_dict['original_cataloging_agency'] = 'DE-16'
                    publication_dict['retro_digitization_info'] = {'place_of_publisher': '', 'publisher': '', 'date_published_online': ''}
                    publication_dict['fields_590'] = ['arom', '2021xhnxamk']
                    publication_dict['volume'] = volume
                    publication_dict['volume_year'] = volume_year
                    if current_item < 1886:
                        publication_dict['host_item']['name'] = 'Mittheilungen des Deutschen Archäologischen Instituts in Athen'
                    elif current_item < 1915:
                        publication_dict['host_item']['name'] = 'Mitteilungen des Kaiserlich-Deutschen Archäologischen Instituts, Athenische Abteilung'
                    else:
                        publication_dict['host_item']['name'] = 'Mitteilungen des Deutschen Archäologischen Instituts, Athenische Abteilung'
                    publication_dict['host_item']['sysnumber'] = sysnumbers[date_of_publication]
                    publication_dict['host_item_is_volume'] = True
                    publication_dict['publication_year'] = date_of_publication
                    place_of_publication = ''
                    publisher = ''
                    for key in years_published_in:
                        if int(publication_dict['publication_year']) >= key:
                            place_of_publication = publishers[str(key)][0]
                            publisher = publishers[str(key)][1]
                            break
                    publication_dict['publication_etc_statement']['publication'] = {'place': place_of_publication,
                                                                                'responsible': publisher,
                                                                                'country_code': 'gw '}
                    publication_dicts.append(publication_dict)
                    items_harvested.append(current_item)
    except Exception as e:
        write_error_to_logfile.write(e)
        write_error_to_logfile.comment('Es konnten keine Artikel für Mitteilungen des Deutschen Archäologischen Instituts, Athenische Abteilung geharvested werden.')
        items_harvested, publication_dicts = [], []
    return publication_dicts, items_harvested

def harvest(path):
    return_string = harvest_records(path, 'am', 'Mitteilungen des Deutschen Archäologischen Instituts, Athenische Abteilung', create_publication_dicts)
    return return_string

if __name__ == '__main__':
    harvest_records('records/am/', 'am', 'Mitteilungen des Deutschen Archäologischen Instituts, Athenische Abteilung', create_publication_dicts)


# model settings
model = dict(
    type='CascadeRCNN',
    num_stages=3,
    pretrained='pretrain_model/cascade_rcnn_dconv_c3-c5_r101_dufpn_1x.pth',
    backbone=dict(
        type='ResNet',
        depth=101,
        num_stages=4,
        out_indices=(0, 1, 2, 3),
        frozen_stages=1,
        norm_cfg=dict(type='BN', requires_grad=True),
        style='pytorch',
        dcn=dict(type='DCN', deformable_groups=1, fallback_on_stride=False),
        stage_with_dcn=(False, True, True, True)),
    neck=dict(
        type='DuFPN',
        in_channels=[256, 512, 1024, 2048],
        out_channels=256,
        num_outs=5),
    rpn_head=dict(
        type='RPNHead',
        in_channels=256,
        feat_channels=256,
        anchor_scales=[8],
        anchor_ratios=[0.4, 0.9, 1.6, 2.9, 4.2, 6.0, 8.6],
        anchor_strides=[4, 8, 16, 32, 64],
        target_means=[.0, .0, .0, .0],
        target_stds=[1.0, 1.0, 1.0, 1.0],
        loss_cls=dict(
            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
        loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)),
    bbox_roi_extractor=dict(
        type='SingleRoIExtractor',
        roi_layer=dict(type='RoIAlign', out_size=7, sample_num=2),
        out_channels=256,
        featmap_strides=[4, 8, 16, 32]),
    bbox_head=[
        dict(
            type='UncertainSharedFCBBoxHead',
            num_fcs=2,
            in_channels=256,
            fc_out_channels=1024,
            roi_feat_size=7,
            num_classes=2,
            target_means=[0., 0., 0., 0.],
            target_stds=[0.1, 0.1, 0.1, 0.1],
            with_uncertain=True,
            reg_class_agnostic=True,
            loss_cls=dict(
                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0,),# smoothing=0.01
            loss_bbox=dict(type='UncertainKLLoss', beta=1.0, loss_weight=1.0)),
        dict(
            type='UncertainSharedFCBBoxHead',
            num_fcs=2,
            in_channels=256,
            fc_out_channels=1024,
            roi_feat_size=7,
            num_classes=2,
            target_means=[0., 0., 0., 0.],
            target_stds=[0.05, 0.05, 0.05, 0.05],
            with_uncertain=True,
            reg_class_agnostic=True,
            loss_cls=dict(
                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0,),# smoothing=0.01
            loss_bbox=dict(type='UncertainKLLoss', beta=1.0, loss_weight=1.0)),
        dict(
            type='UncertainSharedFCBBoxHead',
            num_fcs=2,
            in_channels=256,
            fc_out_channels=1024,
            roi_feat_size=7,
            num_classes=2,
            target_means=[0., 0., 0., 0.],
            target_stds=[0.033, 0.033, 0.033, 0.033],
            with_uncertain=True,
            reg_class_agnostic=True,
            loss_cls=dict(
                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0,),
            loss_bbox=dict(type='UncertainKLLoss', beta=1.0, loss_weight=1.0))
    ])
# model training and testing settings
train_cfg = dict(
    rpn=dict(
        assigner=dict(
            type='MaxIoUAssigner',
            pos_iou_thr=0.7,
            neg_iou_thr=0.3,
            min_pos_iou=0.3,
            ignore_iof_thr=-1),
        sampler=dict(
            type='RandomSampler',
            num=256,
            pos_fraction=0.5,
            neg_pos_ub=-1,
            add_gt_as_proposals=False),
        allowed_border=0,
        pos_weight=-1,
        debug=False),
    rpn_proposal=dict(
        nms_across_levels=False,
        nms_pre=2000,
        nms_post=2000,
        max_num=2000,
        nms_thr=0.7,
        min_bbox_size=0),
    rcnn=[
        dict(
            assigner=dict(
                type='MaxIoUAssigner',
                pos_iou_thr=0.5,
                neg_iou_thr=0.5,
                min_pos_iou=0.5,
                ignore_iof_thr=-1),
            sampler=dict(
                type='RandomSampler',
                num=512,
                pos_fraction=0.25,
                neg_pos_ub=-1,
                add_gt_as_proposals=True),
            pos_weight=-1,
            debug=False),
        dict(
            assigner=dict(
                type='MaxIoUAssigner',
                pos_iou_thr=0.6,
                neg_iou_thr=0.6,
                min_pos_iou=0.6,
                ignore_iof_thr=-1),
            sampler=dict(
                type='RandomSampler',
                num=512,
                pos_fraction=0.25,
                neg_pos_ub=-1,
                add_gt_as_proposals=True),
            pos_weight=-1,
            debug=False),
        dict(
            assigner=dict(
                type='MaxIoUAssigner',
                pos_iou_thr=0.7,
                neg_iou_thr=0.7,
                min_pos_iou=0.7,
                ignore_iof_thr=-1),
            sampler=dict(
                type='RandomSampler',
                num=512,
                pos_fraction=0.25,
                neg_pos_ub=-1,
                add_gt_as_proposals=True),
            pos_weight=-1,
            debug=False)
    ],
    stage_loss_weights=[1, 0.5, 0.25])
test_cfg = dict(
    rpn=dict(
        nms_across_levels=False,
        nms_pre=1000,
        nms_post=1000,
        max_num=1000,
        nms_thr=0.7,
        min_bbox_size=0),
    rcnn=dict(score_thr=0.001,
              nms=dict(type='soft_nms_variance_voting',
                       sigma_t=0.01, iou_thr=0.5, min_score=0.001,
                      ),
              return_variance=False,
              max_per_img=80),)
# dataset settings
dataset_type = 'SonarDataset'
data_root = 'data/'
img_norm_cfg = dict(
    mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True)

train_pipeline = [
    dict(type='LoadImageFromFile'),
    dict(type='LoadAnnotations', with_bbox=True),
    dict(type='Grafting_SideScan', image_root_dir=data_root, use_side=True),
    dict(type='Stretch', stretch_prob=0.75, stretch_ratios=[(0.95, 1.05),(0.95, 1.05)]),
    dict(type='MinIoURandomCrop', min_ious=(0.5, 0.6, 0.7, 0.8, 0.9), min_crop_size=0.5),
    dict(type='AbjointResize', ratio_scales=[[0.8, 0.9, 1.0, 1.1, 1.2],
                                         [0.8, 0.9, 1.0, 1.1, 1.2]], keep_ratio=True),
    dict(type='RandomFlip', flip_ratio=0.66, direction=['horizontal', 'vertical'], filter_prefix=['gx',]), # 'vertical'
    dict(type='Pad', size_divisor=32),
    dict(type='Normalize', **img_norm_cfg),
    dict(type='DefaultFormatBundle'),
    dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels']),
]
test_pipeline = [
    dict(type='LoadImageFromFile'),
    dict(
        type='MultiScaleFlipAug',
        ratio_scale=[[0.8, 0.9, 1.0, 1.1, 1.2], # 1.0, 1.1, 1.2, 1.3, 1.4
                     [0.8, 0.9, 1.0, 1.1, 1.2]],
        flip=True,
        ver_flip=True,
        filter_prefix=['gx'],
        transforms=[
            dict(type='AbjointResize', keep_ratio=True),
            dict(type='RandomFlip'),
            dict(type='Normalize', **img_norm_cfg),
            dict(type='Pad', size_divisor=32),
            dict(type='ImageToTensor', keys=['img']),
            dict(type='Collect', keys=['img']),
        ])
]

data = dict(
    imgs_per_gpu=1,
    workers_per_gpu=2,
    train=dict(
        type=dataset_type,
        # ann_file=data_root + 'annotation/instances_train.json',
        ann_file=data_root + 'annotation/instances_all.json',
        img_prefix=data_root + 'image/',
        pipeline=train_pipeline),
    val=dict(
        type=dataset_type,
        ann_file=data_root + 'annotation/instances_val.json',
        img_prefix=data_root + 'image/',
        pipeline=test_pipeline),
    test=dict(
        type=dataset_type,
        ann_file=data_root + 'annotation/a-test.json', # b-test
        img_prefix=data_root + 'a-test-image/', # b-test-image
        # ann_file=data_root + 'annotation/instances_val.json',
        # img_prefix=data_root + 'image/',
        pipeline=test_pipeline))

evaluation = dict(interval=4, metric='bbox')
# optimizer
optimizer = dict(type='SGD', lr=0.02, momentum=0.9, weight_decay=0.0005)
optimizer_config = dict(grad_clip=dict(max_norm=35, norm_type=2))
# learning policy
lr_config = dict(
    policy='step',
    warmup='linear',
    warmup_iters=100,
    warmup_ratio=1.0 / 3,
    step=[8, 11])
checkpoint_config = dict(interval=6)
# yapf:disable
log_config = dict(
    interval=50,
    hooks=[
        dict(type='TextLoggerHook'),
        # dict(type='TensorboardLoggerHook')
    ])
# yapf:enable
# runtime settings
total_epochs = 12
dist_params = dict(backend='nccl')
log_level = 'INFO'
work_dir = './work_dirs/cascade_rcnn_dconv_c3-c5_r101_dufpn_1x'
load_from = 'pretrain_model/cascade_rcnn_dconv_c3-c5_r101_dufpn_1x.pth'
resume_from = None
workflow = [('train', 1)]


#!/usr/bin/python3
# -*- coding: utf-8 -*-

# This creates an hexahedral mesh of uniform grids
# The inputs of this script are:
#     x_length x_n x_offset y_length y_n y_offset fileout

import os, sys, fileinput
import numpy as np
import argparse
from argparse import RawTextHelpFormatter

parser = argparse.ArgumentParser(description="""
      To be made
      """, formatter_class=RawTextHelpFormatter)
parser.add_argument('xlen',  metavar='X Length', type=float,
                    help='Length in the X direction')
parser.add_argument('xn',    metavar='X Elem',   type=int,
                    help='Number of elements in the X direction')
parser.add_argument('xoff',  metavar='X Offset', type=float,
                    help='How much to shift the mesh in the X direction')
parser.add_argument('ylen',  metavar='Y Length', type=float,
                    help='Length in the Y direction')
parser.add_argument('yn',    metavar='Y Elem',   type=int,
                    help='Number of elements in the Y direction')
parser.add_argument('yoff',  metavar='Y Offset', type=float,
                    help='How much to shift the mesh in the Y direction')
parser.add_argument('prefix', type=str,
                    help='Prefix of the files to save the mesh to')

# This is gives the order of the nodes in an elent given a starting indice
vorder = [[0, 0], [1, 0], [0, 1], [1, 1]]

class make_grid:
    def __init__(self, arg):
        self.tp   = []
        self.tpi  = []
        self.surf = []
        self.edge = []
        self.ns   = 0
        # extract the arguments first
        self.xsize = arg.xlen
        self.xn    = arg.xn
        self.dx    = self.xsize/float(self.xn)
        self.xoff  = arg.xoff
        self.ysize = arg.ylen
        self.yn    = arg.yn
        self.dy    = self.ysize/float(self.yn)
        self.yoff  = arg.yoff
        # compute the number of nodes and elements
        self.nnode = (self.xn + 1) * (self.yn + 1)
        self.nelem = self.xn * self.yn
        # create empty index array
        self.ind = np.zeros((self.xn + 1, self.yn + 1), dtype=int)
        # fill array
        g = 0
        for j in range(self.yn + 1):
            for i in range(self.xn + 1):
                self.ind[i, j] = g
                g = g + 1
        # create empty space array
        self.space = np.zeros((self.nnode, 2))
        # fill array
        for j in range(self.yn + 1):
            for i in range(self.xn + 1):
                self.space[self.ind[i,j]] = [i*self.dx + self.xoff, j*self.dy + self.yoff]
    def __getitem__(self, key):
        # if key is of invalid type or value, the list values will raise the error
        return self.space[self.ind[key]]

def compute_topology(g):
    g.tp = np.zeros((g.nelem, 4), dtype=int)
    n    = 0
    for j in range(g.yn):
        for i in range(g.xn):
            for m in range(4):
                g.tp[n, m] = g.ind[i + vorder[m][0], j + vorder[m][1]]
            n = n + 1
    g.tpi = np.zeros((g.xn, g.yn), dtype=int)
    l = 0
    for j in range(g.yn):
        for i in range(g.xn):
            g.tpi[i, j] = l
            l = l + 1
    return

def compute_surface(g):
    # for the -y surface
    surface = np.zeros((g.xn, 4), dtype=int)
    for i in range(g.xn):
        # Get the element #
        surface[i, 0] = g.tpi[i, 0]
        # Get the two node #
        surface[i, 1] = g.ind[i,     0]
        surface[i, 2] = g.ind[i + 1, 0]
        # label the patch, note the indexing is 1 less than in cheart
        surface[i, 3] = 0
        g.ns = g.ns + 1
    g.surf.append(surface)
    # for the +x surface
    surface = np.zeros((g.yn, 4), dtype=int)
    for j in range(g.yn):
        # Get the element #
        surface[j, 0] = g.tpi[g.xn - 1,  j]
        # Get the two node #
        surface[j, 1] = g.ind[g.xn,  j]
        surface[j, 2] = g.ind[g.xn,  j + 1]
        # label the patch, note the indexing is 1 less than in cheart
        surface[j, 3] = 1
        g.ns = g.ns + 1
    g.surf.append(surface)
    # for the +y surface
    surface = np.zeros((g.xn, 4), dtype=int)
    for i in range(g.xn):
        # Get the element #
        surface[i, 0] = g.tpi[g.xn - 1 - i, g.yn - 1]
        # Get the two node #
        surface[i, 1] = g.ind[g.xn - i, g.yn]
        surface[i, 2] = g.ind[g.xn - i - 1,     g.yn]
        # label the patch, note the indexing is 1 less than in cheart
        surface[i, 3] = 2
        g.ns = g.ns + 1
    g.surf.append(surface)
    # for the -x surface
    surface = np.zeros((g.yn, 4), dtype=int)
    for j in range(g.yn):
        # Get the element #
        surface[j, 0] = g.tpi[0,  g.yn - 1 - j]
        # Get the two node #
        surface[j, 1] = g.ind[0,  g.yn - j    ]
        surface[j, 2] = g.ind[0,  g.yn - j - 1]
        # label the patch, note the indexing is 1 less than in cheart
        surface[j, 3] = 3
        g.ns = g.ns + 1
    g.surf.append(surface)
    return

def compute_pads(g):
    if (g.xn % 10) == 0:
        xl = g.xn//10
    else:
        raise ValueError('xn not divisible by 10')
    if (g.yn % 10) == 0:
        yl = g.yn//10
    else:
        raise ValueError('yn not divisible by 10')
    # for the -y surface
    surface = np.zeros((6*xl, 4), dtype=int)
    m = 0
    for i in [1, 2, 4, 5, 7, 8]:
        for k in range(xl):
            # Get the element #
            surface[m, 0] = g.tpi[xl*i + k, 0]
            # Get the two node #
            surface[m, 1] = g.ind[xl*i + k,     0]
            surface[m, 2] = g.ind[xl*i + k + 1, 0]
            # label the patch, note the indexing is 1 less than in cheart
            surface[m, 3] = 4
            m = m + 1
    g.surf.append(surface)
    # for the +x surface
    surface = np.zeros((6*yl, 4), dtype=int)
    m = 0
    for j in [1, 2, 4, 5, 7, 8]:
        for k in range(yl):
            # Get the element #
            surface[m, 0] = g.tpi[g.xn - 1,  yl*j + k]
            # Get the two node #
            surface[m, 1] = g.ind[g.xn,  yl*j + k]
            surface[m, 2] = g.ind[g.xn,  yl*j + k + 1]
            # label the patch, note the indexing is 1 less than in cheart
            surface[m, 3] = 5
            m = m + 1
    g.surf.append(surface)
    # for the +y surface
    surface = np.zeros((6*xl, 4), dtype=int)
    m = 0
    for i in [1, 2, 4, 5, 7, 8]:
        for k in range(xl):
            # Get the element #
            surface[m, 0] = g.tpi[g.xn - 1 - (xl*i + k), g.yn - 1]
            # Get the two node #
            surface[m, 1] = g.ind[g.xn - (xl*i + k), g.yn]
            surface[m, 2] = g.ind[g.xn - (xl*i + k) - 1,     g.yn]
            # label the patch, note the indexing is 1 less than in cheart
            surface[m, 3] = 6
            m = m + 1
    g.surf.append(surface)
    # for the -x surface
    surface = np.zeros((6*yl, 4), dtype=int)
    m = 0
    for j in [1, 2, 4, 5, 7, 8]:
        for k in range(yl):
            # Get the element #
            surface[m, 0] = g.tpi[0,  g.yn - 1 - (yl*j + k)]
            # Get the two node #
            surface[m, 1] = g.ind[0,  g.yn - (yl*j + k)]
            surface[m, 2] = g.ind[0,  g.yn - (yl*j + k) - 1]
            # label the patch, note the indexing is 1 less than in cheart
            surface[m, 3] = 7
            m = m + 1
    g.surf.append(surface)
    return

def main(args=None):
    args=parser.parse_args(args=args)
    fileX = args.prefix + '_FE.X'
    fileT = args.prefix + '_FE.T'
    fileB = args.prefix + '_FE.B'
    g = make_grid(args)
    print('Made a grid with {} nodes'.format(g.nnode))
    compute_topology(g)
    print('Created a topology for the grid with {} elements'.format(g.nelem))
    compute_surface(g)
    print('Generated a the 6 surfaces facing each side')
    # compute_pads(g)
    with open(fileX, 'w') as f:
        f.write('{:>12d}{:>12d}'.format(g.nnode, 2))
        for [x, y] in g.space:
            f.write('\n{:18} {:18}'.format(x, y))
    print('The X file is saved as {}'.format(fileX))
    with open(fileT, 'w') as f:
        f.write('{:>12d}{:>12d}'.format(g.nelem, g.nnode))
        for elem in g.tp:
            f.write('\n')
            for node in elem:
                f.write(' {:>11d}'.format(node + 1))
    print('The T file is saved as {}'.format(fileT))
    with open(fileB, 'w') as f:
        # f.write('{:>12d}'.format(2*(g.yn + g.xn + g.xn//10 * 6 + g.yn//10 * 6)))
        f.write('{:>12d}'.format(g.ns))
        for surf in g.surf:
            for patch in surf:
                f.write('\n')
                for item in patch:
                    f.write(' {:>11d}'.format(item + 1))
    print('The B file is saved as {}'.format(fileB))
    print('                !!!JOB COMPLETE!!!')

if __name__=="__main__":
    main()