import os

from urllib import parse

import requests
from bs4 import BeautifulSoup

class Episode:
    def __init__(self, webtoon_id, no, url_thumbnail, title, rating, created_date):
        self.webtoon_id = webtoon_id
        self.no = no
        self.url_thumbnail = url_thumbnail
        self.title = title
        self.rating = rating
        self.created_date = created_date

    @property
    def url(self):
        """
        self.webtoon_id, self.no 요소를 사용하여
        실제 에피소드 페이지 URL을 리턴해준
        :return:
        """
        url = 'http://comic.naver.com/webtoon/detail.nhn?'
        params = {
            'titleId': self.webtoon_id,
            'no': self.no,
        }
        episode_url = url + parse.urlencode(params)
        return episode_url

    def get_image_url_list(self,):
        print('get_image_url_list strat')
        # 해당 에피소드의 이미지들의 URL문자열들을 리스트에 담아 리턴
        # 1. html 문자열 변수 할당
        # 파일명: episode_detail-{webtoon_id}-{episode_no}.html
        # 없으면 자신의 url property값으로 requests사용 결과를 저장
        # 2.soup객체 생성 후 파싱
        # div.wt_viewer의 자식 img 요소들의 src속성들을 가져옴
        # 적절히 list에 append하고 리턴하자

        # 웹툰 에피소드의 상세 페이지를 저장할 파일 경로
        # 웹툰의 고유ID와 에피소드의 고유ID를 사용해서 겹치지 않는 파일명을 만들어준다
        file_path = 'data/episode_detail-{webtoon_id}-{episode_no}.html'.format(
            webtoon_id=self.webtoon_id,
            episode_no=self.no,
        )
        print('file_path:', file_path)

        # 위 파일이 있는지 검사
        if os.path.exists(file_path):
            print('os.path.exists: True')
            # 있다면 읽어온 결과를 html변수에 할당
            html = open(file_path, 'rt').read()
        else:
            # 없다면 self.url에 requests를 사용해서 요청
            # 요청 결과를 html변수에 할당
            # 요청의 결과를 file_path에 해당하는 파일에 기록
            print('os.path.exists: False')
            print('http get request, url:', self.url)
            response = requests.get(self.url)
            html = response.text
            html = open(file_path, 'rt').write(html)

        # html문자열로 BeautifulSoup객체 생성
        soup = BeautifulSoup(html, 'lxml')

        # img목록을 찾는다. 위치는 "div.wt_viewer > img"
        img_list = soup.select('div.wt_viewer > img')

        # 이미지 URL(src의 값)을 저장할 리스트
        # url_list=[]
        # for img in img_lis:
        #   url_list.append(img.get('srt'))

        # img목록을 순회하며 각 itme(BeautifulSoup Tag object)에서
        # 'src'속성값을 사용해 리스트 생성
        return [img.get('src') for img in img_list]

    def download_all_images(self):
        for url in self.get_image_url_list():
            self.download(url)

    def download(self, url_img):
        """
        :param url_img: 실제 이미지의 URL
        :return:
        """
        # 서버에서 거부하지 않도록 HTTP헤더 중 'Rferer'항목을 채워서 요청
        url_referer = f'http://comic.naver.com/webtoon/list.nhn?titleId={self.webtoon_id}'
        headers = {
            'Referer': url_referer,
        }
        response = requests.get(url_img, headers=headers)

        # 이미지 URL에서 이미지명을 가져옴
        file_name = url_img.rsplit('/', 1)[-1]

        # 이미지가 저장될 폴더 경로, 폴더가 없으면 생성해준다
        dir_path = f'data/{self.webtoon_id}/{self.no}'
        os.makedirs(dir_path, exist_ok=True)
        # os.mkdir = 디렉터리를 생성한다.
        # os.rmdir = 디렉터리를 삭제한다. (단 디렉터리가 비어있어야 삭제 가능)
        # os.unlink = 파일을 지운다
        # os.rename(src, dfc) src라는 이름의 파일을 dfc이름으로 바꾼.
        # mkdir = 디렉터리를 생성하고 makedirs(exist_ok=True) 이미 디렉터리가 생성되어 있거나 권한이 없을경우 예외를 발생시킨다.

        # 이미지가 저장될 파일 경로, 'wb'모드로 열어 이진데이터를 기록한다.
        file_path = f'{dir_path}/{file_name}'
        open(file_path, 'wb').write(response.content)
        # 이진 데이터만 쓰

class Webtoon:
    def __init__(self, webtoon_id):
        self.webtoon_id = webtoon_id
        self._title = None
        self._author = None
        self._description = None
        self._episode_list = list()
        self._html = ''

    def _get_info(self, attr_name):
        if not getattr(self, attr_name):
            self.set_info()
        return getattr(self, attr_name)

    @property
    def title(self):
        return self._get_info('_title')

    @property
    def author(self):
        return self._get_info('_author')

    @property
    def description(self):
        return self._get_info('_description')

    @property
    def html(self):
        if not self._html:
            file_path = 'data/episode_list-{webtoon_id}.html'.format(webtoon_id=self.webtoon_id)
            url_episode_list = 'http://comic.naver.com/webtoon/list.nhn'
            params = {
                'titleId': self.webtoon_id,
            }
            if os.path.exists(file_path):
                html = open(file_path, 'rt').read()
            else:
                response = requests.get(url_episode_list, params)
                print(response.url)
                html = response.text
                open(file_path, 'wt').write(html)
            self._html = html
        return self._html

    def set_info(self):
        """
        자신의 html 속성을 파싱한 결과를 사용해
        자신의 title, author, description속성값을 할당
        :return:
        """

        soup = BeautifulSoup(self.html, 'lxml')

        h2_title = soup.select_one('dib.detail > h2')
        title = h2_title.contents[0].strip()
        author = h2_title.contents[1].get_text(strip=True)
        description = soup.select_one('div.detail > p').get_text(strip=True)

        self._title = title
        self._author = author
        self._description = description

    def crawl_episode_list(self):
        soup = BeautifulSoup(self.html, 'lxml')
        table = soup.select_one('table.viewList')
        tr_list = table.select('tr')
        episode_list = list()
        for index, tr in enumerate(tr_list[1:]):
            if tr.get('class'):
                continue
            url_thumbnail = tr.select_one('td:nth-of-type(1) img').get('src')
            from urllib import parse
            url_detail = tr.select_one('td:nth-of-type(1) > a').get('href')
            query_string = parse.urlsplit(url_detail).query
            query_dict = parse.parse_qs(query_string)
            no = query_dict['no'][0]

            # 현재 tr의 두 번째 td요소의 자식 a요소의 내용
            title = tr.select_one('td:nth-of-type(2) > a').get_text(strip=True)
            # 현재 tr의 세 번째 td요소의 하위 strong태그의 내용
            rating = tr.select_one('td:nth-of-type(3) strong').get_text(strip=True)
            # 현재 tr의 네 번째 td요소의 내용
            created_date = tr.select_one('td:nth-of-type(4)').get_text(strip=True)

            new_episode = Episode(
                webtoon_id=self.webtoon_id,
                no=no,
                url_thumbnail=url_thumbnail,
                title=title,
                rating=rating,
                created_date=created_date,
            )
            episode_list.append(new_episode)
        self._episode_list = episode_list

    @property
    def episode_list(self):
        if not self._episode_list:
            self.crawl_episode_list()
        return self._episode_list

if __name__ == '__main__':
    webtoon1 = Webtoon(651673)
    print(webtoon1.title)
    print(webtoon1.author)
    print(webtoon1.description)
    e1 = webtoon1.episode_list[0]
    e1.download_all_images()
# if name main 쓰는 이유 모듈을 직접 실행시킬때 실행되게 하고싶으면 사용
#


## module choleski
''' L = choleski(a)
    Choleski decomposition: [L][L]transpose = [a]

    x = choleskiSol(L,b)
    Solution phase of Choleski's decomposition method
'''
from numpy import dot
from math import sqrt
import error

def choleski(a):
    n = len(a)
    for k in range(n):
        try:
            a[k,k] = sqrt(a[k,k] - dot(a[k,0:k],a[k,0:k]))
        except ValueError:
            error.err('Matrix is not positive definite')
        for i in range(k+1,n):
            a[i,k] = (a[i,k] - dot(a[i,0:k],a[k,0:k]))/a[k,k]
    for k in range(1,n): a[0:k,k] = 0.0
    return a

def choleskiSol(L,b):
    n = len(b)
  # Solution of [L]{y} = {b}
    for k in range(n):
        b[k] = (b[k] - dot(L[k,0:k],b[0:k]))/L[k,k]
  # Solution of [L_transpose]{x} = {y}
    for k in range(n-1,-1,-1):
        b[k] = (b[k] - dot(L[k+1:n,k],b[k+1:n]))/L[k,k]
    return b


import time

import logging
import os
print(os.getcwd())
logging.basicConfig(filename='example.log',level=logging.DEBUG)
logging.debug('This message should go to the log file')
logging.info('So should this')
logging.warning('And this, too')

i = 0
while True:
	print(i)
	logging.info(i)
	i+=1
	print("This prints once 1s.")
	time.sleep(1)


import curses

from exceptions import Game2048NoFreeLocationsLeft
from game_2048.game import Game2048

class Game2048Curses(Game2048):

    KEY_QUIT = ord('q')
    KEY_RESET = ord('r')
    KEY_DOWN = 66
    KEY_UP = 65
    KEY_LEFT = 68
    KEY_RIGHT = 67

    KEY_DIRECTION_MAPPING = {
        KEY_DOWN: Game2048.DIRECTION_DOWN,
        KEY_UP: Game2048.DIRECTION_UP,
        KEY_RIGHT: Game2048.DIRECTION_RIGHT,
        KEY_LEFT: Game2048.DIRECTION_LEFT
    }

    templates = {
        'score': 'Score: {score}',
        'level_complete': 'Congratulations your highest item is {highest}!. The new threshold is now {threshold}.',
        'logo': """
 ___   ___    __   ___
(__ \ / _ \  /. | ( _ )
 / _/( (_) )(_  _)/ _ \\
(____)\___/   (_) \___/

""",
        'help_text': 'Use the arrow keys to play. Press q for quit or r to restart.',
        'game_over': 'Game over! Your score is {score}. Press r to restart the game.'
    }

    _std_scr = None
    _highest_value_threshold = 2048

    _message = None

    _line_position = 0

    def __init__(self, auto_start=True, *args, **kwargs):

        super(Game2048Curses, self).__init__(*args, **kwargs)

        if auto_start:
            self._start()

    def _start(self):

        self._std_scr = curses.initscr()
        curses.cbreak()

        try:
            self._run()
        finally:
            curses.endwin()

    def _run(self):

        self._refresh_screen()

        key_ = None

        while key_ != self.KEY_QUIT:

            key_ = self._std_scr.getch()

            if key_ in self.KEY_DIRECTION_MAPPING:

                try:
                    self.move(self.KEY_DIRECTION_MAPPING[key_])
                except Game2048NoFreeLocationsLeft:
                    self._display_game_over_screen()
                    continue

                if self._highest_value >= self._highest_value_threshold:
                    self._highest_value_threshold *= 2

                    self._message = self.templates['level_complete'].format(highest=self._highest_value,
                                                                            threshold=self._highest_value_threshold)

            if key_ == self.KEY_RESET:
                self.reset()

            self._refresh_screen()

    def _update_game(self):

        offset = 1

        self._write_line((8 * '-') * self._grid_size)

        for row_i, row in enumerate(self._grid):
            str_ = ''
            for column in row:
                str_ += '{: ^6} |'.format(column if column else '')

            self._write_line(str_)
            self._write_line((8 * '-') * self._grid_size)

    def _update_logo(self):

        for line in self.templates['logo'].split("\n"):
            self._write_line(content=line)

    def _display_game_over_screen(self):

        self._line_position = 0
        self._std_scr.clear()

        self._update_logo()
        self._write_line(self.templates['game_over'].format(score=self._score))

    def _refresh_screen(self):

        self._line_position = 0
        self._std_scr.clear()

        self._update_logo()
        self._write_line(self.templates['score'].format(score=self._score))
        self._write_line(self._message or '')
        self._message = None
        self._update_game()

        self._line_position += 1

        self._write_line(content=self.templates['help_text'])

        self._write_line('')

    def _write_line(self, content, x_offset=0):

        self._std_scr.addstr(self._line_position, x_offset, content)
        self._line_position += 1

if __name__ == '__main__':

    Game2048Curses()


def neighbordInterface(netconnect, listInterfaces):
    listNeighbord = []
    for interface in listInterfaces:
        output = netconnect.send_command("sh cdp neighbors " + str(interface))
        listNeighbord.append(output)
    return listNeighbord

def neighbordCapabilities(netconnect, listInterfaces):
    listNeighbord = []
    for interface in listInterfaces:
        output = netconnect.send_command("sh cdp neighbors " + str(interface) + " detail | i Capabilities: ")
        capabilities = str(output)
        capabilities = capabilities.split("Capabilities: ")[1]
        listNeighbord.append(interface + ";Capabilities:" + capabilities)
    return listNeighbord

def neighbordResume(netconnect, listInterfaces):
    listNeighbord = []
    for interface in listInterfaces:
        output = str(netconnect.send_command("sh cdp neighbors " + str(interface) + " detail | i (IP address: | Capabilities: |Device ID|Total cdp entries displayed : )"))
        if "Total cdp entries displayed : 1" in output:
            deviceID = output.split("\n")[0].split("Device ID: ")[1]
            ip = output.split("\n")[1].split("IP address: ")[1]
            capabilities = output.split("\n")[2].split("Capabilities: ")[1]
            listNeighbord.append(interface + ";" + deviceID + ";" + ip + ";" + capabilities)
        else:
            listNeighbord.append(interface + ";" + "unknown")
    return listNeighbord


import httplib2
import json
import sys
import mysql.connector
import os
from ftplib import FTP
import gzip
from scythepkg.helpers.fastahelper import FastaParser
import os

def specInfo():
    pass
    http = httplib2.Http(".cache")
    server = "http://rest.ensembl.org"
    ext = "/info/species"
    resp, content = http.request(server+ext, method="GET", headers={"Content-Type":"application/json"})
    data = json.loads(content.decode("utf8"))
    return data

def getSequencesFromFTP(outdir, release, specieslist=[]):
    path=outdir+os.sep+"fa"
    print(outdir, release, specieslist)
    if len(release)==1:
        release = release[0]
        dirlist = []
        if not os.path.isdir(path):
                os.makedirs(path)
        if (specieslist==[]):
            print("Warning: No set of species selected")
            exit(0)
        ftp = FTP('ftp.ensembl.org')
        ftp.login()
        ftp.cwd('pub') #ftp://ftp.ensembl.org/pub/
        ftp.retrlines('LIST', callback=dirlist.append)           # list directory contents
        dirlist = [r for r in dirlist if "release-"+str(release)  in r and "fasta" in r]
        print(dirlist)
        if(len(dirlist )==0):
            print("Nothing available for release-"+str(release))
            exit(1)
        ftprelhome = dirlist[0].split(" ")[-1]
        ftp.cwd(ftprelhome)
        ftp.retrlines('LIST', callback=dirlist.append)

        for s in specieslist:
            tmp = [d for d in dirlist if s in d][0]
            spec = tmp.split(" ")[-1]
            ftp.cwd(spec)
            ftp.cwd('pep')
            falist = []
            ftp.retrlines('LIST', callback=falist.append)
            falist = [f for f in falist if "all.fa" in f][0]
            fafile = falist.split(" ")[-1]
            outfaname = spec+".fa.gz"
            outfa = open(outfaname,'wb')
            ftp.retrbinary("RETR "+fafile,outfa.write)
            outfa.close()
            xtract(outfaname, path)
            ftp.cwd("/pub/"+ftprelhome)
       #nucleotides
        for s in specieslist:
            tmp = [d for d in dirlist if s in d][0]
            spec = tmp.split(" ")[-1]
            ftp.cwd(spec)
            ftp.cwd('cds')
            falist = []
            ftp.retrlines('LIST', callback=falist.append)
            falist = [f for f in falist if "all.fa" in f][0]
            fafile = falist.split(" ")[-1]
            outfaname = spec+".cds.all.fa.gz"
            print(outfaname, "cds\n")
            print(falist, "falist\n")
            outfa = open(outfaname,'wb')
            ftp.retrbinary("RETR "+fafile,outfa.write)
            outfa.close()
            if not os.path.exists(path+"_cds"):
                    os.makedirs(path+"_cds")
            xtract(outfaname, path+"_cds")
            ftp.cwd("/pub/"+ftprelhome)
        ftp.quit()

    else:
        print("Releases:")
        for a in zip(release,specieslist):
            print(a)
        for a in zip(release,specieslist):
            getSequencesFromFTP(outdir, release=[a[0]], specieslist=[a[1]])

def xtract(cfile, outpath = "."):

    if cfile.endswith('.gz'):
        dzf = ".".join(cfile.split(".")[:-1])
        #gzip
        gzf = gzip.open(cfile,'rb')
        content = gzf.read()
        out = open(outpath+os.sep+dzf,'wb')
        out.write(content)
        out.close
        gzf.close()
    else:
        print("Can't extract "+cfile)

###########################
class Pep(object):
    def __init__(self,pep,gene,length ):
        self.gene = gene
        self.pep  = pep
        self.length = length
        self.isLongest = None

def prepareLocFromFasta(fasta, outpath, specname):
    genes = {}
    longest = {}
    print(fasta, outpath)
    if not os.path.isdir(outpath):
        os.makedirs(outpath)

    if not outpath:
        out = fasta+".loc"
    else:
        out = outpath+os.sep+specname+".loc"
    fp = FastaParser()
    out = open(out,'w')
    print(out)
    for i in fp.read_fasta(fasta):
        tmp = i[0].split(" ")
        geneid = [t for t in tmp if "gene:" in t]
        geneid = geneid[0].split("gene:")[-1]
        proteinid = tmp[0]
        protlen = len(i[1])
        peptide = Pep(proteinid,geneid,protlen)
        try:
            genes[geneid].append(peptide)
            if peptide.length > longest[geneid].length:
                longest[geneid].isLongest =False
                longest[geneid]=peptide
                peptide.isLongest = True
        except KeyError as e:
            genes[geneid] = [peptide]
            longest[geneid] = peptide
            peptide.isLongest=True
    for g in genes:
        print(g)
        firstcol = [w for w in genes[g] if w.isLongest==True ]
        restcol = [w for w in genes[g] if w.isLongest!=True ]
        s = firstcol[0].gene+"\t"+firstcol[0].pep+"\t"
        s+="\t".join([v.pep for v in restcol])
        s+="\n"
        out.write(s)
        print(s)
    out.close()
    print("done")
#######################


""" Script for removing NotificationSubscriptions from registrations.
    Registrations shouldn't have them!
"""
import logging
import sys

import django
django.setup()

from website.app import init_app
from django.apps import apps

logger = logging.getLogger(__name__)

def remove_notification_subscriptions_from_registrations(dry_run=True):
    Registration = apps.get_model('osf.Registration')
    NotificationSubscription = apps.get_model('osf.NotificationSubscription')

    notifications_to_delete = NotificationSubscription.objects.filter(node__type='osf.registration')
    registrations_affected = Registration.objects.filter(
        id__in=notifications_to_delete.values_list(
            'node_id', flat=True
        )
    )
    logger.info('{} NotificationSubscriptions will be deleted.'.format(notifications_to_delete.count()))
    logger.info('{} Registrations will be affected: {}'.format(
        registrations_affected.count(),
        list(registrations_affected.values_list('guids___id', flat=True)))
    )

    if not dry_run:
        notifications_to_delete.delete()
        logger.info('Registration Notification Subscriptions removed.')

if __name__ == '__main__':
    dry_run = '--dry' in sys.argv
    init_app(routes=False)
    remove_notification_subscriptions_from_registrations(dry_run=dry_run)


class Solution:
    def diagonalSort(self, mat: List[List[int]]) -> List[List[int]]:
        if not mat: return [[]]
        m = len(mat)
        n = len(mat[0])
        dlen = m + n - 1
        diff = n - m
        diagonals = []
        for i in range(dlen):
            diagonals.append([])
        for i in range(m):
            for j in range(n):
                diagonals[diff + i-j].append(mat[i][j])

        for i in range(dlen):
            diagonals[i] = sorted(diagonals[i])

        for i in range(m):
            for j in range(n):
                i_d = diff + i - j
                j_d = min(j, i)
                mat[i][j] = diagonals[i_d][j_d]

        return mat


from flask import Flask, request

app = Flask(__name__)

@app.route("/")
def index():
    return '''
<form action="/info" method="GET">
<input type="text" name="user" placeholder="Full Name">

<input type="email" name="email" placeholder="Email">

<input type="password" name="pass" placeholder="password">

<input type="submit" value="Send">
</form>
'''

@app.route("/info",methods=['GET','POST'])
def info():
    if request.method == "GET":
        return "
GET Method
Hello "+ request.args.get('user') + \
    " Your Email:"+request.args.get('email')
    else:
        return "
POST Method
Hello "+ request.form['user'] + \
    " Your Email:"+request.form['email']

app.run(debug=True)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import fnmatch
from tinytag import TinyTag
import os, re, requests, sys
import os.path
from os import path
import json
import pygame
import threading, random
NEXT = pygame.USEREVENT + 1
os.environ["SDL_VIDEODRIVER"] = "dummy"

class MuuzikPlayer:
    def __init__(self, config, language):
        self.language = language["musicPlayer"]
        self.information_path = config["information_path"]
        self.pattern = '*.mp3'
        self.playlist = []
        self.artist_list = []
        self.song_list = []
        self.current_track = 0
        self.pathToMusic = config['music_path'].split("\n")
        self.musicFile = []
        self.backgroundthread = None
        self.train_thread = None
        self.running = False
        self.stopTheMusic = False

    def setLanguage(self, language):
        self.language = language

    def scanMusic(self):
        print('test 1')
        numberOfSongsBeforeScan = self.countMySongs()

        rootPaths = ['/home/pi/Music']
        songlist = []
        artistlist = []
        album_list = []
        titlelist = []
        genre_list = []
        pattern = '*.mp3'
        print(rootPaths)
        for rootPath in rootPaths:
            for root, dirs, files in os.walk(rootPath):
                for filename in fnmatch.filter(files, pattern):
                    if path.exists(os.path.join(root, filename)):
                        try:
                            audiofile = TinyTag.get(os.path.join(root, filename))
                            print(audiofile)
                            album = str(audiofile.album)
                            title = str(audiofile.title)
                            genre = str(audiofile.genre)
                            albumArtist = str(audiofile.albumartist)
                            artist = str(audiofile.artist)
                            year = str(audiofile.year)

                            a = albumArtist
                            print(a)
                            if albumArtist == 'None':
                                print(artist)
                                a = artist
                            if title.strip() != '' and title.strip() not in titlelist:
                                titlelist.append(title.strip())

                            if album.strip() != '' and album.strip() not in album_list:
                                album_list.append(album.strip())

                            if genre.strip() not in genre_list:
                                genre_list.append(genre)

                            # print(genre + ' ' + a + ' ' + title + ' ' + os.path.join(root, filename))

                            if (a.strip()) not in artistlist:
                                artistlist.append(a)

                            songlist.append({
                                'song': '{} {} {} {}  {} {}  {} '.format(album.strip(), title.strip(), genre, albumArtist, artist, year,
                                                                         os.path.join(root, filename)),
                                'path': os.path.join(root, filename),
                                'artist': '{}'.format(a),
                                'song_name': '{}'.format(title)
                            })
                        except:
                            print('error')

        with open(self.information_path + '/musicFound.json', 'w') as f:
            json.dump(songlist, f, indent=4)

        print('writing in root')
        with open('/root/.config/rhasspy/profiles/de/slots/music_artists', "w") as txt_file:
            for line in artistlist:
                txt_file.write(re.sub('[^a-zA-Z0-9 ]', '', line) + "\n")
        print('finisched writing in root')

        with open(self.information_path + '/music_artists.ini', "w") as txt_file:
            for line in artistlist:
                txt_file.write(re.sub('[^a-zA-Z0-9 ]', '', line) + "\n")

        with open('/root/.config/rhasspy/profiles/de/slots/music_titles', "w") as txt_file:
            for line in titlelist:
                txt_file.write(re.sub('[^a-zA-Z0-9 ]', '', line) + "\n")

        with open(self.information_path + '/music_titles.ini', "w") as txt_file:
            for line in titlelist:
                txt_file.write(re.sub('[^a-zA-Z0-9 ]', '', line) + "\n")

        with open('/root/.config/rhasspy/profiles/de/slots/music_albums', "w") as txt_file:
            for line in album_list:
                txt_file.write(re.sub('[^a-zA-Z0-9 ]', '', line) + "\n")

        with open(self.information_path + '/music_albums.ini', "w") as txt_file:
            for line in album_list:
                txt_file.write(re.sub('[^a-zA-Z0-9 ]', '', line) + "\n")

        with open('/root/.config/rhasspy/profiles/de/slots/music_genres', "w") as txt_file:
            for line in genre_list:
                txt_file.write(re.sub('[^a-zA-Z0-9 ]', '', line) + "\n")

        with open(self.information_path + '/music_genres.ini', "w") as txt_file:
            for line in genre_list:
                txt_file.write(re.sub('[^a-zA-Z0-9 ]', '', line) + "\n")

        numberOfSongsAfterScan = self.countMySongs()
        if (numberOfSongsAfterScan < numberOfSongsBeforeScan):
            print(' neue Lieder')
            text = self.language["removedSongs"].format(numberOfSongsBeforeScan - numberOfSongsAfterScan)
            self.train()
            return text

        if (numberOfSongsAfterScan > numberOfSongsBeforeScan):
            print(' keine neue Lieder')
            text = self.language["foundNewSongs"].format(numberOfSongsAfterScan - numberOfSongsBeforeScan)
            #self.train(hermes, intentMessage)
            return text

            #return self.language["scanningIsDone"]
        print(' ende Scanning')
        #self.train(hermes, intentMessage)
        #TODO add found artists to slot artists
        #TODO check which songs are new and only add these artists to the artists slot
        # self.train('', hermes, intentMessage)
        return self.language["scanningIsDone"]

    def train(self):
        print('TRAINING')
        # hermes.publish("rhasspy/asr/" + intentMessage["site_id"] + "/train")
        r = requests.post('http://rhasspy:12101/api/train')
        #hermes.publish("hermes/tts/say", json.dumps({"text": 'Ich trainiere jetzt', "siteId": intentMessage["site_id"]}))
        return r

    def playPlaylist(self, playListName):
        if  path.isdir(self.information_path + '/playlists') == False:
            os.mkdir(self.information_path + '/playlists')

        musicFile = open(self.information_path + '/playlists')
        allSongs= json.load(musicFile)
        musicFile.close()
        return len(allSongs)

    def addToFavourites(self, playListName):
        print(self)
        if self.running:
            song = self.playlist(self.current_track)
            playListPath = self.information_path + '/Favourites.m3u'
            if path.exists(playListPath) == False:
                file = open(playListPath, "w+")
                file.append()
            else:
                with open(playListPath, 'a') as file:
                    file.write(song)
            return self.language["addMp3ToPlaylist"]["songAddedToPlaylist"]
        else:
            return self.language["addMp3ToPlaylist"]["noMusicIsRunning"]

    def addMp3ToPlaylist(self, playListName):
        if self.running:
            song = self.playlist(self.current_track)
            playListPath = self.information_path + '/playlists' + playListName +'.m3u'
            if path.exists(playListPath) == False:
                file = open(playListPath, "w+")
                file.append()
            else:
                with open(playListPath, 'a') as file:
                    file.write(song)
            return self.language["addMp3ToPlaylist"]["songAddedToPlaylist"]
        else:
            return self.language["addMp3ToPlaylist"]["noMusicIsRunning"]

    def removeMp3ToPlaylist(self, song):
        song = self.playlist(self.current_track)
        return song

    def getMp3FilesFromPlaylist(self, pathToPlayList):
        songs = []
        with open(pathToPlayList) as fp:
            line = fp.readline()
            cnt = 1
            while line:
                print("Line {}: {}".format(cnt, line.strip()))
                line = fp.readline().lower()
                if (".mp3" in line):
                    songs.append(line)
                cnt += 1
        return songs

    def countMySongs(self):
        try:
            musicFile = open(self.information_path  + '/musicFound.json')
            allSongs= json.load(musicFile)
            musicFile.close()
            return len(allSongs)
        except:
            return 0

    def play(self, music):
        self.playlist = []
        print(music)

        if not str(os.path.exists(self.information_path + '/musicFound.json')):
            print('play Musik')
            self.scanMusic()

        if len(self.musicFile) == 0:
            musicFile = open(self.information_path + '/musicFound.json')
            self.musicFile = json.load(musicFile)
            musicFile.close()

        title = music["title"].lower()
        album = music["album"].lower()
        artist = music["artist"].lower()
        genre = music["genre"].lower()

        i = 0
        while i < len(self.musicFile) -1:
            path = self.musicFile[i]['path']
            artist_temp = self.musicFile[i]['artist'].lower()
            song_temp = self.musicFile[i]['song_name'].lower()

            song = ' ' + self.musicFile[i]['song'].lower()
            i += 1

            if song.find(artist) > 0 \
                    or song.find(album) > 0 \
                    or song.find(title) > 0 \
                    or song.find(genre) > 0:
                self.playlist.append(path)
                self.artist_list.append(artist_temp)
                print(' artist_temp', artist_temp)
                print(' song_temp', song_temp)
                self.song_list.append(song_temp)

        self.current_track = 0
        if len(self.playlist) == 0:
            return self.language["notFound"]
        else:
            return self.playSong()

    def next(self):
        self.stop()
        print(' self.current_track', self.current_track)
        if self.current_track < len(self.playlist) - 1:
            self.current_track += 1
            self.playSong()
            print('AFTER 1 self.current_track', self.current_track)
            return self.language['nextSong']
        else:
            self.current_track = 0
            self.stop()
            print('AFTER 2 self.current_track', self.current_track)
            return self.language['playListEnded']

    def previous(self):
        if self.current_track == 0:
            self.current_track = len(self.playlist) - 1
        else:
            self.current_track -= 1
        self.playSong()
        return self.language['previuosSong']

    #Not implemented yet
    def repeat(self):
        self.playSong()
        return 'okay ich wieder hole'

    def stop(self):
        self.running = False
        self.stopTheMusic = True
        try:
            pygame.mixer.music.stop()
            pygame.quit()
        except Exception as ex:
            print('error while stopping but that is fine')
        # pygame.mixer.set_endevent(type)
        self.backgroundthread = None
        return self.language['stoppedTheMusic']

    def play_music_with_stars(self, stars):
        star_list_file_name = self.information_path + '/' + str(stars) + '_stars_list.json'
        star_list_file = open(star_list_file_name)
        star_list_file_list = json.load(star_list_file)
        star_list_file.close()
        self.playlist = star_list_file_list
        self.playSong()
        return  self.language['startedPlaying']

    def add_stars(self, stars):
        artist = self.artist_list[self.current_track]
        print(artist)
        song = self.song_list[self.current_track]
        print(song)
        path = self.playlist[self.current_track]
        print(path)
        star_list_file_name = self.information_path + '/' + str(stars) + '_stars_list.json'

        if not os.path.exists(star_list_file_name):
            print('Make new File')
            star_list_file_list = []
        else:
            star_list_file = open(star_list_file_name)
            star_list_file_list = json.load(star_list_file)
            star_list_file.close()
            print('loading file')

        write_new_file = True
        i = 0
        print(' total in file' +  str(len(star_list_file_list)))
        while i < len(star_list_file_list):
            #artist = star_list_file_list[i]['artist']
            #song = star_list_file_list[i]['song']
            #path = star_list_file_list[i]['path']

            if path == star_list_file_list[i]['path']:
                print(' found file')
                write_new_file = False
            i += 1

        if write_new_file:
            star_list_file_list.append({
                "song": song,
                "artist": artist,
                "path": path
            })
            with open(star_list_file_name, 'w') as f:
                json.dump(star_list_file_list, f, indent=4)

        print(star_list_file_list)

        return self.language['addedStars'].format(str(stars))

    def what_is_playing(self):
        print(self.current_track)
        print(self.playlist[self.current_track])
        artist = self.artist_list[self.current_track]
        song = self.song_list[self.current_track]

        print(artist)
        print(song)

        return self.language['whatIsPlaying'].format(song, artist)

    def playSong(self):
        # I am sure this is not the right way to use a thread but I have no clue how to play the music while
        # continueing the other scripts
        self.backgroundthread = None
        self.backgroundthread = threading.Thread(target=self.playing)
        self.backgroundthread.start()
        return self.language['startedPlaying']

    def playing(self):
        self.stopTheMusic = False
        # start first track
        self.tracks_number = len(self.playlist)
        NEXT = pygame.USEREVENT + 1
        print(' self.current_track ')
        print(self.current_track)
        print(' self.playlist ')
        print(self.playlist)

        # get the right speed of each song
        audiofile = TinyTag.get(self.playlist[self.current_track])
        pygame.mixer.init(frequency=audiofile.samplerate)
        print('-------------------------------------------------------------------')
        # sometimes needed I dont know why but on my rasp 4 I dont need this line anymore
        #try:
            #screen = pygame.display.set_mode((400, 300))
       # except Exception as ex:
        #    print('error but that is fine')

        print('-------------------------------------------------------------------')
        print(self.playlist)
        # start first track
        if path.exists(self.playlist[self.current_track]):
            pygame.mixer.music.load(self.playlist[self.current_track])
            pygame.mixer.music.play()
            pygame.mixer.music.set_endevent(NEXT)

            self.running = True

            while pygame.mixer.music.get_busy():
               nothing = True

            if self.stopTheMusic == False:
                self.next()
        else:
            self.stop()


import requests
import pandas as pd
from bs4 import BeautifulSoup

filmdata = []
# Set the website URL
year = ['00','01','02','03','04','05','06','07','08','09','10','11','12','13','14','15','16','17','18','19','20','21','22']
for i in year:
    url = (f"https://en.wikipedia.org/wiki/List_of_American_films_of_20{i}")

    # Make a GET request to the website
    response = requests.get(url)

    # Parse the HTML content of the website
    soup = BeautifulSoup(response.content, "html.parser")
    # print(soup)
    movie_titles = []
    # Find all elements with the class "movie-title"
    rows = soup.find_all('tr')
    for row in rows:
        title = row.find('a')
        if title:
            movie_titles.append(title.text)
    print(movie_titles)
    movie_titles = movie_titles[29:]
    movie_titles = movie_titles[:len(movie_titles) - 30]
    print(movie_titles)
    for i in movie_titles:
        filmdata.append(i)

# print(movie_titles)
# Print the list of movie titles
# print(cleaned_titles)

df = pd.DataFrame(filmdata)
df.to_csv("MovieTitles2000-2021_American.csv", index=False)


import os
import argparse
import tqdm
import torch
import numpy as np
import yaml

from mood.dpa.pg_networks import ProgGrowStageType, NetworkType
from mood.dpa.pg_decoders import DECODER_NETWORKS
from mood.dpa.pg_encoders import ENCODER_NETWORKS
from mood.dpa.data_generators import ProgGrowImageGenerator, MixResolution
from mood.utils.datasets import DatasetType, DATASETS
from mood.utils.transforms import TRANSFORMS
from mood.dpa.optimizer import Optimizer
from mood.dpa.rec_losses import ReconstructionLossType
from mood.dpa.pg_rec_losses import PG_RECONSTRUCTION_LOSSES
from mood.utils.utils import save_yaml

torch.backends.cudnn.deterministic = True

class DeepPerceptualAutoencoder:
    def __init__(self, config):
        self.config = config
        self.verbose = config['verbose']
        self.random_seed = config['random_seed']
        self.finetune_from = config['finetune_from']
        self.mode_3d = config.get('mode_3d', False)

        self.checkpoint_root = config['checkpoint_root']
        self.log_root = config['log_root']
        save_yaml(config, os.path.join(self.checkpoint_root, 'train_config.yaml'))
        save_yaml(config, os.path.join(self.log_root, 'train_config.yaml'))

        self.num_workers = config['num_workers']
        self.max_image_res = config['max_image_res']
        self.initial_image_res = config['initial_image_res']
        self.image_dim = config['image_dim']
        self.latent_res = config['latent_res']
        self.latent_dim = config['latent_dim']

        trns_iter = config['trns_iter']
        stab_iter = config['stab_iter']
        self.iters = {
            ProgGrowStageType.stab: stab_iter,
            ProgGrowStageType.trns: trns_iter
        }

        self.iters_per_res = {
            int(res): iters for res, iters in config.get('iters_per_res', {}).items()
        }
        # check iters_per_res
        error_msg = """"
        Incorrect 'iters_per_res' param. Correct example:
        iters_per_res:
          64:
            trns: 1000
            stab: 2000
        """
        for res, values in self.iters_per_res.items():
            if not isinstance(values, dict):
                raise ValueError(error_msg)
            for stage, val in values.items():
                if stage != 'stab' and stage != 'trns':
                   raise ValueError(error_msg)

        self.log_iter = config['log_iter']
        self.val_iter = config['val_iter']
        self.early_stopping_patience = config['early_stopping_patience']
        self.early_stopping_min_delta = config['early_stopping_min_delta']
        self.image_sample_iter = config['image_sample_iter']

        self.batch_sizes = {int(res): batch_size for res, batch_size in config['batch_sizes'].items()}
        # check batch sizes
        _res = self.initial_image_res
        while _res <= self.max_image_res:
            if _res not in self.batch_sizes.keys():
                raise ValueError(f"The batch size isn't specified for the resolution {_res}")
            _res = int(_res * 2)

        "=========================================== initialize ======================================================="

        if self.verbose:
            print(yaml.dump(self.config, default_flow_style=False))

        if self.random_seed is not None:
            torch.manual_seed(self.random_seed)
            np.random.seed(self.random_seed)

        self.stage = ProgGrowStageType.stab
        self.resolution = self.initial_image_res
        self.progress = 0
        self.n_iter = 0
        self.best_val_iter = 0
        self.best_val_loss = 10e6
        self.batch_size = int(self.batch_sizes[self.resolution])

        from mood.utils.loggers import Logger
        os.makedirs(self.checkpoint_root, exist_ok=True)
        self.logger = Logger(self.log_root)

        "=========================================== create data dpa ================================================"

        dataset_type = config['train_dataset']['dataset_type']
        dataset_kwargs = config['train_dataset']['dataset_kwargs']
        transform_kwargs = config['train_dataset']['transform_kwargs']

        transform = TRANSFORMS[DatasetType[dataset_type]](**transform_kwargs)
        dataset = DATASETS[DatasetType[dataset_type]](
            transform=transform,
            **dataset_kwargs
        )

        self.pg_image_gen = ProgGrowImageGenerator(dataset, self.max_image_res,
                                                   batch_size=self.batch_size,
                                                   num_workers=self.num_workers, inf=True, mode_3d=self.mode_3d)

        dataset_type = config['val_dataset']['dataset_type']
        dataset_kwargs = config['val_dataset']['dataset_kwargs']
        transform_kwargs = config['val_dataset']['transform_kwargs']

        transform = TRANSFORMS[DatasetType[dataset_type]](**transform_kwargs)
        val_dataset = DATASETS[DatasetType[dataset_type]](
            transform=transform,
            **dataset_kwargs
        )
        self.val_pg_image_gen = ProgGrowImageGenerator(val_dataset, self.max_image_res,
                                                       batch_size=self.batch_size,
                                                       num_workers=self.num_workers, inf=False, mode_3d=self.mode_3d)

        "============================================= create networks ================================================"

        mtype = config['enc']['type']
        kwargs = config['enc']['kwargs']
        self.enc_pg_networks = ENCODER_NETWORKS[NetworkType[mtype]](
            max_input_res=self.max_image_res,
            output_res=self.latent_res,
            input_dim=self.image_dim,
            output_dim=self.latent_dim,
            mode_3d=self.mode_3d,
            **kwargs
        )

        mtype = config['dec']['type']
        kwargs = config['dec']['kwargs']
        self.dec_pg_networks = DECODER_NETWORKS[NetworkType[mtype]](
            input_res=self.latent_res,
            max_output_res=self.max_image_res,
            input_dim=self.latent_dim,
            output_dim=self.image_dim,
            mode_3d=self.mode_3d,
            **kwargs
        )

        if self.verbose:
            print("================================== MODELS ON LARGEST RESOLUTION ==================================")
            print("====================== Encoder =============================")
            print(self.enc_pg_networks.get_net(ProgGrowStageType.stab, self.max_image_res))
            print("====================== Decoder  ===========================")
            print(self.dec_pg_networks.get_net(ProgGrowStageType.stab, self.max_image_res))

        self.dec = self.dec_pg_networks.get_net(self.stage, self.resolution).cuda()
        self.enc = self.enc_pg_networks.get_net(self.stage, self.resolution).cuda()

        "=========================================== create reconstruction losses ====================================="

        loss_type = config['image_rec_loss']['loss_type']
        loss_kwargs = config['image_rec_loss']['loss_kwargs']
        self.image_rec_loss = PG_RECONSTRUCTION_LOSSES[ReconstructionLossType[loss_type]](
            max_resolution=self.max_image_res, mode_3d=self.mode_3d, **loss_kwargs)
        self.image_rec_loss.set_stage_resolution(self.stage, self.resolution)

        "=========================================== create optimizers ================================================"

        adam_kwargs = config['adam_kwargs']
        self.optimizer = Optimizer(
            enc_params=self.enc.parameters(),
            dec_params=self.dec.parameters(),
            image_rec_loss=self.image_rec_loss,
            adam_kwargs=adam_kwargs,
        ).cuda()

        "=========================================== data for logging ================================================="

        self.pg_image_gen.set_stage_resolution(ProgGrowStageType.stab, self.max_image_res, batch_size=self.batch_size)
        self.display_x = next(self.pg_image_gen)[:4].cpu()
        self.pg_image_gen.set_stage_resolution(self.stage, self.resolution, self.batch_size)
        if self.val_pg_image_gen is not None:
            self.val_pg_image_gen.set_stage_resolution(self.stage, self.resolution, self.batch_size)
        self.mix_res_module = MixResolution(self.stage, self.resolution, self.max_image_res,  mode_3d=self.mode_3d)

        "=========================================== initialization ==================================================="

        self.updating_models_during_training = \
            [self.enc, self.dec, self.image_rec_loss,
             self.pg_image_gen, self.val_pg_image_gen, self.mix_res_module]

        if self.finetune_from is not None:
            self.load_state(torch.load(self.finetune_from))
        else:
            self._init_stage()

    def _create_new_stage(self):
        del self.dec, self.enc, self.updating_models_during_training
        if self.stage == ProgGrowStageType.stab:
            self.stage = ProgGrowStageType.trns
            self.resolution *= 2
        else:
            self.stage = ProgGrowStageType.stab
        self.progress = 0
        self._init_stage()

    def _init_stage(self):
        self.batch_size = int(self.batch_sizes[self.resolution])

        "=========================== update data generators and global params ========================================="

        self.pg_image_gen.set_stage_resolution(self.stage, resolution=self.resolution, batch_size=self.batch_size)
        self.val_pg_image_gen.set_stage_resolution(self.stage, resolution=self.resolution, batch_size=self.batch_size)
        self.mix_res_module = MixResolution(self.stage, self.resolution, self.max_image_res, mode_3d=self.mode_3d)

        "========================================== update networks ==================================================="

        self.enc = self.enc_pg_networks.get_net(self.stage, self.resolution).cuda()
        self.dec = self.dec_pg_networks.get_net(self.stage, self.resolution).cuda()
        self.image_rec_loss.set_stage_resolution(self.stage, self.resolution)

        self.optimizer.set_new_params(
            dec_params=self.dec.parameters(),
            enc_params=self.enc.parameters(),
            image_rec_loss=self.image_rec_loss)

        "========================================== other updates ==================================================="

        self.updating_models_during_training = \
            [self.enc, self.dec, self.image_rec_loss,
             self.pg_image_gen, self.val_pg_image_gen, self.mix_res_module]

        total_iterations = self.iters[self.stage]
        total_iterations = self.iters_per_res\
            .get(self.resolution, {self.stage.name: total_iterations})\
            .get(self.stage.name, total_iterations)

        self.delta_progress = 1.0 / total_iterations
        self.tqdm_logger = tqdm.tqdm(total=total_iterations)

        if self.verbose:
            print("====================== Encoder =============================")
            print(self.enc)
            print("====================== Decoder ===========================")
            print(self.dec)

    def train(self):
        while not (
                self.resolution == self.max_image_res and self.stage == ProgGrowStageType.stab and self.progress >= 1.0):

            "========================================== updating params ==============================================="

            self.n_iter += 1
            self.tqdm_logger.update(1)
            self.progress = np.clip(self.progress + self.delta_progress, 0, 1.0)

            if self.stage == ProgGrowStageType.trns:
                progress_resolution = self.resolution * self.progress + (self.resolution / 2) * (
                        1 - self.progress)

                for model in self.updating_models_during_training:
                    if model is not None:
                        model.set_progress(self.progress)
            else:
                progress_resolution = self.resolution

            "=========================================== train step ==================================================="

            real_x = next(self.pg_image_gen).cuda()
            rec_x = self.dec(self.enc(real_x))
            loss_info = self.optimizer.compute_loss(real_x, rec_x, update_parameters=True)

            del real_x, rec_x

            "============================================== logging ==================================================="

            if self.tqdm_logger.n % self.log_iter == 0:
                self.logger.add_scalars('train/losses', loss_info, self.n_iter)
                self.logger.add_scalar('train/resolution', progress_resolution, self.n_iter)

            if self.tqdm_logger.n % self.image_sample_iter == 0:
                self._save_image_sample()

            "============================================ checkpoint =================================================="

            if self.tqdm_logger.n % self.val_iter == 0:
                val_loss = self._compute_val_loss()
                self.logger.add_scalar('val/total', val_loss, self.n_iter)

                # early stopping at the last stage
                if self.resolution == self.max_image_res and self.stage == ProgGrowStageType.stab:
                    if val_loss < self.best_val_loss - self.early_stopping_min_delta:
                        self.best_val_loss = val_loss
                        self.best_val_iter = self.n_iter
                        self._do_checkpoint()
                        self._save_ad_model()
                    else:
                        if (self.n_iter - self.best_val_iter) / self.val_iter > self.early_stopping_patience:
                            break

            "========================================== create new stage =============================================="

            if abs(self.progress - 1.0) < 1e-6 and \
                    not(self.resolution == self.max_image_res and self.stage == ProgGrowStageType.stab):
                    self._do_checkpoint()
                    self._save_ad_model()
                    self._create_new_stage()

        # if we did not check last iteration for performance, then check and save
        if self.tqdm_logger.n % self.val_iter != 0:
            val_loss = self._compute_val_loss()
            self.logger.add_scalar('val/total', val_loss, self.n_iter)
            if val_loss < self.best_val_loss:
                self.best_val_iter = self.n_iter
                self._do_checkpoint()
                self._save_ad_model()

    def get_state(self):
        return {
            'config': self.config,
            'stage': self.stage.name,
            'resolution': self.resolution,
            'progress': self.progress,
            'n_iter': self.n_iter,
            'best_val_loss': self.best_val_loss,
            'best_val_iter': self.best_val_iter,
            'display_x': self.display_x.data,
            'optimizer': self.optimizer.state_dict(),
            'dec': self.dec.state_dict(),
            'enc': self.enc.state_dict(),
            'mix_res_module': self.mix_res_module.state_dict(),
        }

    def load_state(self, state):
        self.stage = ProgGrowStageType[state['stage']]
        self.resolution = state['resolution']
        self.progress = state['progress']
        self.n_iter = state['n_iter']
        self.best_val_loss = state['best_val_loss']
        self.best_val_iter = state['best_val_iter']
        self.display_x.data = state['display_x']

        self._init_stage()

        self.dec.load_state_dict(state['dec'])
        self.enc.load_state_dict(state['enc'])

        self.mix_res_module.load_state_dict(state['mix_res_module'])

        self.tqdm_logger.update(int(len(self.tqdm_logger) * self.progress))

    def get_state_anomaly_detection_model(self):
        return {
            'niter': self.tqdm_logger.n,
            'config': self.config,
            'stage': self.stage.name,
            'resolution': self.resolution,
            'progress': self.progress,
            'enc': self.enc.state_dict(),
            'dec': self.dec.state_dict(),
        }

    @staticmethod
    def load_anomaly_detection_model(state_dict):
        config = state_dict['config']
        stage = ProgGrowStageType[state_dict['stage']]
        resolution = state_dict['resolution']
        progress = state_dict['progress']

        mtype = config['enc']['type']
        kwargs = config['enc']['kwargs']
        mode_3d =  config.get('mode_3d', False)

        enc_pg_networks = ENCODER_NETWORKS[NetworkType[mtype]](
            max_input_res=config['max_image_res'],
            output_res=config['latent_res'],
            input_dim=config['image_dim'],
            output_dim=config['latent_dim'],
            mode_3d=mode_3d,
            **kwargs
        )

        mtype = config['dec']['type']
        kwargs = config['dec']['kwargs']
        dec_pg_networks = DECODER_NETWORKS[NetworkType[mtype]](
            input_res=config['latent_res'],
            max_output_res=config['max_image_res'],
            input_dim=config['latent_dim'],
            output_dim=config['image_dim'],
            mode_3d=mode_3d,
            **kwargs
        )

        enc = enc_pg_networks.get_net(stage, resolution)
        dec = dec_pg_networks.get_net(stage, resolution)
        enc.load_state_dict(state_dict['enc'])
        dec.load_state_dict(state_dict['dec'])

        loss_type = config['image_rec_loss']['loss_type']
        loss_kwargs = config['image_rec_loss']['loss_kwargs']
        image_rec_loss = PG_RECONSTRUCTION_LOSSES[ReconstructionLossType[loss_type]](
            config['max_image_res'], mode_3d=mode_3d, **loss_kwargs)
        image_rec_loss.set_stage_resolution(stage, resolution)
        image_rec_loss.set_reduction('none')

        mix_res_module = MixResolution(stage, resolution, config['max_image_res'], mode_3d=mode_3d)

        for model in [enc, dec, image_rec_loss, mix_res_module]:
            model.set_progress(progress)

        n_iter = state_dict['niter']

        enc, dec, image_rec_loss = enc.cuda().eval(), dec.cuda().eval(), image_rec_loss.cuda().eval()
        return enc, dec, image_rec_loss, (stage, resolution, progress, n_iter, mix_res_module)

    @staticmethod
    def predict_anomaly_score(loaded_model, image, reduction):
        enc, dec, image_rec_loss, _ = loaded_model

        with torch.no_grad():
            rec_image = dec(enc(image)).detach()
            image_rec_loss.set_reduction(reduction)
            pred = image_rec_loss(image, rec_image)

        return pred

    def _compute_val_loss(self):
        self.enc.eval()
        self.dec.eval()
        torch.set_grad_enabled(False)

        sum_loss = 0
        count = 0

        for real_x in self.val_pg_image_gen:
            real_x = real_x.cuda()
            rec_x = self.dec(self.enc(real_x))

            loss_info = self.optimizer.compute_loss(real_x, rec_x, update_parameters=False)

            val_loss = sum(loss_info.values())

            batch_size = real_x.shape[0]
            sum_loss += val_loss * batch_size
            count += batch_size

        val_avg_loss = sum_loss / count

        self.enc.train()
        self.dec.train()
        torch.set_grad_enabled(True)

        return val_avg_loss

    def _do_checkpoint(self):
        torch.save(self.get_state(), os.path.join(self.checkpoint_root, 'checkpoint.tar'))

    def _save_ad_model(self):
        if self.stage == ProgGrowStageType.stab:
            ad_model_path = 'anomaly_detection_res_{}_iter_{}.tar'.format(self.resolution, self.n_iter)
            torch.save(self.get_state_anomaly_detection_model(), os.path.join(self.checkpoint_root, ad_model_path))

            torch.save(self.get_state_anomaly_detection_model(), os.path.join(self.checkpoint_root, 'anomaly_detection.tar'))

    def _save_image_sample(self):
        torch.set_grad_enabled(False)
        self.optimizer.eval()

        real_images = self.mix_res_module(self.display_x)
        images = torch.cat([
            real_images,
            self.dec(self.enc(real_images.cuda())).cpu().detach()
        ], 0)

        # if 3d mode, get only central slice
        if self.mode_3d:
            images = images[:, :, int(images.shape[2] // 2)]
        name = 'res_{}_stage_{}_niter{}.png'.format(self.resolution, self.stage.name,
                                                       self.tqdm_logger.n)

        self.logger.save_grid(images, grid_size=self.max_image_res * 4, name=name, nrow=min(4, len(self.display_x)))
        self.logger.save_grid(images, grid_size=self.max_image_res * 4, name='sample.png', nrow=min(4, len(self.display_x)))

        torch.set_grad_enabled(True)
        self.optimizer.train()

def main(config):
    trainer = DeepPerceptualAutoencoder(config)
    trainer.train()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('config', type=str, help='Path to config')
    args = parser.parse_args()

    with open(args.config, 'r') as stream:
        config = yaml.load(stream, Loader=yaml.FullLoader)

    main(config)


import requests
import argparse
import sys
from bulk_insert import BulkInserter

def main(args):
    args.n_gt = 1
    args.bulk_size = 5000
    if args.scenario == 'tiny':
        args.n_pt = 10
        args.n_iov = 10
    elif args.scenario == 'tiny-moderate':
        args.n_pt = 10
        args.n_iov = 200
    elif args.scenario == 'moderate':
        args.n_pt = 100
        args.n_iov = 200
    elif args.scenario == 'heavy-usage':
        args.n_pt = 100
        args.n_iov = 5000
    elif args.scenario == 'worst-case':
        args.n_pt = 200
        args.n_iov = 26000
    else:
        sys.exit(f"invalid scenario: '{args.scenario}'. chose from ['tiny', 'tiny-moderate', 'moderate', 'heavy-usage', 'worst-case']")
    bulk_inserter = BulkInserter(args)
    bulk_inserter.prepare()
    bulk_inserter.insert()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--hostname", type=str, default="test111.apps.usatlas.bnl.gov")
    parser.add_argument("--scenario", type=str, default="tiny")
    args = parser.parse_args()
    main(args)


import sys
import pygame
from bullet import Bullet
from alien import Alien
from time import sleep

def check_evens(ship, bullets, settings, screen):
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            sys.exit()
        elif event.type == pygame.KEYDOWN:
            check_keydown_event(event, ship, settings, screen, bullets)
        elif event.type == pygame.KEYUP:
            check_keyup_event(event, ship)

def check_keydown_event(event, ship, settings, screen, bullets):
    if event.key == pygame.K_RIGHT:
        ship.moving_right = True
    elif event.key == pygame.K_LEFT:
        ship.moving_left = True
    elif event.key == pygame.K_SPACE:
        fire_bullets(ship, bullets, settings, screen)
    elif event.key == pygame.K_q:
        sys.exit()

def fire_bullets(ship, bullets, settings, screen):
    if len(bullets) < settings.bullet_allowed:
        new_bullets = Bullet(settings, screen, ship)
        bullets.add(new_bullets)

def check_keyup_event(event, ship):
    if event.key == pygame.K_RIGHT:
        ship.moving_right = False
    elif event.key == pygame.K_LEFT:
        ship.moving_left = False

def update_screen(ship, bullets, aliens, settings, screen):
    screen.fill(settings.bg_color)
    for bullet in bullets:
        bullet.draw_bullet()

    ship.blitme()
    aliens.draw(screen)
    pygame.display.flip()

def update_bullets(bullets, aliens, ship, settings, screen):
    for bullet in bullets.copy():
        if bullet.rect.bottom <= 0:
            bullets.remove(bullet)
    check_bullet_aliens_collisions(aliens, bullets, ship, settings, screen)

def create_fleet(settings, screen, aliens, ship):
    """Создает флот пришельцев."""
    alien = Alien(settings, screen)
    number_aliens_x = get_number_aliens_x(settings, alien.rect.width)
    number_rows = get_number_rows(settings, alien.rect.height, ship.rect.height)
    for alien_row in range(number_rows):
        for alien_number in range(number_aliens_x):
            create_alien(settings, screen, aliens, alien_number, alien_row)

def get_number_aliens_x(settings, alien_width) -> int:
    """Вычисляет количество пришельцев в ряду."""
    avalible_space = settings.screen_width - alien_width * 2
    return int(avalible_space / (alien_width * 2))

def create_alien(settings, screen, aliens, alien_number, row_number):
    """Создает пришельца и размещает его в ряду."""
    new_alien = Alien(settings, screen)
    alien_width = new_alien.rect.width
    new_alien.x = alien_width + alien_width * 2 * alien_number
    new_alien.rect.x = new_alien.x
    new_alien.rect.y = new_alien.rect.height + new_alien.rect.height * 2 * row_number
    aliens.add(new_alien)

def get_number_rows(settings, alien_height, ship_height) -> int:
    """Определяет количество рядов, помещающихся на экране."""
    avalible_space_y = settings.screen_height - (alien_height * 3) - ship_height
    return int(avalible_space_y / (alien_height * 2))

def update_aliens(settings, stats, aliens, ship, bullets, screen):
    """
    Проверяет, достиг ли флот края экрана,
    после чего обновляет позиции всех пришельцев во флоте.
    """
    check_fleet_edges(settings, aliens)
    aliens.update()

    if pygame.sprite.spritecollideany(ship, aliens):
        ship_hit(stats, aliens, bullets, ship, settings, screen)

def change_fleet_directions(settings, aliens):
    """Опускает весь флот и меняет направление флота."""
    for alien in aliens.sprites():
        alien.rect.y += settings.aliens_fleet_drop_speed
    settings.fleet_direction *= -1

def check_fleet_edges(settings, aliens):
    """Реагирует на достижение пришельцем края экрана."""
    for alien in aliens.sprites():
        if alien.check_edges():
            change_fleet_directions(settings, aliens)
            break

def check_bullet_aliens_collisions(aliens, bullets, ship, settings, screen):
    """Проверка попаданий в пришельцев."""
    collisions = pygame.sprite.groupcollide(bullets, aliens, True, True)
    if len(aliens) == 0:
        bullets.empty()
        create_fleet(settings, screen, aliens, ship)

def ship_hit(stats, aliens, bullets, ship, settings, screen):
    """Обрабатывает столкновение корабля с пришельцем."""
    stats.ship_left -= 1

    aliens.empty()
    bullets.empty()

    create_fleet(settings, screen, aliens, ship)
    ship.center_ship()

    sleep(0.5)


from typing import List

"""
expression = "2*3-4*5
"""

class Solution:
    OP_SET = ("+", "-", "*")

    def diffWaysToCompute(self, expression: str) -> List[int]:
        if expression.isdigit():
            return [int(expression)]

        res = []
        for i, char in enumerate(expression):
            if char in self.OP_SET:
                left = self.diffWaysToCompute(expression[0:i])
                right = self.diffWaysToCompute(expression[i+1:])

                for x in left:
                    for y in right:
                        if char == "+":
                            res.append(x+y)
                        elif char == "-":
                            res.append(x-y)
                        else:
                            res.append(x*y)
        return res

    # def solve(self, expression: List, ans: List):
    #     print("exp", expression)
    #     if len(expression) == 1:
    #         ans.append(int(expression[0]))
    #     for i in range(len(expression)):
    #         if expression[i] in self.OP_SET:
    #             c = self.cal(expression[i-1], expression[i+1], expression[i])
    #             self.solve(expression[:i-1] + [str(c)] + expression[i+2:], ans)

if __name__ == "__main__":
    solution = Solution()
    print(solution.diffWaysToCompute("2*3-4*5"))
    print(solution.diffWaysToCompute("2-1-1"))

241. 为运算表达式设计优先级.py

241. 为运算表达式设计优先级.py


"""
Authors: Spotlight Pathology Ltd
License: Apache License V.2
"""

import torch
from train.segmentation.segmentation_model import Unet
from torch.optim import Adam, SGD, lr_scheduler
from warmup_scheduler import GradualWarmupScheduler

DEVICE = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

def setup_training(args, label_ignore):
    """
    Setup model, optimizers and losses for Unet multiclass.
    """
    # setup model
    # ---------------------------------------------------------------------
    model = Unet(
        args.backbone,
        num_classes=args.n_classes,
        in_channels=3,
        model_size=args.tile,
        freeze_encoder=(args.freeze_encoder == 'y')
    )

    # move model to CUDA before constructing optimizers
    # https://pytorch.org/docs/stable/optim.html
    model.to(DEVICE)

    # setup optimizer
    # ---------------------------------------------------------------------
    params = filter(lambda p: p.requires_grad, model.parameters())
    if args.optimizer == 'Adam':
        optimizer = Adam(params, lr=args.lr, weight_decay=args.weight_decay)
    elif args.optimizer == 'SGD':
        optimizer = SGD(
            params,
            lr=args.lr,
            momentum=0.9,
            weight_decay=args.weight_decay
        )
    else:
        raise ValueError('Parser arg optimizer only supports SGD, Adam.')

    # setup loss
    # ---------------------------------------------------------------------
    criterion = torch.nn.CrossEntropyLoss(
        weight=torch.Tensor(args.clw).to(DEVICE),
        ignore_index=label_ignore  # ignored/ does not contribute to gradient
    )

    # setup lr scheduler
    # ---------------------------------------------------------------------
    scheduler = lr_scheduler.ExponentialLR(optimizer, gamma=0.9)
    if args.warmup == 'y':
        scheduler = GradualWarmupScheduler(
            optimizer, multiplier=1, total_epoch=5, after_scheduler=scheduler
        )

    return model, optimizer, criterion, scheduler


import streamlit as st
import pandas as pd
import numpy as np
import plotly.express as px
import seaborn as sns
import json

def main():
    st.title("Análise da quantidade de produtos conforme suas caracterizações")

    analysis_options = ["Material", "Número de features", "Categoria"]
    st.sidebar.header("Visualizações")
    analysis = st.sidebar.selectbox("Escolha a característa para contagem:", analysis_options)

    st.header(f"Distribuição da categoria \"{analysis}\"")

    #load dataframe
    data_path = r"/data/products_clean.json"
    with open(data_path) as json_file:
        products = json.load(json_file)
        df = pd.json_normalize(products["products"], max_level=0)
    df["features"] = df["features"][df["features"].notnull()].apply(lambda x: len(x))

    if analysis == "Material":
        count = df["material"].value_counts()

    if analysis == "Número de features":
        count = df["features"].value_counts()

    if analysis == "Categoria":
        count = df["product_category"].value_counts()

    sorted_count = count.sort_values(ascending=False)
    results = sorted_count.iloc[:8].copy()
    try:
        results["Other"] = sorted_count.iloc[8:].sum()
    except KeyError:
        pass

    fig = px.pie(results, values=results.values, names=results.index)
    st.plotly_chart(fig)

if __name__ == "__main__":
    main()


"""Tests the controller and database collection objects methods
directly. These tests rely on the `./tests/Pd` directory, which has the model
outputs that temporary directories will be compared to.
"""
import numpy as np
from os import path
import pytest
from matdb.utility import reporoot, relpath
from matdb.atoms import AtomsList
import six

def _mimic_vasp(folder, xroot, prefix="W.1"):
    """Copies a `vasprun.xml` and `OUTCAR ` output files from the given folder into
    the execution directory to mimic what VASP would have done.

    Args:
        folder (str): path to the folder where the model files are stored.
        xroot (str): path to the root folder where the config steps are stored.
    """
    from matdb.utility import chdir
    from glob import glob
    from os import path
    from matdb.utility import symlink

    files = ["vasprun.xml", "OUTCAR"]

    with chdir(folder):
        for vaspfile in files:
            pattern = vaspfile + "__*"
            for dft in glob(pattern):
                name, config = dft.split("__")
                xpath = path.join(xroot, path.join(*config.split("_")), prefix)
                #We want to make some testing assertions to ensure that the
                #stubs ran correctly.
                #Bypass checking the DynMatrix subfolder
                if "DynMatrix" in xpath:
                    continue
                assert path.isfile(path.join(xpath, "CONTCAR"))
                assert path.isfile(path.join(xpath, ".matdb.module"))
                target = path.join(xpath, name)
                symlink(target, path.join(folder, dft))

@pytest.fixture()
def Pd(tmpdir):
    from matdb.utility import relpath, copyonce
    from matdb.database import Controller
    from os import mkdir, symlink, remove, path

    target = relpath("./tests/Pd/matdb.yml")
    dbdir = str(tmpdir.join("pd_db"))
    mkdir(dbdir)
    copyonce(target, path.join(dbdir, "matdb.yml"))
    target = path.join(dbdir,"matdb")

    #We need to copy the POSCAR over from the testing directory to the temporary
    #one.
    from shutil import copy
    POSCAR = relpath("./tests/Pd/POSCAR")
    mkdir(path.join(dbdir,"seed"))
    copy(POSCAR, path.join(dbdir,"seed","Pd"))
    # the source file `matdb.yml` linked to might be gone, that left `matdb.yml` not an valid "file"
    # we need to get rid if it anyway
    try:
        remove("matdb.yml")
    except:
       pass
    symlink("{}.yml".format(target),"matdb.yml")

    result = Controller("matdb", dbdir)
    remove("matdb.yml")
    result = Controller(target, dbdir)
    return result

@pytest.fixture()
def Pd_copy(tmpdir):
    from matdb.utility import relpath, copyonce
    from matdb.database import Controller
    from os import path, remove, mkdir

    target = relpath("./tests/Pd/matdb_copy.yml")
    dbdir = str(tmpdir.join("pd_db_copy"))
    mkdir(dbdir)
    copyonce(target, path.join(dbdir, "matdb.yml"))
    target = path.join(dbdir,"matdb")

    from shutil import copy
    POSCAR = relpath("./tests/Pd/POSCAR")

    if path.isfile("matdb_copy.yml"):
        remove("matdb_copy.yml")

    result = Controller(target, dbdir)

    mkdir(path.join(dbdir,"seed"))
    copy(POSCAR, path.join(dbdir, "seed", "Pd"))
    result = Controller(target, dbdir)
    return result

@pytest.fixture()
def Pd_split(tmpdir):
    from matdb.utility import relpath, copyonce
    from matdb.database import Controller
    from os import mkdir, path

    target = relpath("./tests/Pd/matdb_split.yml")
    dbdir = str(tmpdir.join("pd_db_splits"))
    mkdir(dbdir)
    copyonce(target, path.join(dbdir, "matdb.yml"))
    target = path.join(dbdir,"matdb")

    from shutil import copy
    POSCAR = relpath("./tests/Pd/POSCAR")
    mkdir(path.join(dbdir,"seed"))
    copy(POSCAR, path.join(dbdir,"seed","Pd-1"))
    copy(POSCAR, path.join(dbdir,"seed","Pd-2"))
    copy(POSCAR, path.join(dbdir,"seed","Pd-3"))
    copy(POSCAR, path.join(dbdir,"seed","Pd-4"))
    copy(POSCAR, path.join(dbdir,"seed","Pd-5"))

    result = Controller(target, dbdir)
    return result

def test_Pd_setup(Pd, Pd_copy):
    """Makes sure the initial folders were setup according to the spec.
    """
    #raise Exception("RAWR")
    Pd.setup()
    modelroot = path.join(Pd.root, "Manual","phonon.manual","Pd")
    #import pdb; pdb.set_trace()
    assert Pd["Manual/phonon/Pd/"].root == modelroot

    #The matdb.yml file specifies the following database:
    dbs = ["Manual/phonon.manual/Pd/"]
    #Each one should have a folder for: ["hessian", "modulations"]
    #On the first go, the modulations folder will be empty because the DFT
    #calculations haven't been performed yet. However, hessian should have DFT
    #folders ready to go.
    folders = {
        "__files__": ["compute.pkl","jobfile.sh"],
        "S1.1": {
            "__files__": ["INCAR", "POSCAR", "POTCAR", "PRECALC", "KPOINTS"]
        }
    }

    from matdb.utility import compare_tree
    for db in dbs:
        dbfolder = path.join(Pd.root, db)
        compare_tree(dbfolder, folders)

    #import pdb; pdb.set_trace()
    #Now we will test some of the border cases of the database __init__ method
    Pd_copy.setup()

    db = "Manual/phonon.manual/Pd"
    dbfolder = path.join(Pd_copy.root, db)
    compare_tree(dbfolder, folders)

def test_steps(Pd):
    """Tests compilation of all steps in the database.
    """
    assert Pd.steps() == ['manual/phonon']
    Pd.setup()
    steps = sorted(['manual/phonon/Pd'])
    assert Pd.steps() == steps

    seqs = sorted(['Pd'])
    assert Pd.sequences() == seqs

def test_find(Pd):
    """Tests the find function and the __getitem__ method with pattern matching.
    """
    Pd.setup()
    steps = Pd.find("manual/phonon")
    model = ['phonon']
    assert model == [s.parent.name for s in steps]
    model = [path.join(Pd.root,'Manual/phonon.manual')]
    assert sorted(model) == sorted([s.root for s in steps])

    steps = Pd.find("*/phonon")
    model = ['phonon']
    assert model == [s.parent.name for s in steps]
    model = [path.join(Pd.root,'Manual/phonon.manual')]
    assert model == [s.root for s in steps]

    steps = Pd.find("manual/phonon/Pd")
    model = ['manual']
    assert model == [s.parent.name for s in steps]
    model = [path.join(Pd.root,'Manual/phonon.manual/Pd')]
    assert model == [s.root for s in steps]

    steps = Pd.find("phonon")
    model = ['phonon']
    assert model == [s.name for s in steps]
    model = [Pd.root]
    assert model == [s.root for s in steps]

    steps = Pd.find('*')
    model = ['phonon']
    assert model == [s.name for s in steps]
    model = [Pd.root]
    assert model == [s.root for s in steps]

    steps = Pd.find("manual/phonon/Pd/S1.1")
    model = [('phonon','manual')]
    assert model == [(s.parent.parent.name,s.name) for s in steps]
    model = [path.join(Pd.root,'Manual/phonon.manual/Pd')]
    assert model == [s.root for s in steps]

    # test uuid finding.
    assert all([Pd.find(s.uuid)==s for s in steps])

    # test the __getitem__ method
    model = 'phonon'
    modelroot = path.join(Pd.root,'Manual/phonon.manual')
    group = Pd["manual/phonon"]
    assert group.parent.name == model
    assert group.root == modelroot

    model = 'manual'
    modelroot = path.join(Pd.root,'Manual/phonon.manual/Pd')
    group = Pd["manual/phonon/Pd"]
    assert group.parent.name == model
    assert group.root == modelroot

    group = Pd["manual/phonon/Pd/S1.1"]
    assert group.parent.name == model
    assert group.root == modelroot

    group = Pd["enumeration/phonon"]
    assert group == None

def test_execute(Pd, capsys):
    """Tests the execute and extract methods
    """
    from os import path
    from matdb.utility import relpath, chdir
    from matdb.msg import verbosity

    verbosity = 2

    Pd.status()
    output = capsys.readouterr()
    status = "ready to execute 0/0; finished executing 0/0;"
    assert status in output.out

    # test to ensure execute prints error message if ran before setup
    Pd.execute(env_vars={"SLURM_ARRAY_TASK_ID":"1"})
    output = capsys.readouterr()
    status = "Group phonon.manual is not ready to execute yet, or is already executing. Done."
    assert status in output.out

    # Run setup and status to make sure the staus output is correct
    Pd.setup()
    Pd.status()
    output = capsys.readouterr()
    status = "ready to execute 1/1; finished executing 0/1;"
    assert status in output.out

    # Execute the jobfile and test an incomplete OUTCAR to check the status
    Pd.execute(env_vars={"SLURM_ARRAY_TASK_ID":"1"})
    folder = path.join(reporoot, "tests", "data", "Pd", "manual_recover")
    _mimic_vasp(folder, Pd.root,"S1.1")

    Pd.status(True)
    busy_status = "Pd./Manual/phonon.manual/Pd/S1.1"
    output = capsys.readouterr()
    assert busy_status in output.out

    #now use a complete OUTCAR and test the status again
    folder = path.join(reporoot, "tests", "data", "Pd", "manual")
    _mimic_vasp(folder,Pd.root,"S1.1")

    Pd.status()
    output = capsys.readouterr()
    status = "ready to execute 1/1; finished executing 1/1;"
    assert status in output.out

    # Run exctract and test to see if the status is correct
    Pd.extract()
    Pd.status()
    output = capsys.readouterr()
    status = "ready to execute 1/1; finished executing 1/1;"
    assert status in output.out

    # Run extract again to make sure the atoms.h5 files are no rewritten
    Pd.extract()

def test_recovery(Pd):
    """Tests the rerun on unfinshed jobs
    """
    from os import path
    from matdb.utility import symlink, chdir
    from glob import glob

    Pd.setup()
    Pd.execute(env_vars={"SLURM_ARRAY_TASK_ID":"1"})

    files = ["vasprun.xml", "OUTCAR"]
    folder = path.join(reporoot, "tests", "data", "Pd", "manual_recover")
    with chdir(folder):
        for vaspfile in files:
            pattern = vaspfile + "__*"
            for dft in glob(pattern):
                name, config = dft.split("__")
                xpath = path.join(Pd.root, path.join(*config.split("_")), "S1.1")
                target = path.join(xpath, name)
                symlink(target, path.join(folder, dft))

    Pd.extract()

    Pd.recover(True)
    assert path.isfile(path.join(Pd.root,"Manual","phonon.manual","Pd","recovery.sh"))
    assert path.isfile(path.join(Pd.root,"Manual","phonon.manual","Pd","failures"))

    folder = path.join(reporoot, "tests", "data", "Pd", "manual")
    _mimic_vasp(folder,Pd.root,"S1.1")
    Pd.recover(True)
    assert not path.isfile(path.join(Pd.root,"Manual","phonon.manual","Pd","recovery.sh"))
    assert not path.isfile(path.join(Pd.root,"Manual","phonon.manual","Pd","failures"))

def test_hash(Pd):
    """Tests the hash_dbs and verify_hash methods
    """
    from os import path, chdir
    Pd.setup()
    Pd.execute(env_vars={"SLURM_ARRAY_TASK_ID":"1"})
    folder = path.join(reporoot, "tests", "data", "Pd", "manual")
    _mimic_vasp(folder,Pd.root,"S1.1")
    db_hash = Pd.hash_dbs()
    assert Pd.verify_hash(db_hash)

def test_finalize(Pd):
    """ Test the finalize function in the controller module
    """
    from os import path
    Pd.setup()
    Pd.execute(env_vars={"SLURM_ARRAY_TASK_ID":"1"})
    folder = path.join(reporoot,"tests","data","Pd","manual")
    _mimic_vasp(folder,Pd.root,"S1.1")

    Pd.extract()
    Pd.split()
    Pd.finalize()

    from matdb.io import load_dict_from_h5
    from matdb import __version__
    import h5py

    str_ver = []
    for item in __version__:
        str_ver.append(str(item))
    mdb_ver = ".".join(str_ver)
    target = path.join(Pd.root, "final_{}.h5".format(mdb_ver))
    with h5py.File(target, "r") as hf:
        loaded_final = load_dict_from_h5(hf)
    assert path.isfile(target)

def test_split(Pd_split):
    """ Test the split function in the controller object
    """
    from os import path
    Pd_split.setup()
    Pd_split.execute(env_vars={"SLURM_ARRAY_TASK_ID":"1"})
    folder = path.join(reporoot,"tests","data","Pd","manual_split")
    _mimic_vasp(folder,Pd_split.root,"S1.1")

    Pd_split.extract()
    Pd_split.split()

    for dbname, db in Pd_split.collections.items():
        for s, p in db.splits.items():
            tfile = path.join(db.train_file(s).format(s))
            hfile = path.join(db.holdout_file(s).format(s))
            sfile = path.join(db.super_file(s).format(s))

            tal = AtomsList(tfile)
            hal = AtomsList(hfile)
            sal = AtomsList(sfile)

            assert len(tal) == int(np.ceil(5*p))
            assert len(hal) == int(np.ceil((5-len(tal))*p))
            assert len(sal) == 5-len(tal)-len(hal)

def test_Pd_hessian(Pd):
    """Tests the `niterations` functionality and some of the standard
    methods of the class on simple Pd.

    """
    from os import remove
    Pd.setup()

    #Test the status, we should have some folder ready to execute.
    Pd.status()
    Pd.status(busy=True)

    #Test execution command; this uses stubs for all of the commands that would
    #be executed.
    Pd.execute(env_vars={"SLURM_ARRAY_TASK_ID": "1"})
    folder = path.join(reporoot, "tests", "data", "Pd", "recover")
    _mimic_vasp(folder, Pd.root)

    #Now that we have vasp files, we can queue recovery. The recover
    #`vasprun.xml` files that we linked to are not complete for all the
    #structures (on purpose).
    Pd.recover()
    recoveries = ["Manual/phonon/Pd"]

    for rkey in recoveries:
        assert path.isfile(path.join(Pd[rkey].root, "recovery.sh"))

    Pd.execute(env_vars={"SLURM_ARRAY_TASK_ID": "1"}, recovery=True)
    folder = path.join(reporoot, "tests", "data", "Pd", "manual")
    _mimic_vasp(folder,Pd.root,"S1.1")
    remove(path.join(Pd[rkey].root, "recovery.sh"))

    #Now that we have recovered vasp files, we can queue a *second*
    #recovery. All but one of the `vasprun.xml` files that we linked to are
    #complete for all the structures. This test that we can recover and execute
    #multiple times in order.
    Pd.recover()
    okay = ["Manual/phonon/Pd"]
    for rkey in okay:
        assert not path.isfile(path.join(Pd[rkey].root, "recovery.sh"))


import karelia
import time

def on_kill():
    """Define actions to be performed when the bot receives a !kill command"""
    print("Got killed :-(")

# Initialise a bot with nick (or [nick1, nick2]) and room the bot will connect to
bot = karelia.bot("ExampleBot", "test")

# Define response messages to the botrulez.
bot.stock_responses['short_help'] = "Short help message"
bot.stock_responses['long_help'] = ["Long help message can be sent over multiple messages","This message was generated by Example Bot"]

# Assign on_kill function to bot's on_kill.
bot.on_kill = on_kill

# Connect to the room
bot.connect()

while True:
    try:
        while True:
            message = bot.parse()
            print(message.type)
            if message.type == "send-event":
                print(f"{message.data.sender.name}: {message.data.content}")
                if message.data.content.split()[0] == "!changenick":
                    bot.change_nick(message.data.content.split()[1][1:])

    except SystemExit:
        # Catches !kill commands to the bot and ctrl+c/equivalent
        on_kill()
        sys.exit(0)
    except:
        # Catches any other exception
        bot.log()
        bot.disconnect()
    finally:
        time.sleep(1)
        bot.connect()


import os
import sys
import pandas as pd
import matplotlib.pyplot as plt

# creaa un piccolo dataframe con i dati con una descrizione e pochi dati
df = pd.DataFrame({'a': [1, 2, 3, 4, 5, 6, 7, 8, 9],
                     'b': [1, 2, 3, 4, 5, 6, 7, 8, 9],
                        'c': [1, 2, 3, 4, 5, 6, 7, 8, 9],

                        'd': [1, 2, 3, 4, 5, 6, 7, 8, 9],
                        'e': [1, 2, 3, 4, 5, 6, 7, 8, 9],
                        'f': [1, 2, 3, 4, 5, 6, 7, 8, 9],

                        'g': [1, 2, 3, 4, 5, 6, 7, 8, 9]})

# crea un grafico con i dati del dataframe
df.plot()
plt.show()

df = pd.read_csv('Analisi_trade.csv')

# estraggo il secondo campo di simbolo
df['Scadenza'] = df['Simbolo'].str.split(' ').str[1]

# faccio un ripilogo dei dati per scadenza  e con solo il realizzato totale e lo memorizzo in un df
df2 = df.groupby(['Scadenza'])['Realizzato Totale'].sum().to_frame()

df2.plot()
plt.show()


import numpy as np
import sawyer_control.envs.sawyer_reaching as sawyer_reaching

from multiworld.core.multitask_env import MultitaskEnv
from multiworld.core.serializable import Serializable
from gym.spaces import Dict

class SawyerReachXYZEnv(sawyer_reaching.SawyerReachXYZEnv, MultitaskEnv):
    def __init__(self,
                 **kwargs
                 ):
        Serializable.quick_init(self, locals())
        sawyer_reaching.SawyerReachXYZEnv.__init__(self, **kwargs)
        self.observation_space = Dict([
            ('observation', self.observation_space),
            ('desired_goal', self.goal_space),
            ('achieved_goal', self.goal_space),
            ('state_observation', self.observation_space),
            ('state_desired_goal', self.goal_space),
            ('state_achieved_goal', self.goal_space),
        ])

    def step(self, action):
        self._act(action)
        observation = self._get_obs()
        reward = MultitaskEnv.compute_reward(self, action, observation)
        info = self._get_info()
        done = False
        return observation, reward, done, info

    def compute_rewards(self, actions, obs):
        achieved_goals = obs['state_achieved_goal']
        desired_goals = obs['state_desired_goal']
        hand_pos = achieved_goals
        goals = desired_goals
        distances = np.linalg.norm(hand_pos - goals, axis=1)
        if self.reward_type == 'hand_distance':
            r = -distances
        elif self.reward_type == 'hand_success':
            r = -(distances > self.indicator_threshold).astype(float)
        else:
            raise NotImplementedError("Invalid/no reward type.")
        return r

    def _get_obs(self):
        ee_pos = self._get_endeffector_pose()
        state_obs = super()._get_obs()
        return dict(
            observation=state_obs,
            desired_goal=self._state_goal,
            achieved_goal=ee_pos,

            state_observation=state_obs,
            state_desired_goal=self._state_goal,
            state_achieved_goal=ee_pos,
        )

    def reset(self):
        if self.action_mode == "position":
            self._position_act(self.reset_pos - self._get_endeffector_pose(), in_reset=True)
        else:
            self._reset_robot()
        goal = self.sample_goal()
        self._state_goal = goal['state_desired_goal']
        return self._get_obs()

    """
    Multitask functions
    """

    def get_goal(self):
        return {
            'desired_goal': self._state_goal,
            'state_desired_goal': self._state_goal,
        }

    def sample_goal(self):
        return MultitaskEnv.sample_goal(self)

    def sample_goals(self, batch_size):
        goals = super().sample_goals(batch_size)
        return {
            'desired_goal': goals,
            'state_desired_goal': goals,
        }

    def set_to_goal(self, goal):
        goal = goal['state_desired_goal']
        super().set_to_goal(goal)

if __name__=="__main__":
    env = SawyerReachXYZEnv()
    env.reset()


#-*- coding:utf-8 -*-
'''
DOMAIN ADAPTATION OF LINEAR CLASSIFIERS (aka DALC)
See: http://arxiv.org/abs/1506.04573

Kernel class and KernelClassifier class

@author: Pascal Germain -- http://researchers.lille.inria.fr/pgermain/
'''
import numpy as np
from scipy.spatial.distance import cdist
from scipy.sparse import issparse

# kernel functions
def linear_kernel(point_1, point_2):
    return np.dot(point_1, point_2)

def linear_matrix(X1, X2):
    return np.dot(X1, X2.T)

def rbf_kernel(point_1, point_2, gamma):
    diff = point_1 - point_2
    return np.exp(-gamma * np.dot(diff, diff))

def rbf_matrix(X1, X2, gamma):
    return np.exp(-gamma * cdist(X1, X2, 'sqeuclidean') )

def precomputed_matrix(X1, X2):
    return X2.T

class Kernel:
    """
    Wrapper around a kernel function.
    """
    def __init__(self, kernel_str='linear', kernel_func=None, matrix_func=None, **kernel_args):
        """Create a kernel
        kernel_str: kernel name ('linear', 'rbf', 'precomputed', 'custom')
        kernel_func: if kernel_str='custom', function to compute the kernel between two example vectors
        matrix_func: if kernel_str='custom', function to compute the kernel between two example matrices
        kernel_args: extra kernel arguments (e.g., gamma for the 'rbf' kernel)"""
        self.name = kernel_str
        self.args = kernel_args

        if self.name == 'custom':
            self.kernel_func = kernel_func
            self.matrix_func = matrix_func

        elif self.name == 'rbf':
            self.args.setdefault('gamma', 1.0)
            self.kernel_func = rbf_kernel
            self.matrix_func = rbf_matrix

        elif self.name == 'linear':
            self.kernel_func = linear_kernel
            self.matrix_func = linear_matrix

        elif self.name == 'precomputed':
            self.kernel_func = None
            self.matrix_func = precomputed_matrix

        else:
            raise Exception("Unknown kernel.")

    def create_matrix(self, X1, X2=None):
        """Compute the kernel matrix between two example matrices (X1 and X2)
        If X2 is not specified, then X2=X1"""
        if self.matrix_func is not None:
            if X2 is None: X2 = X1
            kernel_matrix = self.matrix_func(X1, X2, **self.args)

        elif self.kernel_func is not None:
            if X2 is not None:
                dim1 = np.size(X1,0)
                dim2 = np.size(X2,0)
                kernel_matrix = np.zeros( (dim1,dim2) )

                for i in range(dim1):
                    for j in range(dim2):
                        kernel_matrix[i,j] = self.kernel_func(X1[i], X2[j], **self.args)
            else:
                dim1 = np.size(X1,0)
                kernel_matrix = np.zeros( (dim1,dim1) )

                for i in range(dim1):
                    for j in range(i+1):
                        value = self.kernel_func(X1[i], X1[j], **self.args)
                        kernel_matrix[i,j] = value
                        kernel_matrix[j,i] = value

        else:
            raise Exception('kernel function unspecified.')

        if issparse(kernel_matrix):
            return kernel_matrix.toarray()
        else:
            return kernel_matrix

class KernelClassifier:
    """
    A linear classifier in the kernel space.
    """
    def __init__(self, kernel, X, alpha_vector=None):
        """Create a kernel classifier.
        kernel: the kernel object
        X: training examples matrix
        alpha_vector: weight vector (optional) """
        self.kernel = kernel
        self.X1 = X
        self.X1_shape = np.shape(X) if X is not None else (0,0)

        if alpha_vector is None:
            self.alpha_vector = np.zeros(self.X1_shape[0])
        else:
            self.alpha_vector = alpha_vector

    def create_matrix(self, X2):
        """Create a kernel matrix between the training matrix and X2"""
        return self.kernel.create_matrix(self.X1, X2)

    def create_self_matrix(self):
        """Create a kernel matrix between the training matrix and itself"""
        return self.kernel.create_matrix(self.X1)

    def predict(self, X=None, kernel_matrix=None):
        """Classifies examples (i.e., return a prediction vector)"""
        if kernel_matrix is None:
            if X is not None:
                kernel_matrix = self.create_matrix(X)
            else:
                raise Exception('Nothing to classify!')

        return np.dot(kernel_matrix.T, self.alpha_vector)

    def calc_risk(self, Y, X=None, kernel_matrix=None, predictions=None):
        """Compute the classification risk on a datatset."""
        if predictions is None:
            predictions = self.predict(X, kernel_matrix)
        return np.mean( predictions * Y <= 0.0 )

    def write_to_file(self, filename):
        """Write the weight vector into a file."""
        self.alpha_vector.tofile(filename, '\n')


import statistics

with open("in.txt") as f:
	crabs = [int(x) for x in f.readline().rstrip().split(',')]

med = int(statistics.median(crabs))
avg = int(statistics.mean(crabs))

print("part1: fuel spent: {}".format(sum([abs(crabs[i]-med) for i in range(len(crabs))])))

# the sum of 1+2+3+4+..+n is n(n+1)/2
tot = 0
for i in range(len(crabs)):
	tot += abs(crabs[i]-avg)*(abs(crabs[i]-avg)+1)/2

print("part2: fuel spent: {}".format(int(tot)))


import json
import requests
import time

from headers_generation import get_headers

def get_html(url: str) -> str:
    rq = requests.get(url, headers=get_headers())
    print('Getting HTML-code from ', url)
    return rq.text

def is_empty(html: str, vacancy_list_attribute: str) -> bool:
    data_dict = json.loads(html)

    if not data_dict.get(vacancy_list_attribute):
        return True

    return False

def get_offers_links(html, vacancy_list_attribute: str, url_attribute: str):
    links = []

    data_dict = json.loads(html)
    vacancy = data_dict[vacancy_list_attribute]

    for item in vacancy:
        links.append(item[url_attribute])
    return links

class HabrCrawler:
    def __init__(self, query: str = '', areas: list = None, per_page: int = 100, specializations: list = None,
                 time_wait: float = 0.5):
        self.query = query
        self.per_page = per_page
        self.areas = areas
        self.specializations = specializations
        self.root_url = 'https://career.habr.com/api/frontend/vacancies?'
        self.time_wait = time_wait

    def get_vacancies_urls(self, limit: int = None):
        all_links = []
        page = 0

        vacancy_root_url = 'https://career.habr.com'

        while True:
            url = f"{self.root_url}page={page}&per_page={self.per_page}"
            if self.areas:
                for area in self.areas:
                    url += f"&area={area}"
            if self.specializations:
                for specialization in self.specializations:
                    pass
            time.sleep(self.time_wait)
            html = get_html(url)
            if is_empty(html, vacancy_list_attribute='list'):
                break

            all_links += [f"{vacancy_root_url}{x}" for x in get_offers_links(html,
                                                                             vacancy_list_attribute='list',
                                                                             url_attribute='href')]
            page += 1

            if len(all_links) > limit:
                all_links = all_links[:limit]
                break

        return list(set(all_links))

class HHCrawler:
    def __init__(self, query: str = '', areas: list = None, per_page: int = 100, specializations: list = None,
                 time_wait: float = 0.0):
        self.query = query

        if areas is None:
            self.areas = [113]

        self.per_page = per_page

        if specializations is None:
            self.specializations = [160,
                                    10,
                                    12,
                                    150,
                                    25,
                                    165,
                                    34,
                                    36,
                                    73,
                                    155,
                                    96,
                                    164,
                                    104,
                                    157,
                                    107,
                                    112,
                                    113,
                                    148,
                                    114,
                                    116,
                                    121,
                                    124,
                                    125,
                                    126]

            self.root_url = 'https://api.hh.ru/vacancies?'
            self.time_wait = time_wait

    def get_vacancies_urls(self, limit_per_specialization: int = None):
        all_links = []

        for specialization in self.specializations:
            page = 0

            links_to_add = []

            while True:
                url = f"{self.root_url}text={self.query}&page={page}&per_page={self.per_page}" \
                      f"&professional_role={specialization}"
                for area in self.areas:
                    url += f"&area={area}"
                time.sleep(self.time_wait)
                html = get_html(url)
                if is_empty(html, vacancy_list_attribute='items'):
                    break

                links_to_add += get_offers_links(html, vacancy_list_attribute='items', url_attribute='url')
                page += 1

                if len(links_to_add) > limit_per_specialization:
                    links_to_add = links_to_add[:limit_per_specialization]
                    break

            all_links.extend(links_to_add)

        return list(set(all_links))