import os, json
from Task4 import capitalize_string
from Task5 import News, Tweet, Ads

DEFAULT_FILENAME = 'Task8_default.json'

class FeedJSON:
    def __init__(self, file_path):
        if not file_path:
            self.__create_default_file()
            self.file_path = DEFAULT_FILENAME
        else:
            self.file_path = file_path
        self.fail_flag = False
        self.read_file(self.file_path)
        if not self.fail_flag:
            os.remove(self.file_path)

    def __create_default_file(self):
        with open(DEFAULT_FILENAME, 'w') as default_file:
            content = {
                "feeds": [
                    {
                        "id": 1,
                        "type": "N",
                        "text": "This IS the eXAMple of task 8! news text",
                        "city": "Prague"
                    },
                    {
                        "id": 2,
                        "type": "A",
                        "text": "This IS the eXAMple of task 8! aDS text",
                        "city": "Brno",
                        "expired": "2024-01-01"
                    },
                    {
                        "id": 3,
                        "type": "T",
                        "text": "This IS the eXAMple of task 8! twEET text",
                        "author": "Unknown"
                    }
                ]
            }
            json_data = json.dumps(content)
            default_file.write(json_data)

    def read_file(self, filepath):
        with open(filepath, 'r') as file:
            feeds = json.load(file)
            for feed in feeds['feeds']:
                if feed['type'] == "N":
                    text = capitalize_string(feed['text'])
                    city = feed['city']
                    News(text, city)
                elif feed['type'] == "A":
                    text = capitalize_string(feed['text'])
                    city = feed['city']
                    expired = feed['expired']
                    Ads(text, city, expired)
                elif feed['type'] == "T":
                    text = capitalize_string(feed['text'])
                    author = feed['author']
                    Tweet(text, author)
                else:
                    self.fail_flag = True
                    print("Unsupported feed content: ", feed)


import pandas as pd
import requests
from bs4 import BeautifulSoup as bs
from bs4.element import NavigableString
import re

class crawler:
    '''
    Module for crawling meta data in URLs
    '''
    def __init__(self, headers, album_url, artist_url, music_url, outpath):
        self.headers = headers
        self.album_url = album_url
        self.artist_url = artist_url
        self.music_url = music_url
        self.outpath = outpath

############################################################################################################################
### album_url 파싱
############################################################################################################################

    def album_meta(self, album_id, idx, img_size=282):
        '''
        <params>
        album_url : str, album url for html parsing
        idx : int, album index => album cover image fileName
            ex) if idx = 12,
                - album idx = 12
                - album cover image fileName = 12.jpg
        img_size : int, image size to extract album cover image size

        <result>
        return df_result, artist_ids, song_ids

        df_result : dict, album meta data dictionary
        artist_ids : list, artist's id in Melon, multi layers
        song_ids : list, song's id in Melon, multi layers
        '''

        # 결과 저장 딕셔너리 생성
        result = {
            'album_id': album_id,
            'album_title': None,
            'artist_id': None,
            'album_artist': None,
            'album_cover': None,
            'album_cover_url': None,
            'album_date': None,
            'album_genre': None,
            'album_url': self.album_url,
        }

        ## 앨범 내 아티스트id와 음원id를 저장할 리스트 생성
        ## 앨범 참여 아티스트가 다수, 한 앨범에 여러 곡 있을 수 있기 때문
        artist_ids = []
        song_ids = []

        # album_url 파싱
        res_album = requests.get(self.album_url.format(album_id), headers=self.headers)
        source_album = res_album.text
        # html5lib을 이용할 경우에만 album_url 제대로 파싱 가능
        # lxml, html.parser 모두 안됨... 아마 웹브라우저와 같은 방식의 페이지 파싱 방식이 요구되는 듯
        soup_album = bs(source_album, 'html5lib')

        ## album meta data 추출 부분
        ## 1. 앨범명 추출
        album_title = soup_album.find('div', class_='song_name').text.replace('\n', '').replace('\t', '').replace('앨범명', '')
        result['album_title'] = album_title

        ## 2. 앨범 커버 추출 (default size = 282 x 282)
        album_cover = soup_album.find('a', id='d_album_org')
        img_url = album_cover.find('img')['src'].replace('/282/', f'/{img_size}/')
        ## 이미지 저장은 DB 구축된 후에
        # req.urlretrieve(img_url, self.outpath + f'{idx}.jpg')
        result['album_cover'] = idx
        result['album_cover_url'] = img_url

        ## 앨범 메타 데이터
        meta_album = soup_album.find('dl', class_='list')

        dt = meta_album.findAll('dt')
        dd = meta_album.findAll('dd')

        for i in range(len(dt)):
            ## 3. 앨범 발매일 추출
            if dt[i].text == '발매일':
                result['album_date'] = dd[i].text
            ## 4. 앨범 장르 추출 -> 여러 개일 경우 만들기
            if dt[i].text == '장르':
                result['album_genre'] = dd[i].text

        ## artist id 및 name 추출 부분
        div_artist = soup_album.find('div', class_='artist')
        artists = div_artist.findAll('a', class_='artist_name')
        artist_name = []

        ## artists가 존재하는 경우
        if artists:

            for a in artists:
                ## artist id 추출
                artist_href = a['href']
                artist_id = re.findall('\d+', artist_href)[0]
                artist_ids.append(artist_id)

                ## album artist 이름 추출
                name = a['title']
                artist_name.append(name)

        ## artists가 존재하지 않는 경우(ex. Various Artists)
        else:
            artist_name.append(div_artist.text.strip())

        result['artist_id'] = ', '.join(map(str, artist_ids))
        result['album_artist'] = ', '.join(map(str, artist_name))

        ## song id 추출 부분
        song_list = soup_album.find('div', class_='service_list_song d_song_list')
        tbody = song_list.find('tbody')
        tr = tbody.findAll('tr')

        ## music_url에 필요한 song id 추출
        for t in tr:
            song_href = t.find('a', class_='btn button_icons type03 song_info')

            if song_href:
                song_id = re.findall('\d+', song_href['href'])[0]
                #             print(song_id)
                song_ids.append(song_id)

        # 결과 저장 데이터프레임 생성
        df_result = pd.DataFrame([result])

        return df_result, artist_ids, song_ids

############################################################################################################################
### artist_url 파싱
############################################################################################################################

    def artist_meta(self, artist_ids, album_id, img_size=416):
        '''
        <params>
        artist_ids : list, artist's id for artist_url parsing
        album_id : int, album's id for distinct
        img_size : int, image size to extract artist cover image size

        <result>
        return df_result

        result : DataFrame, artist meta data DataFrame
        '''

        # 결과 저장 데이터프레임 생성
        df_result = pd.DataFrame()

        for idx in artist_ids:
            url = self.artist_url.format(idx)
            # 결과 저장 딕셔너리 생성
            result = {
                'artist_id': idx,
                'artist_name': None,
                'real_name': None,
                'type': None,
                'gender': None,
                'nation': None,
                'artist_url': url,
                'sim_artist': None,
                'agency_artist': None,
                'relate_artist': None,
                'affected_artist': None,
                'effected_artist': None,
                'artist_cover_url' : None,
            }

            # artist_url 파싱
            res_artist = requests.get(url, headers=self.headers)
            source_artist = res_artist.text
            soup_artist = bs(source_artist, 'html5lib')

            wrap_dtl_artist = soup_artist.find('div', class_='wrap_dtl_atist')
            ## 1. 아티스트명 추출
            artist_name = wrap_dtl_artist.find('p', class_='title_atist')  # .text.replace('아티스트명', '')
            remove_child_tags = [bs_object for bs_object in artist_name if isinstance(bs_object, NavigableString)]
            result['artist_name'] = remove_child_tags[0]

            ## 2. 실명이 따로 존재하는 경우
            real = wrap_dtl_artist.find('span', class_='realname')
            if real:
                real_name = re.sub('[()]', '', real.text.strip())
                result['real_name'] = real_name

            ## 3. 아티스트 이미지 url 추출
            artist_img = wrap_dtl_artist.find('span', class_='thumb')
            img_url = artist_img.find('img')['src'].replace('resize/416', f'resize/{img_size}')

            if 'default/noArtist' not in img_url:
                result['artist_cover_url'] = img_url

            ## 아티스트 성별, 유형 추출
            meta_artist = soup_artist.find('div', class_='section_atistinfo03')

            dt = meta_artist.findAll('dt')
            dd = meta_artist.findAll('dd')

            for i in range(len(dt)):
                ## 아티스트 유형, 성별 추출
                if dt[i].text == '유형':
                    type_gender = dd[i].text.replace('\n', '').replace('\t', '').split('|')

                    if len(type_gender) > 1:
                        ## 4. 유형
                        result['type'] = type_gender[0]
                        ## 5. 성별
                        result['gender'] = type_gender[1]
                    else:
                        ## 유형, 성별 중 1개만 나와있을 경우 유형/성별 파악
                        if type_gender[0] in ['솔로', '그룹']:
                            result['type'] = type_gender[0]
                        elif type_gender[0] in ['남성', '여성', '혼성']:
                            result['gender'] = type_gender[0]

            ######################################
            ## 우선 보류, 테이블 복잡도 등 원인
            ## 아티스트 그룹 확인
            #         group_check = meta_artist.find('div', class_='wrap_gmem')

            #         if group_check:
            #             print('그룹임')
            #         else:
            #             print('솔로임ㅋ')
            ######################################

            ## 6. 아티스트 국적 추출
            meta_artist = soup_artist.find('div', class_='section_atistinfo04')

            dt = meta_artist.findAll('dt')
            dd = meta_artist.findAll('dd')

            for i in range(len(dt)):
                ## 아티스트 국적있으면 추출
                if dt[i].text == '국적':
                    result['nation'] = dd[i].text

            ## 7. 연관 아티스트 추출 (5가지 수집)
            ## 유사 / 같은 소속사 / 관련 / 영향을 받은 / 영향을 준
            meta_rltn = soup_artist.find('div', class_='section_atistinfo06 d_artist_list')
            if meta_rltn:
                h4 = meta_rltn.findAll('h4', class_='title line arr')
                wrap_list = meta_rltn.findAll('div', class_='wrap_list')

                for i in range(len(h4)):
                    ## 유사 아티스트탭이 있을 경우에 해당 목록 전부 수집
                    if h4[i].text == '유사 아티스트':
                        sim_artists = []
                        thumb = wrap_list[i].findAll('a', class_='thumb')

                        for a in thumb:
                            sim_id = re.findall('\d+', a['href'])[0]
                            sim_artists.append(sim_id)

                        result['sim_artist'] = ', '.join(map(str, sim_artists))

                    ## 같은 소속사 아티스트탭이 있을 경우에 해당 목록 전부 수집
                    if h4[i].text == '같은 소속사 아티스트':
                        agency_artist = []
                        thumb = wrap_list[i].findAll('a', class_='thumb')

                        for a in thumb:
                            agency_id = re.findall('\d+', a['href'])[0]
                            agency_artist.append(agency_id)

                        result['agency_artist'] = ', '.join(map(str, agency_artist))

                    ## 관련 아티스트탭이 있을 경우에 해당 목록 전부 수집
                    if h4[i].text == '관련 아티스트':
                        relate_artist = []
                        thumb = wrap_list[i].findAll('a', class_='thumb')

                        for a in thumb:
                            relate_id = re.findall('\d+', a['href'])[0]
                            relate_artist.append(relate_id)

                        result['relate_artist'] = ', '.join(map(str, relate_artist))

                    ## 영향을 받은 아티스트탭이 있을 경우에 해당 목록 전부 수집
                    if h4[i].text == '영향을 받은 아티스트':
                        affected_artist = []
                        thumb = wrap_list[i].findAll('a', class_='thumb')

                        for a in thumb:
                            affected_id = re.findall('\d+', a['href'])[0]
                            affected_artist.append(affected_id)

                        result['affected_artist'] = ', '.join(map(str, affected_artist))

                    ## 영향을 준 아티스트탭이 있을 경우에 해당 목록 전부 수집
                    if h4[i].text == '영향을 준 아티스트':
                        effected_artist = []
                        thumb = wrap_list[i].findAll('a', class_='thumb')

                        for a in thumb:
                            effected_id = re.findall('\d+', a['href'])[0]
                            effected_artist.append(effected_id)

                        result['effected_artist'] = ', '.join(map(str, effected_artist))

            df = pd.DataFrame([result])
            df_result = pd.concat([df_result, df])

        return df_result

############################################################################################################################
### music_url 파싱
############################################################################################################################

    def song_meta(self, song_ids, album_id):
        '''
        <params>
        song_ids : list, song's id for artist_url parsing
        album_id : int, album's id for distinct

        <result>
        return df_result

        df_result : DataFrame, song meta data DataFrame
        '''

        # 결과 저장 데이터프레임 생성
        df_result = pd.DataFrame()

        for idx in song_ids:
            url = self.music_url.format(idx)
            # 결과 저장 딕셔너리 생성
            # check_adult는 0: 일반, 1: 19금곡
            result = {
                'album_id': album_id,
                'song_id': idx,
                'song_title': None,
                'artist_id': None,
                'artist_name': None,
                'genre': None,
                'lyrics': None,
                'composer_id': None,
                'composer': None,
                'lyricist_id': None,
                'lyricist': None,
                'arranger_id': None,
                'arranger': None,
                'check_adult': None,
                'song_url': url,
            }

            res_song = requests.get(url, headers=self.headers)
            source_song = res_song.text
            soup_song = bs(source_song, 'lxml')

            ## 1. 음원명 추출
            song_title = soup_song.find('div', class_='song_name').text.replace('\n', '').replace('\t', '').replace('곡명', '').strip()
            # 19금 음원의 경우 '19금'과 개행이 또 안에 있어서 제거
            result['song_title'] = song_title.replace('19금', '').strip()

            ## 2. 19세 음원 확인값
            if '19금' in song_title:
                result['check_adult'] = 1
            else:
                result['check_adult'] = 0

            ## 3. 아티스트 ID 추출
            div_artist = soup_song.find('div', class_='artist')
            song_artists = div_artist.findAll('a', class_='artist_name')
            artist_ids = []
            artist_names = []

            for artist in song_artists:
                artist_href = artist['href']
                artist_id = re.findall('\d+', artist_href)[0]
                artist_ids.append(artist_id)

                artist_name = artist['title']
                artist_names.append(artist_name)

            result['artist_id'] = ', '.join(map(str, artist_ids))
            result['artist_name'] = ', '.join(map(str, artist_names))

            ## 4. 음원 장르 추출
            meta_song = soup_song.find('div', class_='meta')

            dt = meta_song.findAll('dt')
            dd = meta_song.findAll('dd')

            for i in range(len(dt)):
                ## 음원 장르 추출 -> 여러 개일 경우 만들기
                if dt[i].text == '장르':
                    result['genre'] = dd[i].text

            ## 5. 음원 가사 추출
            lyrics = soup_song.find('div', class_='lyric')
            ## 가사 개행이 제대로 안돼서 야매로 개행 만들기
            lyrics = str(lyrics).replace('
',
                                         '').replace('
', '').replace('
', '\n').strip()
            result['lyrics'] = lyrics

            ## 6. 음원 작사/작곡 추출
            prdcr = soup_song.find('div', class_='section_prdcr')
            ## prdcr 유무 판단 (없는 경우 존재하기 때문)
            if prdcr:
                a = prdcr.findAll('a', class_='artist_name')
                span = prdcr.findAll('span', class_='type')

                lyricist = []
                composer = []
                arranger = []

                lyricist_id = []
                composer_id = []
                arranger_id = []

                for i in range(len(a)):
                    ## 작사가, 작곡가, 편곡가 추출
                    if span[i].text == '작사':
                        lyricist.append(a[i].text)
                        href = a[i]['href']
                        idx = re.findall('\d+', href)[0]
                        lyricist_id.append(idx)

                    if span[i].text == '작곡':
                        composer.append(a[i].text)
                        href = a[i]['href']
                        idx = re.findall('\d+', href)[0]
                        composer_id.append(idx)

                    if span[i].text == '편곡':
                        arranger.append(a[i].text)
                        href = a[i]['href']
                        idx = re.findall('\d+', href)[0]
                        arranger_id.append(idx)

                result['lyricist'] = ', '.join(map(str, lyricist))
                result['composer'] = ', '.join(map(str, composer))
                result['arranger'] = ', '.join(map(str, arranger))

                result['lyricist_id'] = ', '.join(map(str, lyricist_id))
                result['composer_id'] = ', '.join(map(str, composer_id))
                result['arranger_id'] = ', '.join(map(str, arranger_id))

            df = pd.DataFrame([result])
            df_result = pd.concat([df_result, df])

        return df_result

############################################################################################################################


import random
import matplotlib.pyplot as plt
from tqdm import tqdm

# Parâmetros do problema

GENES = 36

# Parâmetros do AG

MAX_GERACOES = 0
TAM_POPULACAO = 0
PROB_CRUZAMENTO = 0.0
PROB_MUTACAO = 0.0

# Gera um indivíduo aleatório
def geraIndividuo ():
    genes = []

    for i in range(GENES):
        genes.append(random.randint(0,1) )

    return { 'genes': genes, 'fitness':  0}

# Gera um vetor de indivíduos aleatórios
def geraPopulacao (tam_populacao):
    pop = []
    for i in range(tam_populacao):
        pop.insert(i, geraIndividuo())

    return pop

# Calcula a fitness de todos os indivíduos de uma população
def avaliacao (pop):
    for i in range(len(pop)):

        pop[i]['fitness'] = (9
            + pop[i]['genes'][1]
            *pop[i]['genes'][4]  - pop[i]['genes'][22]*pop[i]['genes'][13] + pop[i]['genes'][23]*pop[i]['genes'][3]  - pop[i]['genes'][20]*pop[i]['genes'][9]
            + pop[i]['genes'][35]*pop[i]['genes'][14] - pop[i]['genes'][10]*pop[i]['genes'][25] + pop[i]['genes'][15]*pop[i]['genes'][16] + pop[i]['genes'][2]*pop[i]['genes'][32]
            + pop[i]['genes'][27]*pop[i]['genes'][18] + pop[i]['genes'][11]*pop[i]['genes'][33] - pop[i]['genes'][30]*pop[i]['genes'][31] - pop[i]['genes'][21]*pop[i]['genes'][24]
            + pop[i]['genes'][34]*pop[i]['genes'][26] - pop[i]['genes'][28]*pop[i]['genes'][6]  + pop[i]['genes'][7]*pop[i]['genes'][12]  - pop[i]['genes'][5]*pop[i]['genes'][8]
            + pop[i]['genes'][17]*pop[i]['genes'][19] - pop[i]['genes'][0]*pop[i]['genes'][29]  + pop[i]['genes'][22]*pop[i]['genes'][3]  + pop[i]['genes'][20]*pop[i]['genes'][14]
            + pop[i]['genes'][25]*pop[i]['genes'][15] + pop[i]['genes'][30]*pop[i]['genes'][11] + pop[i]['genes'][24]*pop[i]['genes'][18] + pop[i]['genes'][6]*pop[i]['genes'][7]
            + pop[i]['genes'][8]*pop[i]['genes'][17]  + pop[i]['genes'][0]*pop[i]['genes'][32])

# Faz o sorteio de um indice
def giraRoleta(soma, roleta):
    numAleatorio = random.randrange(0, soma)
    for i in range(len(roleta)):
        if(i == 0):
            if(numAleatorio < roleta[i]):
                idxSorteado = i
        else:
            if(numAleatorio >= roleta[i-1] and numAleatorio < roleta[i]):
                idxSorteado = i
    return idxSorteado

# Seleciona os indíviduos com o método de roleta
def selecaoRoleta (pop):
    pares = []
    roleta = []
    soma = 0

    for i in range(len(pop)):
        if i == 0:
            roleta.insert(i, pop[i]['fitness'])
            soma = pop[i]['fitness']
        else:
            soma = pop[i]['fitness'] + soma
            roleta.insert(i, soma)

    for i in range(len(pop)):

        idx_ind1 = giraRoleta(soma, roleta)
        idx_ind2 = giraRoleta(soma, roleta)
        if(idx_ind2 == idx_ind1):
            idx_ind2 = giraRoleta(soma, roleta)
            if(idx_ind2 == idx_ind1):
                idx_ind2 = idx_ind1 + 1

        pares.append({'ind1': pop[idx_ind1], 'ind2': pop[idx_ind2]})

    return pares

# Seleciona o indivíduo mais forte entre 2 indivíduos aleatórios
def duelo (pop):
    ind1 = random.randrange(0, len(pop))
    ind2 = random.randrange(0, len(pop))

    if ind1 == ind2:
        ind2 = random.randrange(0, len(pop))

    if pop[ind1]['fitness'] > pop[ind2]['fitness']:
        indVencedor = ind1
    else:
        indVencedor = ind2

    return indVencedor

# Seleciona os indivíduos pelo método de Torneio
def selecaoTorneio(pop):
    pares = []
    for i in range(len(pop)):
        idx_ind1 = duelo(pop)
        idx_ind2 = duelo(pop)
        if idx_ind2 == idx_ind1:
            idx_ind2 = duelo(pop)
            if idx_ind2 == idx_ind1:
                idx_ind2 = idx_ind1 + 1

        pares.append({'ind1': pop[idx_ind1], 'ind2': pop[idx_ind2]})

    return pares

# Cruzamento entre 2 indivíduos utilizando 1 ponto de corte
def cruzamentoUmponto(ind1, ind2):
    genes_filho = []

    pc = random.randint(0, GENES-1)

    for i in range(GENES):
        if i <= pc:
            genes_filho.append( ind1['genes'][i])
        else:
            genes_filho.append( ind2['genes'][i])

    filho = {
        'genes': genes_filho,
        'fitness': 0
    }
    return filho

# Cruzamento entre 2 indivíduos utilizando 2 pontos de corte
def cruzamentoDoispontos(ind1, ind2):
    genes_filho = []

    pc1 = random.randint(0, GENES-1)
    pc2 = random.randint(pc1, GENES-1)

    for i in range(GENES):
        if i <= pc1:
            genes_filho.append(ind1['genes'][i])
        elif i > pc1 and i <= pc2:
            genes_filho.append(ind2['genes'][i])
        else:
            genes_filho.append(ind1['genes'][i])

    filho = {
        'genes': genes_filho,
        'fitness': 0
    }
    return filho

# Realiza mutacao por escolha aleatória de um bit
def mutacao(ind):
    geneAleatorio = random.randint(0, GENES-1)
    indMutado = {
        'genes': ind['genes'].copy(),
        'fitness': 0
    }
    if indMutado['genes'][geneAleatorio] == 0:
        indMutado['genes'][geneAleatorio] = 1
    else:
        indMutado['genes'][geneAleatorio] = 0

    return indMutado

 # Aplica os operadores genéticos de cruzamento e mutação utilizando o vetor de pares.

def operadoresGeneticos(pares, opCruzamento):
    pop_nova = []

    for par in pares:
        ind1 = par['ind1']
        ind2 = par['ind2']

        probCruzamento = random.random()
        probMutacao = random.random()

        if probCruzamento <= PROB_CRUZAMENTO:
            if opCruzamento == 1:
                filho = cruzamentoUmponto(ind1, ind2)
            else:
                filho = cruzamentoDoispontos(ind1, ind2)
        elif probCruzamento < 0.5:
            filho = ind1
        else:
            filho = ind2

        if probMutacao <= PROB_MUTACAO:
            filho = mutacao(filho)

        pop_nova.append(filho)

    return pop_nova

# Aplica o elitismo de um elemento.

def elitismo(pop_antiga, pop_nova):
    melhorElemento = 0
    for i in range(len(pop_antiga)):
        if pop_antiga[i]['fitness'] > pop_antiga[melhorElemento]['fitness']:
            melhorElemento = i

    pop_nova[0] = pop_antiga[melhorElemento]

# Calcula a media de Fitness de uma populacao
def fitnessMedio(pop):
    soma = 0
    total = len(pop)
    for ind in pop:
        soma = ind['fitness'] + soma

    media = soma / total

    return media

# Seleciona o melhor elemento da populacao
def melhorElemento(pop):
    melhorElemento = 0
    for i in range(len(pop)):
        if pop[i]['fitness'] > pop[melhorElemento]['fitness']:
            melhorElemento = i

    return pop[melhorElemento]['fitness']

# Seleciona o pior elemento da população
def piorElemento(pop):
    piorElemento = 0
    for i in range(len(pop)):
        if pop[i]['fitness'] < pop[piorElemento]['fitness']:
            piorElemento = i

    return pop[piorElemento]['fitness']

# LOOP principal do AG

def main():

    print ('MENU DE CONFIGURAÇÃO DO AG \n')
    print('-----------------------------------------------------------------------------------------------')
    print ('Defina a quantidade de indivíduos da população: ')
    TAM_POPULACAO = int(input('Resposta: '))
    print('-----------------------------------------------------------------------------------------------')
    print ('Escolha o tipo de seleção que deseja utilizar: (1) Roleta (2) Torneio')
    Op_Selecao = int(input('Resposta: '))
    print('-----------------------------------------------------------------------------------------------')
    print ('Escolha o tipo de cruzamento que deseja utilizar: (1) 1 ponto de corte (2) 2 pontos de corte')
    Op_Cruzamento = (input('Resposta: '))
    print('-----------------------------------------------------------------------------------------------')
    print ('Defina probabilidade de mutação que deseja utilizar: (de 1 a 100)')
    PROB_MUTACAO = float(input('Resposta: '))
    print('-----------------------------------------------------------------------------------------------')
    print ('Defina a probabilidade de cruzamento: (de 1 a 100)')
    PROB_CRUZAMENTO = float(input('Resposta: '))
    print('-----------------------------------------------------------------------------------------------')
    print ('Deseja utilizar o elitismo no algoritmo: (1) Sim (2) Não')
    Op_Elitismo = int(input('Resposta: '))
    print('-----------------------------------------------------------------------------------------------')
    print ('Defina a quantidade de gerações: ')
    MAX_GERACOES = int(input('Resposta: '))
    print('\n')

    PROB_MUTACAO = PROB_MUTACAO / 100
    PROB_CRUZAMENTO = PROB_CRUZAMENTO / 100

    Populacao = geraPopulacao(TAM_POPULACAO)
    avaliacao(Populacao)
    geracao = 0
    xMedia = []
    yMedia = []
    xMelhorInd = []
    yMelhorInd = []
    xPiorInd = []
    yPiorInd = []
    xSolucaoOtima = []
    ySolucaoOtima = []
    with tqdm(total=MAX_GERACOES) as geracoes:
        for geracao in range(MAX_GERACOES):

            geracoes.update(1)

            # print("Geração: ", geracao)

            # print("Melhor elemento >>>", melhorElemento(Populacao))
            # print("Media da populacao >>>", fitnessMedio(Populacao))

            # print("\n---------------------------------------------------------------")

            if Op_Selecao == 1:
                pares = selecaoRoleta(Populacao)
            else:
                pares = selecaoTorneio(Populacao)

            popFilha = operadoresGeneticos(pares, Op_Cruzamento)
            avaliacao(popFilha)

            if Op_Elitismo == 1:
                elitismo(Populacao, popFilha)
                Populacao = popFilha
            else:
                Populacao = popFilha

            avaliacao(Populacao)

            x = geracao
            y = float(fitnessMedio(Populacao))
            xMInd = geracao
            yMInd = float(melhorElemento(Populacao))
            xPInd = geracao
            yPInd = float(piorElemento(Populacao))
            xSolucaoOtima.append(geracao)
            ySolucaoOtima.append(27)

            xMedia.append(x)
            yMedia.append(y)
            xMelhorInd.append(xMInd)
            yMelhorInd.append(yMInd)
            xPiorInd.append(xPInd)
            yPiorInd.append(yPInd)

            geracao = geracao + 1

    plt.figure(figsize=(5, 2.7), layout='constrained')
    plt.plot(xPiorInd, yPiorInd, label='Pior indivíduo da população')
    plt.plot(xMedia, yMedia, label='Media das populações')
    plt.plot(xMelhorInd, yMelhorInd, label='Melhor indivíduo da população')
    plt.plot(xSolucaoOtima, ySolucaoOtima, label='Solução ótima conhecida')
    plt.xlabel('Gerações')
    plt.ylabel('Fitness')
    plt.title("Evolução do fitness")
    plt.legend()
    plt.show()

main()


import opentracing

from .tracer import TornadoTracer
from . import httpclient

def tracer_config(__init__, app, args, kwargs):
    """
    Wraps the Tornado web application initialization so that the
    TornadoTracer instance is created around an OpenTracing-compatible tracer.
    """
    __init__(*args, **kwargs)

    tracer = app.settings.get('opentracing_tracer')

    if 'opentracing_trace_all' in app.settings:
        tracer._trace_all = app.settings['opentracing_trace_all']

    if 'opentracing_trace_client' in app.settings:
        tracer._trace_client = app.settings['opentracing_trace_client']

    if 'opentracing_start_span_cb' in app.settings:
        tracer._start_span_cb = app.settings['opentracing_start_span_cb']

    httpclient._set_tracing_enabled(tracer._trace_client)
    if tracer._trace_client:
        httpclient._set_tracing_info(tracer._tracer, tracer._start_span_cb)


class Solution:
    def findRestaurant(self, list1: List[str], list2: List[str]) -> List[str]:

        sum_of_index = 0
        min_sum = 2000
        restaurant = []

        for item in range(len(list1)):
            if list1[item] in list2:
                sum_of_index = item + list2.index(list1[item])

                if sum_of_index <= min_sum and list1[item] not in restaurant:
                    restaurant.append(list1[item])
                    min_sum = min(sum_of_index,min_sum)

        return restaurant

string/Minimum Index Sum of Two Lists.py


from django.shortcuts import redirect, render, redirect, get_object_or_404
from django.template import RequestContext
from django.http import HttpResponseRedirect
from django.contrib import messages
from django.contrib.auth.forms import AuthenticationForm
from django.contrib.auth import authenticate, login
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.db import connection
import cx_Oracle
# Create your views here.

@login_required
def home(request):
    data = {
        'ordenes':listado_ordenes()
    }
    return render(request, './home_cocina.html',data)

@login_required
def receta(request):
    data = {
        'recetas':listado_ingredienter()
    }

    return render(request, './registro_receta.html', data)

def receta_crear(request):

    data = {
        'carta' :listado_cartas()
    }

    if request.method == 'POST':

        nome = request.POST.get('nom')
        desce = request.POST.get('desc')
        porcione = request.POST.get('porcion')
        cartae = request.POST.get('carta')

        salida = agregar_receta( nome, desce, porcione,cartae)

        if salida == 1:
            data['mensaje'] = 'Agregado correctamente'
        else:
            data['mensaje'] = 'No se ha agregado'

    return render(request, './crear_receta.html' , data)

#Listado reservas cursor
def listado_cartas():
    django_cursor = connection.cursor()
    #Cursor que llama
    cursor = django_cursor.connection.cursor()
    #Cursor que recibe
    out_cur = django_cursor.connection.cursor()

    cursor.callproc("SP_LISTAR_CARTA", [out_cur])

    lista= []
    for fila in out_cur:
        lista.append(fila)
    return lista

def agregar_receta( nom, desc, porcion, carta):
    django_cursor = connection.cursor()
    #Cursor que llama
    cursor = django_cursor.connection.cursor()
    salida = cursor.var(cx_Oracle.NUMBER)
    cursor.callproc('SP_AGREGAR_RECETA_2',[nom, desc , porcion, carta, salida] )
    return salida.getvalue()

def listar_recetas():
    django_cursor = connection.cursor()
    cursor = django_cursor.connection.cursor()
    out_cur = django_cursor.connection.cursor()

    cursor.callproc("SP_LISTAR_CARTA_DETALLE", [out_cur])

    lista = []
    for fila in out_cur:
        lista.append(fila)
    return lista

def receta_listado(request):
    data = {

        'receta': listar_recetas()
    }

    return render(request,'./listar_recetas.html',data)

# LISTAR LAS RECETAS PARA MODFICAR
def modificar_receta(request):

    data = {
        'categorias' :listar_categoria()
    }

    return render(request, './lista_recetamod.html' , data)

def ingresar_modificar_receta(ID_RECETA, NOM_RECETA, DESC_RECETA, PORCION, CARTA_ID_CARTA):
    django_cursor = connection.cursor()
    cursor = django_cursor.connection.cursor()
    salida = cursor.var(cx_Oracle.NUMBER)
    cursor.callproc('SP_MODIFICAR_RECETAR',[ID_RECETA, NOM_RECETA, DESC_RECETA, PORCION, CARTA_ID_CARTA, salida])
    return salida.getvalue()

def proveedor(request):
    data = {
        'proveedor':listar_proveedor()
    }

    return render(request, './listar_proveedores.html', data)

def suministro(request):
    data = {
        'suministro':listar_suministro()
    }
    return render(request, './listar_suministros.html', data)

def eliminar_receta( id ):
    django_cursor = connection.cursor()
    #Cursor que llama
    cursor = django_cursor.connection.cursor()
    salida = cursor.var(cx_Oracle.NUMBER)
    cursor.callproc('SP_ELIMINAR_RECETAR',[id , salida ])
    return salida.getvalue()

def receta_eliminar(request, id):

    data = {
        'categorias' :listar_categoria()

    }

    salida = eliminar_reserva(id)

    if salida == 1:
        data['mensaje'] = 'Borrado correctamente'
        data['reservas'] = listadoReserva()
    else:
        data['mensaje'] = 'No se pudo borrar'
        data['reservas'] = listadoReserva()

    return render(request, './lista_recetamod.html', data)

def buscar_receta(id):
    django_cursor = connection.cursor()

    #Cursor que llama
    cursor = django_cursor.connection.cursor()
    #Cursor que recibe
    out_cur = django_cursor.connection.cursor()

    #Llamada al cursor
    cursor.callproc("SP_BUSCAR_RECETAR", [out_cur,id])

    #llenamos la lista
    lista= []
    for fila in out_cur:
        lista.append(fila)
    return lista

def buscar_receta_lista(request, id):

    listadoReserva = buscar_receta(id)

    data = {
    'categorias' : listar_categoria

    }

    return render(request, './lista_recetamod.html', data)

def modificar_orden_rellenar(request):
    data= {

    'categorias' :listar_categoria()
    }

    if request.method == 'POST':
        ID_ORDEN = request.POST.get('id_orden')
        FECHA = request.POST.get('fecha')
        HORA = request.POST.get('hora')
        MESA_ID_MESA = request.POST.get('mesa_id_mesa')
        salida = ingresar_ingrediente(ID_ORDEN, FECHA, HORA, MESA_ID_MESA)
        if salida == 1:
            data['mensaje'] = 'Modificado correctamente'
        else:
            data['mensaje'] = 'no se ha Modificado'
    return render(request, './modificar_ordenes.html', data)

#LISTAR ORDENES
def listado_ordenes():
    django_cursor = connection.cursor()

    #Cursor que llama
    cursor = django_cursor.connection.cursor()
    #Cursor que recibe
    out_cur = django_cursor.connection.cursor()

    #Llamada al cursor
    cursor.callproc("SP_LISTAR_ORDENES", [out_cur])

    #llenamos la lista
    lista= []
    for fila in out_cur:
        lista.append(fila)
    return lista

def listar_recetas():
    django_cursor = connection.cursor()

    #Cursor que llama
    cursor = django_cursor.connection.cursor()
    #Cursor que recibe
    out_cur = django_cursor.connection.cursor()

    #Llamada al cursor
    cursor.callproc("SP_LISTAR_RECETA", [out_cur])

    #llenamos la lista
    lista= []
    for fila in out_cur:
        lista.append(fila)
    return lista

def orden_listado(request):
    data = {
        'ordenes':listado_ordenes()
    }
    return render(request, './ordenes_listado.html', data)

def orden_detalle(request, id):

    data = {
        'ordenes' : detalles_ordenes(id)
    }

    return render(request, './detalle_orden.html', data)

def detalles_ordenes(id):
    django_cursor = connection.cursor()
    #Cursor que llama
    cursor = django_cursor.connection.cursor()
    #Cursor que recibe
    out_cur = django_cursor.connection.cursor()

    #Llamada al cursor
    cursor.callproc("SP_LISTAR_DETALLE_ORDEN_2", [out_cur, id])

    #llenamos la lista
    lista= []
    for fila in out_cur:
        lista.append(fila)
    return lista

def orden_actualizar(request, id, sec, sec2):

    salida = actualizar_ordenes(id, sec, sec2)

    if salida == 1:
        data = {
        'ordenes':listado_ordenes()
        }
        return render(request, './ordenes_listado.html', data)
    else:
        data = {
        'ordenes' : detalles_ordenes(id)
         }
        return render(request, './detalle_orden.html', data)

def actualizar_ordenes(id,sec, sec2):
    django_cursor = connection.cursor()
    #Cursor que llama
    cursor = django_cursor.connection.cursor()
    salida = cursor.var(cx_Oracle.NUMBER)
    cursor.callproc('SP_MODIFICAR_ORDENES',[id, sec , sec2 ,salida ])
    return salida.getvalue()


import numpy as np

def feedfoward(weights,topology,inputData,d_length):

    I,Y = init_variables(topology,len(inputData),d_length)
    #newData = preparateInputData(inputData,length)
    newData = inputData
    #print("input: " + str(newData))
    for ctr in range( len(topology) + 1 ):
        if ctr == 0:
            I[ctr] = np.dot(weights[ctr] , newData.transpose())
            Y[ctr] = np.concatenate( (-1*np.ones((1,1)), np.tanh(I[ctr])) )
        elif ctr == len(topology):
            I[ctr] = np.dot(weights[ctr],Y[ctr-1])
            Y[ctr] = np.tanh(I[ctr])
        else:
            I[ctr] = np.dot(weights[ctr],Y[ctr-1])
            Y[ctr] = np.concatenate( (-1*np.ones((1,1)), np.tanh(I[ctr])) )
    #print("saida: " + str(Y[len(topology)]))
    return realizeProbability(Y[len(topology)])
'''
def preparateInputData(inputData, length):
        #Normalização dos dados e guardar os maiores e menores valores por coluna
        for column in range(len(inputData)):
            inputData[column] = ( inputData[column] - (-length) ) / ( 2*length) -0.5
        #Concatenar uma matriz coluna igual a -1
        aux_matrix = -1 * np.ones( (1,1) )

        new_inputData = np.concatenate((aux_matrix,inputData[np.newaxis]),axis=1)

        return new_inputData
'''

def init_variables(topology,x_train_length,d_train_length):
    I = []
    Y = []
    accumulator = []
    gradient = []

    # Inicializar as entradas e saídas de cada camada e o gradiente ( que possui as mesmas dimensões da entrada da camada)
    for index in range(len(topology) +1):
        if index == len(topology):
            Iaux = np.zeros((d_train_length,1))
            Yaux = np.zeros((d_train_length,1))
        else:
            Iaux = np.zeros((topology[index],1))
            Yaux = np.zeros((topology[index]+1,1))

        I.append(Iaux)
        Y.append(Yaux)

    return I,Y

def realizeProbability(vector):
    result = []
    for index in range(len(vector)):
        if vector[index] == max(vector):
            result.append(1)
        else:
            result.append(-1)

    return result

def relu(matrix):
    for row in range(len(matrix)):
        for col in range(len(matrix[0])):
            if matrix[row][col] < 0:
                matrix[row][col] = 0
    return matrix


import importlib
import inspect
import os
import subprocess
import sys

import structlog

from azimuth.config import AzimuthValidationError, CustomObject

log = structlog.get_logger(__file__)

def install(remote: str):
    """
    Install a Python package.

    Args:
        remote: Can be a path to a folder with a setup.py
                        or a package name in pip
                        or a **public** git repository remote.

    Raises:
        FileNotFoundError if we can't find the remote locally or on Pypi.
    """
    if os.path.exists(remote):
        if os.path.isfile(os.path.join(remote, "setup.py")):
            # Install locally
            log.info(subprocess.check_call(["pip", "install", remote]))
        elif os.path.isfile(os.path.join(remote, "requirements.txt")):
            # Install deps locally
            log.info(
                subprocess.check_call(
                    [
                        "pip",
                        "install",
                        "-t",
                        remote,
                        "-r",
                        os.path.join(remote, "requirements.txt"),
                    ]
                )
            )
        # In any case, add the repo the system path.
        sys.path.insert(0, remote)
    else:
        # We assume that this is a package
        try:
            subprocess.check_call([sys.executable, "-m", "pip", "install", remote])
        except subprocess.CalledProcessError:
            # Was probably a misspecified folder
            raise AzimuthValidationError(f"Can't find remote {repr(remote)} locally or on Pypi.")
    importlib.invalidate_caches()

def load_custom_object(
    value: CustomObject, reject=None, allow_install_package=True, force_kwargs=False, **kwargs
):
    """
    Load an object dynamically and recursively.

    Args:
        value: Custom objects to instantiate.
        reject: List of `kwargs` to remove from the func call.
        allow_install_package: If True, will install missing packages.
        force_kwargs: If True will send all supplied kwargs to the class.
                        This can break the supplied class
                        if they don't accept extra parameters in **kwargs.

        kwargs: Args to add to the function.

    Returns:
        Object
    """
    if value.remote and allow_install_package:
        install(value.remote)
    return load_class(kwargs, reject, value, force_kwargs=force_kwargs)

def get_class_init_varnames(cls):
    # We check if func.__init__ is overloaded, otherwise func.__init__ has no attribute __code__.
    return cls.__init__.__code__.co_varnames if cls.__init__ != object.__init__ else []

def load_args(v, kwargs):
    if isinstance(v, CustomObject):
        return load_custom_object(v, **kwargs)
    if isinstance(v, list):
        return [load_args(vi, kwargs) for vi in v]
    else:
        return v

def load_class(kwargs, reject, value, force_kwargs=False):
    """
    Will recursively load an object.

    Args:
        kwargs: additional kwargs for the function.
        reject: list of arguments you do not want to include.
        value: A Dict with key `class` where the classname is.
        force_kwargs: If True will send all supplied kwargs to the class.
                        This can break the supplied class
                        if they don't accept extra parameters in **kwargs.

    Returns:
        object, the loaded object.
    """
    try:
        func = load_obj(value.class_name)
    except (ModuleNotFoundError, AttributeError) as e:
        raise AzimuthValidationError(f"Invalid class_name {repr(value.class_name)}: {e}")

    varnames = get_class_init_varnames(func) if inspect.isclass(func) else func.__code__.co_varnames

    not_present = {k: v for k, v in kwargs.items() if k not in varnames}

    # Manage kwargs (we override the value if needed here!!)
    kwargs_ins = {**value.kwargs, **kwargs}
    # Argument supplied by us can't modify **kwargs from the method.
    kwargs_ins = {k: v for k, v in kwargs_ins.items() if k not in not_present or force_kwargs}

    if reject is not None:
        kwargs_ins = {k: v for k, v in kwargs.items() if k not in reject}
    kwargs_ins = {k: load_args(v, kwargs) for k, v in kwargs_ins.items()}

    try:
        obj = func(*value.args, **kwargs_ins)
    except Exception as e:
        raise AzimuthValidationError(e)
    return obj

def load_obj(cls):
    """Get the function object and load the required modules."""
    mod, cls = cls.rsplit(".", 1)
    return getattr(importlib.import_module(mod), cls)


import random
class students:
    def __init__(self):
        self.name = input(" please enter the student name:")
        self.age = input(" please enter the student Age:")
        self.gender = input(" please enter the student gender:")
        self.rollno = str(random.randrange(0,50))
    def student_details(self):
        print("new student name is : " + self.name + " age = "+ self.age + "gender: " + self.gender +  " assigned rollnumber :" + self.rollno)

class first_std(students):
    def __init__(self):
        super().__init__()
        self.sub = input(" please enter the student sub:")

    # def student_details(self):
    #     print("gender: "+ self.gender)

class sec_std(students):
    pass

A1 = students()
A1.student_details()
A2 = first_std()
A2.student_details()
A3 = sec_std()
A3.student_details()
# A4 = students()
# A4.student_details()


# Find the missing number(type : easy; Asked in global analytics for optimized solution(XOR))
#For a given list with numbers from 1 to n(in this case n=10), find the missing number in the list
#Fast solution
a=[1,2,4,3,6,7,9,8,10]
n = len(a)
#Sum of n numbers is N*N+1/2
print((((n+1)/2)*(2+n))-sum(a))
#Space complexity : O(n)
#Drawback: If n is very large, the sum of n numbers (n*(n+1)/2) can give an overflow error, since n+1 could be larger than the largest value
# allowed to be stored in memory
#Faster Solution
a=[1,2,4,3,6,7,9,8,10]

#USing XOR operation, the XOR of the XOR of all numbers upto n and XOR of all numbers in a list will be the missing number
X1 = a[0]
X2 = 1
for i in range(1,len(a)):
    X1^=a[i]

for i in range(2,len(a)+2):
    X2^=i

print(X1^X2)


import pytest
import gentrade.util.growtree as growtree
import talib as ta
import talib.abstract as taa
from deap import gp
from pytest_cases import parametrize_with_cases

class TreeCases:

    def case_tree1(self, ohlcv):
        tree_str = "ge(PPO(transform_price(ohlcv, 'close'), 29, 434, 5), MIDPRICE(ohlcv, 885))"
        ppo = ta.PPO(ohlcv.close, fastperiod=547, slowperiod=735, matype=1)
        signals_ref = ppo >= ta.MIDPRICE(ohlcv.high, ohlcv.low, timeperiod=885)
        return tree_str, signals_ref

    def case_tree2(self, ohlcv):
        tree_str = "lt(SMA(transform_price(ohlcv, 'close'), 100), SMA(transform_price(ohlcv, 'open'), 200))"
        signals_ref = ta.SMA(ohlcv.close, timeperiod=100) < ta.SMA(ohlcv.open, timeperiod=200)
        return tree_str, signals_ref

    def case_tree3(self, ohlcv):
        tree_str = "ge(fama(transform_price(ohlcv, 'medprice'), 0.18, 4), TRIMA(transform_price(ohlcv, 'low'), 262))"
        fastlimit, limit_div = 0.18, 4
        slowlimit = round(fastlimit / limit_div, 2)
        slowlimit = min(0.99, max(0.01, slowlimit))
        medprice = ta.abstract.MEDPRICE(ohlcv)
        fama = ta.MAMA(medprice, fastlimit, slowlimit)[1]
        trima = ta.TRIMA(ohlcv.low, 262)
        signals_ref = fama >= trima
        return tree_str, signals_ref

@parametrize_with_cases('tree,signals_ref', cases=TreeCases)
def test_tree(tree, signals_ref, ohlcv, pset_ta):
    f = gp.compile(tree, pset_ta)
    signals_test = f(ohlcv, ohlcv.open, ohlcv.high, ohlcv.low, ohlcv.close, ohlcv.volume)
    assert len(signals_ref) == len(signals_test)
    assert (signals_ref == signals_test).all()

@pytest.mark.parametrize('tree_size', [(2, 5), (2, 10)])
@pytest.mark.parametrize('gen_func', [growtree.genGrow, growtree.genHalfAndHalf, growtree.genFull])
def test_gen_grow(tree_size, gen_func, pset_ta):
    min_, max_ = tree_size
    min_height, max_height = max_, 0
    population = []
    for _ in range(1000):
        try:
            tree = gen_func(pset_ta, min_=min_, max_=max_)
            prim_tree = gp.PrimitiveTree(tree)
        except Exception as e:
            pass
        else:
            min_height = min(min_height, prim_tree.height)
            max_height = max(max_height, prim_tree.height)
            population.append(tree)
    assert min_height >= min_
    assert max_height <= max_
    assert len(population) == 1000


#Packages
import pandas as pd
import numpy as np
import pyreadr as pyr
import sklearn as sk
import scipy as sc
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from statsmodels.formula.api import ols
import statsmodels.api as sm
from statsmodels.stats.outliers_influence import OLSInfluence
from sklearn.metrics import mean_squared_error
from matplotlib import pyplot as plt
import seaborn as sns
from sklearn.model_selection import LeaveOneOut
from sklearn.model_selection import cross_val_score
from bootstrap_stat import bootstrap_stat as bp
from scipy.stats import bootstrap
from sklearn.impute import KNNImputer

#Setting work space and master data frame:
bd=pyr.read_r("bdPS1.Rdata")
print(bd.keys()) #Load database.
df0=bd["df"]
df=df0.drop(df0[df0.age<18].index) #Restrict the data to legal age individuals (age>=18).
df=df.drop(df[df.ocu==0].index) #Restrict the data to employed individuals (ocu==1).
dfsi=(df[["age", "maxEducLevel", "sizeFirm", "oficio", "hoursWorkUsual", "formal", "ingtot", "mes", "estrato1", "sex",
"regSalud", "p6426", "p7090","p7510s7a1", "impa", "isa", "impaes", "isaes", "relab"]]) #Relevant variables selection.

#Reassining variables codification:
dfsi["impa"].replace(0, 1, inplace=True)
dfsi["impa"].replace(np.nan, 0, inplace=True)
dfsi["impaes"].replace(0, 1, inplace=True)
dfsi["impaes"].replace(np.nan, 0, inplace=True)
dfsi["isa"].replace(0, 1, inplace=True)
dfsi["isa"].replace(np.nan, 0, inplace=True)
dfsi["isaes"].replace(0, 1, inplace=True)
dfsi["isaes"].replace(np.nan, 0, inplace=True)

#Create compiled variables:
dfsi["impacomp"]=dfsi["impa"]+dfsi["impaes"]
dfsi["isacomp"]=dfsi["isa"]+dfsi["isaes"]

#Recover missing values:
dfsi['impacomp']=dfsi['impacomp'].replace(0,np.nan)
dfsi['isacomp']=dfsi['isacomp'].replace(0,np.nan)
dfsi['impacomp']=dfsi['impacomp'].replace(1, 0)
dfsi['isacomp']=dfsi['isacomp'].replace(1, 0)
dfsi['impacomp']=dfsi['impacomp'].replace(2, 0)
dfsi['isacomp']=dfsi['isacomp'].replace(2, 0)

#Missing values count/share.
dfsi.isnull().sum()
dfsi.isnull().sum()/len(dfsi)

#Imputing missing values:
imputer=KNNImputer(n_neighbors=19)
imputer.fit(dfsi)
dfi=pd.DataFrame(imputer.transform(dfsi))
dict=({0: "age", 1: "maxEducLevel", 2: "sizeFirm", 3: "oficio", 4: "hoursWorkUsual", 5: "formal", 6: "ingtot", 7: "mes", 8: "estrato1",
9: "sex", 10: "regSalud", 11: "p6426", 12: "p7090", 13: "p7510s7a1", 14: "impa", 15: "isa", 16: "impaes", 17: "isaes", 18: "relab",
19: "impacomp", 20: "isacomp"})
dfi.rename(columns=dict, inplace=True)

#Convert categorical variables to dummy variables:
oficio_d=pd.get_dummies(dfi["oficio"], prefix="oficio")
sizeFirm_d=pd.get_dummies(dfi["sizeFirm"], prefix="size")
estrato1_d=pd.get_dummies(dfi["estrato1"], prefix="estrato")
regSalud_d=pd.get_dummies(dfi["regSalud"], prefix="regs")
maxeduc_d=pd.get_dummies(dfi["maxEducLevel"], prefix="educ")
relab_d=pd.get_dummies(dfi["relab"], prefix="relab")
mes_d=pd.get_dummies(dfi["mes"], prefix="mes")

#Merge dummy's variables data frame with master data frame:
dfi=pd.merge(dfi, oficio_d, left_index=True, right_index=True)
dfi=pd.merge(dfi, sizeFirm_d, left_index=True, right_index=True)
dfi=pd.merge(dfi, estrato1_d, left_index=True, right_index=True)
dfi=pd.merge(dfi, regSalud_d, left_index=True, right_index=True)
dfi=pd.merge(dfi, maxeduc_d, left_index=True, right_index=True)
dfi=pd.merge(dfi, relab_d, left_index=True, right_index=True)
dfi=pd.merge(dfi, mes_d, left_index=True, right_index=True)
dfi["p7090"]=np.where(dfi["p7090"]==2, 0, 1)

#Transform categorial atribute variables to numerical variable:
dfi["hoursWorkUsual"]=pd.to_numeric(dfi["hoursWorkUsual"])
dfi["formal"]=pd.to_numeric(dfi["formal"])
dfi["p6426"]=pd.to_numeric(dfi["p6426"])

#Descriptive statistics data frame:
ds=(dfi[["age", "hoursWorkUsual", "p6426", "sex", "formal", "p7090", "estrato_1.0", "estrato_2.0", "estrato_3.0",
"estrato_4.0", "estrato_5.0", "estrato_6.0", "regs_1.0", "regs_2.0", "regs_3.0", "educ_1.0", "educ_3.0", "educ_4.0",
"educ_5.0", "educ_6.0", "educ_7.0"]].describe(include="all"))
ds=ds.T
ds=ds[["count", "mean", "std", "min", "50%", "max"]]
ds=ds.round(2)
print(ds.to_latex())

#Create dependent variable:
dfi["earnings"]=dfi["impacomp"]+dfi["isacomp"]
dfi["earnings"].describe()

#Winsorize data:
sc.stats.mstats.winsorize(dfi["earnings"], limits=[0,0.05], inplace=True)
sc.stats.mstats.winsorize(dfi["age"], limits=[0,0.05], inplace=True)
dfi["earnings"].describe()

#Create cuadratic age variable:
dfi["age2"]=dfi["age"]**2

#OLS regresion model:
y=dfi["earnings"]
x=dfi[["age", "age2"]]
x=sm.add_constant(x)
model1=sm.OLS(y,x)
results=model1.fit()
yhatp1=results.predict()
b1=results.params[1]
b2=results.params[2]
elasticidad=-b1/2*b2
print(results.summary())
print(np.sqrt(results.mse_resid))

#Bootstrap
seboot=[0]*3
for j in range(0,3):
    def estimador(dfi):
        y=dfi["earnings"]
        x=dfi[["age", "age2"]]
        x=sm.add_constant(x)
        model1=sm.OLS(y,x)
        results= model1.fit()
        return results.params[j]

    dist = bp.EmpiricalDistribution(dfi)
    sea=bp.standard_error(dist, estimador, B=1000)
    seboot[j]=sea

b0=results.params[0]
b1=results.params[1]
b2=results.params[2]

b0low=b0-seboot[0]*1.96
b1low=b1-seboot[1]*1.96
b2low=b2-seboot[2]*1.96

b0upper=b0+seboot[0]*1.96
b1upper=b1+seboot[1]*1.96
b2upper=b2+seboot[2]*1.96

yhatlow=b0low+b1low*dfi['age']+b2low*dfi['age2']
yhatupper=b0upper+b1upper*dfi['age']+b2upper*dfi['age2']

#Figura 1
x_f1=pd.DataFrame(dfi['age'])
x_f1["Edad"]=x_f1
y_f1=pd.DataFrame(yhatp1)
y_f1["COP"]=y_f1
z_f1=pd.DataFrame(yhatlow)
z_f1["3"]=z_f1
w_f1=pd.DataFrame(yhatupper)
w_f1["4"]=w_f1

plot_data1=pd.concat([x_f1,y_f1,z_f1,w_f1], axis=1, join="inner")
plot_data1

fig, ax = plt.subplots()
ax= sns.lineplot(x='Edad', y='COP', data=plot_data1, label='Ingreso laboral', linewidth=2, color="black")
ax2 = sns.lineplot(x='Edad', y='4', data=plot_data1, label='IC banda superior', linewidth=1, color="gray")
ax1 = sns.lineplot(x='Edad', y='3', data=plot_data1, label='IC banda inferior', linewidth=1, color="gray")

#Create female variable:
dfi["female"]=np.where(dfi["sex"]==1, 0, 1)
dfi=dfi.drop(dfi[dfi.earnings==0].index)
dfi["log_earnings"]=np.log(dfi["earnings"])
dfi["log_earnings"].describe()

#OLS regresion model:
y=dfi["log_earnings"]
x=dfi["female"]
x=sm.add_constant(x)
model2=sm.OLS(y,x)
results= model2.fit()
print(results.summary())
print(np.sqrt(results.mse_resid))

#rMSE to COP units:
y_hat=results.predict()
from math import e
y_hat2=e**(y_hat)
E = dfi["earnings"]-y_hat2
E2 = E**2
MSE=sum(E2)/(len(E2)-2)
print(np.sqrt(MSE))

#Create interactions variable:
dfi["female_age"]=dfi["female"]*dfi["age"]
dfi["female_age2"]=dfi["female"]*dfi["age2"]

#OLS regresion model:
y=dfi["log_earnings"]
x=dfi[["age", "female", "female_age", "female_age2", "age2"]]
x=sm.add_constant(x)
model3=sm.OLS(y,x)
results= model3.fit()
yhatp3=results.predict()
print(results.summary())
results.params

#rMSE to COP units:
y_hat=results.predict()
from math import e
y_hat2=e**(y_hat)
E = dfi["earnings"]-y_hat2
E2 = E**2
MSE=sum(E2)/(len(E2)-6)
print(np.sqrt(MSE))

#Bootstrap
seboot=[0]*6
for j in range(0,6):
    def estimador3(dfi):
        y=dfi["log_earnings"]
        x=dfi[["age", "female", "female_age", "female_age2", "age2"]]
        x=sm.add_constant(x)
        model1=sm.OLS(y,x)
        results= model1.fit()
        return results.params[j]

    dist = bp.EmpiricalDistribution(dfi)
    sea=bp.standard_error(dist, estimador3, B=1000)
    seboot[j]=sea

seboot

for k in range(0,6):
    locals()['b'+str(k)]=results.params[k]
    locals()['b'+str(k)+'low']=locals()['b'+str(k)]-seboot[k]*1.96
    locals()['b'+str(k)+'upper']=locals()['b'+str(k)]+seboot[k]*1.96

yhatlow=b0low+b1low*dfi['age']+b5low*dfi['age2']+b2low*dfi['female']+b3low*dfi['female_age']+b4low*dfi['female_age2']
yhatupper=b0upper+b1upper*dfi['age']+b5upper*dfi['age2']+b2upper*dfi['female']+b3upper*dfi['female_age']+b4upper*dfi['female_age2']

yhatupperh= np.ma.masked_where(dfi['female']==0, yhatupper)
yhatlowh = np.ma.masked_where(dfi['female']==0, yhatlow)

yhatupperm= np.ma.masked_where(dfi['female']==1, yhatupper)
yhatlowm = np.ma.masked_where(dfi['female']==1,yhatlow )

yhatp3m=np.ma.masked_where(dfi['female']==1, yhatp3)
yhatp3h=np.ma.masked_where(dfi['female']==0, yhatp3)

agem=np.ma.masked_where(dfi['female']==1, dfi['age'])
ageh=np.ma.masked_where(dfi['female']==0, dfi['age'])

#Figura 2m
x_f2m=pd.DataFrame(agem)
x_f2m["Edad"]=x_f2m
y_f2m=pd.DataFrame(yhatp3m)
y_f2m["COP"]=y_f2m
z_f2m=pd.DataFrame(yhatlowm)
z_f2m["3"]=z_f2m
w_f2m=pd.DataFrame(yhatupperm)
w_f2m["4"]=w_f2m

plot_data2m=pd.concat([x_f2m,y_f2m,z_f2m,w_f2m], axis=1, join="inner")
plot_data2m

fig, ax = plt.subplots()
ax= sns.lineplot(x='Edad', y='COP', data=plot_data2m, label='Ingreso laboral mujeres', linewidth=2, color="red")
ax2 = sns.lineplot(x='Edad', y='4', data=plot_data2m, label='IC banda superior', linewidth=1, color="orange")
ax1 = sns.lineplot(x='Edad', y='3', data=plot_data2m, label='IC banda inferior', linewidth=1, color="orange")

#Figura 2h
x_f2h=pd.DataFrame(ageh)
x_f2h["Edad"]=x_f2h
y_f2h=pd.DataFrame(yhatp3h)
y_f2h["COP"]=y_f2h
z_f2h=pd.DataFrame(yhatlowh)
z_f2h["3"]=z_f2h
w_f2h=pd.DataFrame(yhatupperh)
w_f2h["4"]=w_f2h

plot_data2h=pd.concat([x_f2h,y_f2h,z_f2h,w_f2h], axis=1, join="inner")
plot_data2h

fig, ax = plt.subplots()
ax= sns.lineplot(x='Edad', y='COP', data=plot_data2h, label='Ingreso laboral hombres', linewidth=2, color="blue")
ax2 = sns.lineplot(x='Edad', y='4', data=plot_data2h, label='IC banda superior', linewidth=1, color="green")
ax1 = sns.lineplot(x='Edad', y='3', data=plot_data2h, label='IC banda inferior', linewidth=1, color="green")

oficio=["oficio_1.0"]
for i in range(2,99):
    oficio.append("oficio_"+str(i)+".0")
    list1=['oficio_10.0', 'oficio_22.0', 'oficio_23.0', 'oficio_24.0', 'oficio_25.0', 'oficio_26.0', 'oficio_27.0', 'oficio_28.0', 'oficio_29.0', 'oficio_46.0', 'oficio_47.0', 'oficio_48.0', 'oficio_64.0', 'oficio_65.0', 'oficio_66.0', 'oficio_67.0', 'oficio_68.0', 'oficio_69.0', 'oficio_71.0']
    oficio2=[i for i in oficio if i not in list1]

#Long regression
#Define exogenous and endogenous variables:
y=dfi["log_earnings"]
x0=dfi[["female", "age", "age2", "female_age", "female_age2"]]
x1=(dfi[["sizeFirm", "hoursWorkUsual", "formal", "p6426", "educ_3.0", "educ_4.0",
"educ_5.0", "educ_6.0", "educ_7.0", "relab_2.0", "relab_3.0" , "relab_4.0", "relab_5.0", "relab_6.0",
"relab_7.0", "relab_8.0" , "relab_9.0", "mes_1.0", "mes_2.0", "mes_3.0", "mes_4.0", "mes_5.0", "mes_6.0",
"mes_7.0", "mes_9.0", "mes_10.0", "mes_11.0", "mes_12.0"]])
x2=dfi[oficio2]
x=pd.concat([x0,x1,x2], axis=1, join="inner")
x=sm.add_constant(x)

#OLS regresion model:
model2=sm.OLS(y,x)
results= model2.fit()
print(results.summary())
print(np.sqrt(results.mse_resid))

#rMSE to COP units:
y_hat=results.predict()
from math import e
y_hat2=e**(y_hat)
E = dfi["earnings"]-y_hat2
E2 = E**2
MSE=sum(E2)/(len(E2)-131)
print(np.sqrt(MSE))

#FWL theorem
#Define exog and endog variables
y=dfi["log_earnings"]
x0=dfi[["female", "age", "age2", "female_age", "female_age2"]]
x_f=pd.concat([x1,x2], axis=1, join="inner")
x_f=sm.add_constant(x_f)
x0=sm.add_constant(x0)

#Residuals of log_earnings regressed on controls
model3=sm.OLS(y,x_f)
results_f= model3.fit()
resid_a=results_f.resid

#Residuals of female regressed on controls
model4=sm.OLS(x0, x_f)
results_f2= model4.fit()
resid_b=results_f2.resid

#Delete constant from residuals b
resid_b=resid_b.drop(["const"], axis=1)

#Results of resid_a  regressed on resid_b
model_FWL=sm.OLS(resid_a,resid_b)
results_FWL=model_FWL.fit()
print(results_FWL.summary())
print(np.sqrt(results.mse_resid))

#rMSE to COP units:
y_hat=results.predict()
from math import e
y_hat2=e**(y_hat)
E = dfi["earnings"]-y_hat2
E2 = E**2
MSE=sum(E2)/(len(E2)-5)
print(np.sqrt(MSE))

#Bootstrap FWL
se_boot = []
for koef in range(0,5):

    def estm(dfi):

        #FWL theorem
        #Define exog and endog variables
        oficio=["oficio_1.0"]
        for i in range(2,99):
            oficio.append("oficio_"+str(i)+".0")
            list1=['oficio_10.0', 'oficio_22.0', 'oficio_23.0', 'oficio_24.0', 'oficio_25.0', 'oficio_26.0', 'oficio_27.0', 'oficio_28.0', 'oficio_29.0', 'oficio_46.0', 'oficio_47.0', 'oficio_48.0', 'oficio_64.0', 'oficio_65.0', 'oficio_66.0', 'oficio_67.0', 'oficio_68.0', 'oficio_69.0', 'oficio_71.0']
            oficio2=[i for i in oficio if i not in list1]

        y=dfi["log_earnings"]
        x0=dfi[['female', "age", "age2", "female_age", "female_age2"]]
        x1=dfi[["sizeFirm", "hoursWorkUsual","formal", "p6426","educ_3.0", "educ_4.0",
            "educ_5.0", "educ_6.0", "educ_7.0", "relab_2.0", "relab_3.0" , "relab_4.0","relab_5.0" , "relab_6.0" , "relab_7.0" , "relab_8.0" , "relab_9.0",
            "mes_1.0", "mes_2.0", "mes_3.0", "mes_4.0", "mes_5.0", "mes_6.0", "mes_7.0","mes_9.0", "mes_10.0", "mes_11.0", "mes_12.0"]]
        x2=dfi[oficio2]
        x_f = pd.concat([x1,x2], axis=1, join="inner")
        x_f=sm.add_constant(x_f)
        x0=sm.add_constant(x0)

        #Residuals of log_earnings regressed on controls
        model3=sm.OLS(y,x_f)
        results_f= model3.fit()
        resid_a=results_f.resid

        #residuals of female regressed on controls
        model4=sm.OLS(x0, x_f)
        results_f2= model4.fit()
        resid_b=results_f2.resid

        #Delete constant from residuals b
        resid_b=resid_b.drop(['const'], axis=1)

        #Results of resid_a  regressed on resid_b
        model_FWL=sm.OLS(resid_a,resid_b)
        results_FWL=model_FWL.fit()

        return results_FWL.params[koef]

    dist = bp.EmpiricalDistribution(dfi)
    se_koef=bp.standard_error(dist,estm, B=5)
    se_boot = se_boot + [str(se_koef)]
    print(se_koef)

#Create new variables
dfi['age_hours']=dfi['age']*dfi['hoursWorkUsual']
dfi['hours2']=dfi['hoursWorkUsual']**2
dfi['age_hours2']=dfi['age']*dfi['hours2']

for j in range(1,8):
    if j==2:
        pass
    else:
        dfi["female_educ"+str(j)]=dfi["female"]*dfi["educ_"+str(j)+".0"]

for j in range(1,10):
        dfi["female_relab"+str(j)]=dfi['female']*dfi["relab_"+str(j)+".0"]

for j in range(1,8):
    if j==2:
        pass
    else:
        dfi["age_educ"+str(j)]=dfi["age"]*dfi["educ_"+str(j)+".0"]

#Create new variables
dfi["p6426_2"]=dfi["p6426"]**2
dfi["p6426_3"]=dfi["p6426"]**3
dfi["p6426_age"]=dfi["p6426"]*dfi["age"]
dfi["p6426_2_age"]=dfi["p6426_2"]*dfi["age"]
dfi["p6426_3_age"]=dfi["p6426_3"]*dfi["age"]

dfi["p7510s7a1_hours"]=dfi["p7510s7a1"]*dfi["hoursWorkUsual"]

#Prediction
#Split the data
np.random.seed(500)
dfi_train, dfi_test = train_test_split(
                                dfi,
                                test_size=.3,
                                random_state=911)

#Define X matrix
oficio=["oficio_1.0"]
for i in range(2,99):
    oficio.append("oficio_"+str(i)+".0")
    list1=['oficio_10.0', 'oficio_22.0', 'oficio_23.0', 'oficio_24.0', 'oficio_25.0', 'oficio_26.0', 'oficio_27.0', 'oficio_28.0', 'oficio_29.0', 'oficio_46.0', 'oficio_47.0', 'oficio_48.0', 'oficio_64.0', 'oficio_65.0', 'oficio_66.0', 'oficio_67.0', 'oficio_68.0', 'oficio_69.0', 'oficio_71.0']
    oficio2=[i for i in oficio if i not in list1]

x1=(dfi_train[["sizeFirm", "hoursWorkUsual", "formal", "p6426", "educ_3.0", "educ_4.0",
"educ_5.0", "educ_6.0", "educ_7.0", "relab_2.0", "relab_3.0" , "relab_4.0", "relab_5.0", "relab_6.0",
"relab_7.0", "relab_8.0" , "relab_9.0", "mes_1.0", "mes_2.0", "mes_3.0", "mes_4.0", "mes_5.0", "mes_6.0",
"mes_7.0", "mes_9.0", "mes_10.0", "mes_11.0", "mes_12.0"]])
x2=dfi_train[oficio2]
X_train=pd.concat([x1,x2], axis=1, join="inner")

x11=(dfi_test[["sizeFirm", "hoursWorkUsual", "formal", "p6426", "educ_3.0", "educ_4.0",
"educ_5.0", "educ_6.0", "educ_7.0", "relab_2.0", "relab_3.0" , "relab_4.0", "relab_5.0", "relab_6.0",
"relab_7.0", "relab_8.0" , "relab_9.0", "mes_1.0", "mes_2.0", "mes_3.0", "mes_4.0", "mes_5.0", "mes_6.0",
"mes_7.0", "mes_9.0", "mes_10.0", "mes_11.0", "mes_12.0"]])
x22=dfi_test[oficio2]
X_test=pd.concat([x11,x22], axis=1, join="inner")

rmsemodels=[0]*10

#Models
#Model 1 earnings age age2
y_train=dfi_train["log_earnings"]
y_test=dfi_test["log_earnings"]
x_train=dfi_train[["age", "age2"]]
x_test=dfi_test[["age", "age2"]]
model1=LinearRegression(fit_intercept=True)
results1= model1.fit(x_train,y_train)
y_hattest=results1.predict(x_test)

rmse1=mean_squared_error(y_test,y_hattest, squared=False)
rmse1
rmsemodels[0]=rmse1
rmse1

#Models
#Model 2 log_earnings female
y_train=dfi_train["log_earnings"]
y_test=dfi_test["log_earnings"]
x_train=dfi_train[["female"]]
x_test=dfi_test[["female"]]
model2=LinearRegression(fit_intercept=True)
results2= model2.fit(x_train,y_train)
y_hattest=results2.predict(x_test)
rmse2=mean_squared_error(y_test,y_hattest, squared=False)
rmse2
rmsemodels[1]=rmse2

#Models
#Model 3 log_earnings female age age2 agefemale agefemale
y_train=dfi_train["log_earnings"]
y_test=dfi_test["log_earnings"]
x_train=dfi_train[["female", "age", "age2", "female_age", "female_age2"]]
x_test=dfi_test[["female", "age", "age2", "female_age", "female_age2"]]
model3=LinearRegression(fit_intercept=True)
results3= model3.fit(x_train,y_train)
y_hattest=results3.predict(x_test)
rmse3=mean_squared_error(y_test,y_hattest, squared=False)
rmse3
rmsemodels[2]=rmse3

#Models
#Model 4 log_earnings female age age2 agefemale age2female X
y_train=dfi_train["log_earnings"]
y_test=dfi_test["log_earnings"]

x_trainsub=dfi_train[["female", "age", "age2", "female_age", "female_age2"]]
x_testsub=dfi_test[["female", "age", "age2", "female_age", "female_age2"]]

x_train=pd.concat([X_train,x_trainsub], axis=1, join="inner")
x_test=pd.concat([X_test,x_testsub], axis=1, join="inner")

model4=LinearRegression(fit_intercept=True)
results4= model4.fit(x_train,y_train)
y_hattest=results4.predict(x_test)
rmse4=mean_squared_error(y_test,y_hattest, squared=False)
rmse4

rmsemodels[3]=rmse4

#Models
#Model 5 age age2 hoursworkusual hoursworkusual2 agehoursworkusual agehoursworkusual2 X
y_train=dfi_train["log_earnings"]
y_test=dfi_test["log_earnings"]

x_trainsub=dfi_train[["age", "age2", "hours2", "age_hours" , "age_hours2"]]
x_testsub=dfi_test[["age", "age2", "hours2", "age_hours" , "age_hours2"]]

x_train=pd.concat([X_train,x_trainsub], axis=1, join="inner")
x_test=pd.concat([X_test,x_testsub], axis=1, join="inner")

model5=LinearRegression(fit_intercept=True)
results5= model5.fit(x_train,y_train)
y_hattest=results5.predict(x_test)
rmse5=mean_squared_error(y_test,y_hattest, squared=False)
rmse5

rmsemodels[4]=rmse5

#Models
#Model 6 age age2 female educ relab formal oficio p7090 mes reg_salud estrato
y_train=dfi_train["log_earnings"]
y_test=dfi_test["log_earnings"]

educ=["educ_1.0"]
for i in range(3,7):
    educ.append("educ_"+str(i)+".0")
x_educt=dfi_train[educ]
x_eductt=dfi_test[educ]

relab=["relab_2.0"]
for i in range(3,9):
    relab.append("relab_"+str(i)+".0")
x_relabt=dfi_train[relab]
x_relabtt=dfi_test[relab]

regs=["regs_1.0"]
for i in range(2,3):
    regs.append("regs_"+str(i)+".0")
x_regst=dfi_train[regs]
x_regstt=dfi_test[regs]

mes=["mes_1.0"]
for i in range(2,12):
    mes.append("mes_"+str(i)+".0")
x_mest=dfi_train[mes]
x_mestt=dfi_test[mes]

estrato2=["estrato_1.0"]
for i in range(2,6):
    estrato2.append("estrato_"+str(i)+".0")
x_est=dfi_train[estrato2]
x_estt=dfi_test[estrato2]

oficio=["oficio_1.0"]
for i in range(2,99):
    oficio.append("oficio_"+str(i)+".0")
    list1=['oficio_10.0', 'oficio_22.0', 'oficio_23.0', 'oficio_24.0', 'oficio_25.0', 'oficio_26.0', 'oficio_27.0', 'oficio_28.0', 'oficio_29.0', 'oficio_46.0', 'oficio_47.0', 'oficio_48.0', 'oficio_64.0', 'oficio_65.0', 'oficio_66.0', 'oficio_67.0', 'oficio_68.0', 'oficio_69.0', 'oficio_71.0']
    oficio2=[i for i in oficio if i not in list1]
x_oficiot=dfi_train[oficio2]
x_oficiott=dfi_test[oficio2]

x_trainsub=dfi_train[["age", "age2", "female" , "p7090" ]]
x_testsub=dfi_test[["age", "age2", "female" , "p7090" ]]

x_train=pd.concat([x_regst, x_est, x_educt, x_relabt, x_mest, x_oficiot,x_trainsub], axis=1, join="inner")
x_test=pd.concat([x_regstt, x_estt, x_eductt, x_relabtt, x_mestt, x_oficiott,x_testsub], axis=1, join="inner")

#Run Model 6
model6=LinearRegression(fit_intercept=True)
results6= model6.fit(x_train,y_train)
y_hattest=results6.predict(x_test)
rmse6=mean_squared_error(y_test,y_hattest, squared=False)
rmse6

rmsemodels[5]=rmse6

#Models
#Model 7 age age2 female educ femalexeduc p7090 femalexrelab relab X
y_train=dfi_train["log_earnings"]
y_test=dfi_test["log_earnings"]

female_edu=["female_educ1"]
for i in range(3,8):
    female_edu.append("female_educ"+str(i))
x_feduct=dfi_train[female_edu]
x_feductt=dfi_test[female_edu]

female_relab=["female_relab1"]
for i in range(2,10):
    female_relab.append("female_relab"+str(i))
x_ferelabt=dfi_train[female_relab]
x_ferelabtt=dfi_test[female_relab]

x_trainsub=dfi_train[["age", "age2", "female" ,"p7090"]]
x_testsub=dfi_test[["age", "age2", "female" ,"p7090"]]

x_train=pd.concat([X_train ,x_trainsub,x_feduct, x_ferelabt], axis=1, join="inner")
x_test=pd.concat([X_test,x_testsub,x_feductt, x_ferelabtt], axis=1, join="inner")

#Run Model 7
model7=LinearRegression(fit_intercept=True)
results7= model7.fit(x_train,y_train)
y_hattest=results7.predict(x_test)
rmse7=mean_squared_error(y_test,y_hattest, squared=False)
rmse7

rmsemodels[6]=rmse7

#Models
#Model 8 age age2  p7510s7a1 p7090 X p7510s7a1_hours hoursworkusual
y_train=dfi_train["log_earnings"]
y_test=dfi_test["log_earnings"]

x_trainsub=dfi_train[["age", "age2", "p7510s7a1" , "p7090", "p7510s7a1_hours" ]]
x_testsub=dfi_test[["age", "age2",  "p7510s7a1" , "p7090", "p7510s7a1_hours" ]]

x_train=pd.concat([X_train,x_trainsub], axis=1, join="inner")
x_test=pd.concat([X_test, x_testsub], axis=1, join="inner")

model8=LinearRegression(fit_intercept=True)
results8= model8.fit(x_train,y_train)
y_hattest=results8.predict(x_test)
rmse8=mean_squared_error(y_test,y_hattest, squared=False)
rmse8

rmsemodels[7]=rmse8

#Models
#Model 9 p6426 p6426_2 p6426_3 age p7090 age2 p6426Xage p6426_2age p6426_3age  X
y_train=dfi_train["log_earnings"]
y_test=dfi_test["log_earnings"]

x_trainsub=dfi_train[["p6426_2" , "p6426_3" , "age" , "age2" , "p6426_age" , "p6426_2_age" , "p6426_3_age"]]
x_testsub=dfi_test[["p6426_2" , "p6426_3" , "age" , "age2" , "p6426_age" , "p6426_2_age" , "p6426_3_age"]]

x_train=pd.concat([X_train,x_trainsub], axis=1, join="inner")
x_test=pd.concat([X_test, x_testsub], axis=1, join="inner")

model9=LinearRegression(fit_intercept=True)
results9= model9.fit(x_train,y_train)
y_hattest=results9.predict(x_test)
rmse9=mean_squared_error(y_test,y_hattest, squared=False)
rmse9

rmsemodels[8]=rmse9

#Models
#Model 10  agexeduc educ age p7090 age2 X
y_train=dfi_train["log_earnings"]
y_test=dfi_test["log_earnings"]

x_trainsub=dfi_train[["age" , "p7090" , "age2"]]
x_testsub=dfi_test[["age" , "p7090" , "age2"]]

x_train=pd.concat([X_train,x_trainsub], axis=1, join="inner")
x_test=pd.concat([X_test, x_testsub], axis=1, join="inner")

model10=LinearRegression(fit_intercept=True)
results10= model10.fit(x_train,y_train)
y_hattest=results10.predict(x_test)
rmse10=mean_squared_error(y_test,y_hattest, squared=False)
rmse10

rmsemodels[9]=rmse10

#Influence statistic
#Model 5
y_train=dfi_train["log_earnings"]
y_test=dfi_test["log_earnings"]

x_trainsub=dfi_train[["age", "age2", "hours2", "age_hours" , "age_hours2"]]
x_testsub=dfi_test[["age", "age2",  "hours2", "age_hours" , "age_hours2"]]

x_train=pd.concat([X_train,x_trainsub], axis=1, join="inner")
x_test=pd.concat([X_test,x_testsub], axis=1, join="inner")

#compute diagonal of PX
olsmod= sm.OLS(y_test, x_test)
olsmodfit=olsmod.fit()
a=OLSInfluence(olsmodfit)
h=a.hat_matrix_diag

#Residuals in test
olsmod2= sm.OLS(y_train, x_train)
olsmodfit2=olsmod2.fit()
y_hattest2=olsmodfit2.predict(x_test)
residuals2=y_test-y_hattest

residuals2=np.array(residuals2)

#Create the influence for all j in test
influence=[0.0]*4880
influence=np.array(influence)
influence

for j in range(0,4880):
    influence[j]=residuals2[j]/(1-h[j])

plt.xlabel('Valor del estadistico de influencia')
plt.ylabel('Frecuencia')
n, bins, patches = plt.hist(influence , bins=100, range=[0,1] ,facecolor='g', alpha=0.2)
plt.savefig('influencia.png')

#CrossValidation
#Model 5 age age2 hoursworkusual hoursworkusual2 agehoursworkusual agehoursworkusual2 X
#Define Cross-validation variables
y_cv=dfi["log_earnings"]
x_cvsub=dfi[["age", "age2", "hours2", "age_hours" , "age_hours2"]]

oficio=["oficio_1.0"]
for i in range(2,99):
    oficio.append("oficio_"+str(i)+".0")
    list1=['oficio_10.0', 'oficio_22.0', 'oficio_23.0', 'oficio_24.0', 'oficio_25.0', 'oficio_26.0', 'oficio_27.0', 'oficio_28.0', 'oficio_29.0', 'oficio_46.0', 'oficio_47.0', 'oficio_48.0', 'oficio_64.0', 'oficio_65.0', 'oficio_66.0', 'oficio_67.0', 'oficio_68.0', 'oficio_69.0', 'oficio_71.0']
    oficio2=[i for i in oficio if i not in list1]

x1=(dfi[["sizeFirm", "hoursWorkUsual", "formal", "p6426", "educ_3.0", "educ_4.0",
"educ_5.0", "educ_6.0", "educ_7.0", "relab_2.0", "relab_3.0" , "relab_4.0", "relab_5.0", "relab_6.0",
"relab_7.0", "relab_8.0" , "relab_9.0", "mes_1.0", "mes_2.0", "mes_3.0", "mes_4.0", "mes_5.0", "mes_6.0",
"mes_7.0", "mes_9.0", "mes_10.0", "mes_11.0", "mes_12.0"]])
x2=dfi[oficio2]
x_cv0=pd.concat([x1,x2], axis=1, join="inner")

x_cv=pd.concat([x_cv0,x_cvsub], axis=1, join="inner")

#CrossValidation of Model 5 ###############################3
#define cross-validation method to use
cv = LeaveOneOut()
#Define cross-validation model
model5=LinearRegression(fit_intercept=True)
#use LOOCV to evaluate model
scores = cross_val_score(model5, x_cv, y_cv, scoring='neg_root_mean_squared_error',
                         cv=cv, n_jobs=-1)

rMSE5 = np.mean(np.absolute(scores))

#CrossValidation Model 7 age age2 female educ femalexeduc p7090 femalexrelab relab X
#Define cross-validation variables
y_cv2=dfi["log_earnings"]

female_edu=["female_educ1"]
for i in range(3,8):
    female_edu.append("female_educ"+str(i))
x_feduct=dfi[female_edu]

female_relab=["female_relab1"]
for i in range(2,10):
    female_relab.append("female_relab"+str(i))
x_ferelabt=dfi[female_relab]

x_cv2sub=dfi[["age", "age2", "female" ,"p7090"]]

oficio=["oficio_1.0"]
for i in range(2,99):
    oficio.append("oficio_"+str(i)+".0")
    list1=['oficio_10.0', 'oficio_22.0', 'oficio_23.0', 'oficio_24.0', 'oficio_25.0', 'oficio_26.0', 'oficio_27.0', 'oficio_28.0', 'oficio_29.0', 'oficio_46.0', 'oficio_47.0', 'oficio_48.0', 'oficio_64.0', 'oficio_65.0', 'oficio_66.0', 'oficio_67.0', 'oficio_68.0', 'oficio_69.0', 'oficio_71.0']
    oficio2=[i for i in oficio if i not in list1]

x1=(dfi[["sizeFirm", "hoursWorkUsual", "formal", "p6426", "educ_3.0", "educ_4.0",
"educ_5.0", "educ_6.0", "educ_7.0", "relab_2.0", "relab_3.0" , "relab_4.0", "relab_5.0", "relab_6.0",
"relab_7.0", "relab_8.0" , "relab_9.0", "mes_1.0", "mes_2.0", "mes_3.0", "mes_4.0", "mes_5.0", "mes_6.0",
"mes_7.0", "mes_9.0", "mes_10.0", "mes_11.0", "mes_12.0"]])
x2=dfi[oficio2]
x_cv02=pd.concat([x1,x2], axis=1, join="inner")

x_cv2=pd.concat([x_cv2sub,x_feduct, x_ferelabt, x_cv02 ], axis=1, join="inner")

#CrossValidation of Model 7 ###############################3
#define cross-validation method to use
cv = LeaveOneOut()
#Define cross-validation model
model7=LinearRegression(fit_intercept=True)
#use LOOCV to evaluate model
scores2 = cross_val_score(model7, x_cv2, y_cv2, scoring='neg_root_mean_squared_error',
                         cv=cv, n_jobs=-1)

#view RMSE
rMSE7 = np.mean(np.absolute(scores2))


from bint import tokens

class EndOfTokenisation(Exception):
    pass

class BintLexer:
    operator_chars = ['m', 'o', 'd', '+', '-', '*',
            '\\', '<', '>', '=', ',', '(', ')']

    def tokenise(self, text):
        """Tokenises the passed in string. """
        self.text = text
        self.char = 0
        try:
            self.current_char = text[0]
        except IndexError:
            return []  # Empty string

        try:
            self.next_char = text[1]
        except IndexError:
            self.next_char = ''

        parsed_tokens = []

        while True:
            if self.char >= len(self.text):
                break  # All done

            try:
                if self.current_char == '"':
                    parsed_tokens.append(self.read_string())
                elif self.current_char.isdigit():
                    parsed_tokens.append(self.read_number())
                elif self.current_char.isalpha():

                    if text[self.char:].startswith('mod'):
                        parsed_tokens.append(self.read_op())
                    else:
                        parsed_tokens.append(self.read_identifier())

                elif self.current_char in self.operator_chars:
                    parsed_tokens.append(self.read_op())
                elif self.current_char == '\n':
                    parsed_tokens.append(tokens.EndLineToken())
                    self.get_next_char()
                elif self.current_char.isspace():
                    self.get_next_char()

            except EndOfTokenisation:
                break

        return parsed_tokens

    def read_op(self):
        """ Reads an operator from the text to tokenise. """
        op_string = ''

        while self.current_char in self.operator_chars:
            op_string += self.current_char
            try:
                self.get_next_char()
            except EndOfTokenisation:
                return tokens.OpToken(op_string)

        return tokens.OpToken(op_string)

    def read_number(self):
        """ Gets a token of a number in the text. """
        num = 0

        while self.current_char.isdigit():
            num *= 10
            num += int(self.current_char)

            try:
                self.get_next_char()
            except EndOfTokenisation:
                return tokens.NumberToken(num)

        return tokens.NumberToken(num)

    def read_string(self):
        """ Reads in a string token. """
        self.get_next_char()  # Drop opening quote.
        chars = ''
        num_quotes = 0

        while True:
            if self.current_char != '"':
                chars += self.current_char
            elif num_quotes == 1:
                chars += '"'
                num_quotes = 0
            elif num_quotes == 0 and self.next_char == '"':
                num_quotes += 1
            else:
                try:
                    self.get_next_char()  # Drop ending quote
                except EndOfTokenisation:
                    return tokens.StringToken(chars)
                break

            try:
                self.get_next_char()
            except EndOfTokenisation:
                return tokens.StringToken(chars)

        return tokens.StringToken(chars)

    def read_identifier(self):
        """ Reads in a identifier token. """
        chars = ''

        while self.is_identifier_char(self.current_char):
            chars += self.current_char
            try:
                self.get_next_char()
            except EndOfTokenisation:
                return tokens.IdentifierToken(chars)

        return tokens.IdentifierToken(chars)

    def is_identifier_char(self, char):
        """ Checks if the current character is an identifier character.

        Valid identifier characters are [0-9A-Za-z_]
        """
        return (char.isalpha() or char.isdigit() or char == '_')

    def get_next_char(self):
        """ Updates the current character to the next character.

        Throws IndexError if no more characters are available.
        """
        self.char += 1

        print('Parsing char #{}'.format(self.char))

        try:
            self.current_char = self.text[self.char]
        except IndexError:
            raise EndOfTokenisation()

        try:
            self.next_char = self.text[self.char + 1]
        except IndexError:
            self.next_char = ''


import dgl
import networkx as nx
import numpy as np
import torch as th
np.random.seed(0)
th.manual_seed(0)
th.backends.cudnn.deterministic = True
th.backends.cudnn.benchmark = False
from graphgallery.datasets import NPZDataset

def load_graphgallery_data(dataset):
    # set `verbose=False` to avoid additional outputs
    data = NPZDataset(dataset, verbose=False)
    graph = data.graph
    nx_g = nx.from_scipy_sparse_matrix(graph.adj_matrix)

    for node_id, node_data in nx_g.nodes(data=True):
        node_data["features"] = graph.feat[node_id].astype(np.float32)
        if dataset in ['blogcatalog', 'flickr']:
            node_data["labels"] = graph.y[node_id].astype(np.long) - 1
        else:
            node_data["labels"] = graph.y[node_id].astype(np.long)

    dgl_graph = dgl.from_networkx(nx_g, node_attrs=['features', 'labels'])
    dgl_graph = dgl.add_self_loop(dgl_graph)
    dgl_graph = dgl.to_simple(dgl_graph, copy_ndata=True)
    dgl_graph = dgl.to_bidirected(dgl_graph, copy_ndata=True)

    print(f"Graph has {dgl_graph.number_of_nodes()} nodes, {dgl_graph.number_of_edges()} edges.")
    return dgl_graph, graph.num_classes

def node_sample(g, prop=0.5):
    '''
    sample target/shadow graph (1:1)
    '''
    node_number = len(g.nodes())
    node_index_list = np.arange(node_number)
    np.random.seed(0)
    np.random.shuffle(node_index_list)
    split_length = int(node_number * prop)

    train_index = np.sort(node_index_list[:split_length])
    test_index = np.sort(node_index_list[split_length: ])

    return train_index, test_index

def split_target_shadow(g):
    '''
    generate input data for target gnn model
    '''

    target_index, shadow_index = node_sample(g, 0.5)

    target_g = g.subgraph(target_index)
    shadow_g = g.subgraph(shadow_index)

    return target_g, shadow_g

def split_train_test(g):
    '''
    generate input data for target gnn model
    '''

    train_index, test_index = node_sample(g, 0.5)

    train_g = g.subgraph(train_index)
    test_g = g.subgraph(test_index)

    return train_g, test_g


"""Uptime detection class"""

class Uptime:
    """Returns a pretty-formatted string representing the host uptime"""
    def __init__(self):
        with open('/proc/uptime') as file:
            fuptime = int(file.read().split('.')[0])

        day, fuptime = divmod(fuptime, 86400)
        hour, fuptime = divmod(fuptime, 3600)
        minute = fuptime // 60

        uptime = ''
        if day:
            uptime += str(day) + ' day'
            if day > 1:
                uptime += 's'

            if hour or minute:
                if bool(hour) != bool(minute):
                    uptime += ' and '

                else:
                    uptime += ', '

        if hour:
            uptime += str(hour) + ' hour'
            if hour > 1:
                uptime += 's'

            if minute:
                uptime += ' and '

        if minute:
            uptime += str(minute) + ' minute'
            if minute > 1:
                uptime += 's'

        else:
            if not day and not hour:
                uptime = '< 1 minute'

        self.value = uptime


"""
PROJET MORPION
Auteur : Kyllian BEGUIN
Script principal
"""
# ZONE D'IMPORT
from morpion_functions import var_init, display_board, checker, play_movement, victory_check
from random import randint

# ZONE D'INITIALISATION DES VARIABLES
board_size, board_surface = var_init()
victory = False
player = False
round_nb = 1

# ZONE DE CALCUL DES VARIABLES
board = [[str(u + i + 1).zfill(len(list(str(board_surface)))) for i in range(board_size)] for u in
         range(0, board_surface, board_size)]
board_list = [item for sublist in board for item in sublist]

# SCRIPT PRINCIPAL
if __name__ == '__main__':
    while (victory == False) and (round_nb < board_surface):
        display_board(board)
        print("\n\n", "#" * 50, "\n\n", sep='')

        if not player:
            movement = randint(1, 9)
            movement = checker(board, movement)
        elif player:
            movement_str = input("Give a movement: ")
            movement = checker(board, movement_str)

        board = play_movement(movement, board)
        victory = victory_check(board)

        player = not player
        round_nb += 1

    if round_nb == board_surface:
        print("It's a draw !!\n\n")
    display_board(board)


from collections import namedtuple
import pandas as pd

def useNamedTuple(n, colmnNames, data):
    col = colmnNames.split()
    student = namedtuple('student', col)
    total = 0
    for i in range(n):
        # fields could be in different orders
        field1, field2, field3, field4 = data[i].split()

        row = student(field1, field2, field3, field4)
        total += int(row.MARKS)
    print('{:.2f}'.format(total/n))

def usePandas(n, colmnNames, data):
    col = colmnNames.split()
    def item_looper():
        for i in range(n):
            # fields could be in different orders
            field1, field2, field3, field4 = data[i].split()
            yield field1, field2, field3, field4

    df_list = [item_info for item_info in item_looper()]
    df = pd.DataFrame(df_list, columns=col)
    df = df.astype({'MARKS': 'int', 'CLASS': 'long', 'NAME': 'string', 'ID': 'long'})
    df.set_index('ID', inplace=True)
    print('{:.2f}'.format(df['MARKS'].mean(axis=0)))

useNamedTuple(5, "MARKS      CLASS      NAME       ID",
              ["92         2          Calum      1",
               "82         5          Scott      2",
               "94         2          Jason      3",
               "55         8          Glenn      4",
               "82         2          Fergus     5"])

usePandas(5, "MARKS      CLASS      NAME       ID",
          ["92         2          Calum      1",
           "82         5          Scott      2",
           "94         2          Jason      3",
           "55         8          Glenn      4",
           "82         2          Fergus     5"])