# -*- coding: utf-8 -*-

################################################################################################
# Plataforma para la Iniciativa Regional de Monitoreo Hidrológico de Ecosistemas Andinos (iMHEA)
# basada en los desarrollos realizados por:
#     1) FONDO PARA LA PROTECCIÓN DEL AGUA (FONAG), Ecuador.
#         Contacto: info@fonag.org.ec
#     2) EMPRESA PÚBLICA METROPOLITANA DE AGUA POTABLE Y SANEAMIENTO DE QUITO (EPMAPS), Ecuador.
#         Contacto: paramh2o@aguaquito.gob.ec
#
#  IMPORTANTE: Mantener o incluir esta cabecera con la mención de las instituciones creadoras,
#              ya sea en uso total o parcial del código.

from math import isnan

from django.db import connection

from anuarios.anuario import Anuarios
from anuarios.models import Var6Anuarios as HumedadSuelo
from anuarios.models import Var8Anuarios as PresionAtmosferica
from anuarios.models import Var9Anuarios as TemperaturaAgua
from anuarios.models import Var10Anuarios as Caudal
from anuarios.models import Var11Anuarios as NivelAgua
from home.functions import dictfetchall

class FormatoI(Anuarios):
    maximo = 0
    minimo = 0
    promedio = 0

    def consulta(self, estacion, variable, periodo, tipo):
        cursor = connection.cursor()
        tabla = "validacion_var" + str(variable) + "validado"
        sql = "SELECT "
        if tipo == "maixmo":
            sql += "max(valor) as valor, "
        elif tipo == "minimo":
            sql += "min(valor) as valor, "
        else:
            sql += "avg(valor) as valor, "

        sql += "date_part('month',fecha) as mes "
        sql += "FROM " + tabla + " "
        sql += (
            "WHERE estacion_id=" + str(estacion.est_id) + " and valor!='NaN'::numeric "
        )
        sql += "and date_part('year',fecha)=" + str(periodo)
        # if variable == 8:
        # sql += " and maximo!=0 and minimo!=0 and valor!=0 "
        if variable == 6 or variable == 8:
            sql += " and (maximo!=0 or minimo!=0 or valor!=0) "
        sql += "GROUP BY mes ORDER BY mes"
        cursor.close()

    def matriz(self, estacion, variable, periodo):
        pass

def matrizI(estacion, variable, periodo, tipo):
    datos = []
    cursor = connection.cursor()
    # tabla = variable.var_codigo + '.m' + periodo
    if tipo == "validado":
        tabla = "validacion_var" + str(variable) + "validado"
    else:
        tabla = "medicion_var" + str(variable) + "medicion"
    # máximos absolutos y máximos promedio
    sql = "SELECT max(max_abs) as maximo,  max(valor) as valor, "
    # sql = "SELECT max(maximo) as maximo,  max(valor) as valor, "
    sql += "date_part('month',fecha) as mes "
    # sql += "FROM " + tabla + " "
    sql += "FROM mensual_var" + str(variable) + "mensual "
    sql += "WHERE estacion_id=" + str(estacion.est_id) + " and valor!='NaN'::numeric "
    sql += "and date_part('year',fecha)=" + str(periodo) + " "
    sql += (
        "and vacios < (SELECT v.vacios FROM variable_variable v WHERE v.var_id = "
        + str(variable)
        + ") "
    )
    # if variable == 8:
    # sql += " and maximo!=0 and minimo!=0 and valor!=0 "
    if variable == 6 or variable == 8:
        # sql += " and (maximo!=0 or minimo!=0 or valor!=0) "
        sql += " and (max_abs!=0 or min_abs!=0 or valor!=0) "
    sql += "GROUP BY mes ORDER BY mes"
    cursor.execute(sql)
    med_max = dictfetchall(cursor)
    # mínimos absolutos y mínimos promedio
    # sql = "SELECT min(minimo) as minimo,  min(valor) as valor, "
    sql = "SELECT min(min_abs) as minimo,  min(valor) as valor, "
    sql += "date_part('month',fecha) as mes "
    # sql += "FROM " + tabla + " "
    sql += "FROM mensual_var" + str(variable) + "mensual "
    sql += "WHERE estacion_id=" + str(estacion.est_id) + " and valor!='NaN'::numeric "
    sql += "and date_part('year',fecha)=" + str(periodo) + " "
    sql += (
        "and vacios < (SELECT v.vacios FROM variable_variable v WHERE v.var_id = "
        + str(variable)
        + ") "
    )
    # if variable == 8:
    # sql += " and maximo!=0 and minimo!=0 and valor!=0 "
    if variable == 6 or variable == 8:
        # sql += " and (maximo!=0 or minimo!=0 or valor!=0) "
        sql += " and (max_abs!=0 or min_abs!=0 or valor!=0) "
    sql += "GROUP BY mes ORDER BY mes"
    cursor.execute(sql)
    med_min = dictfetchall(cursor)
    # valores promedio mensuales
    sql = "SELECT avg(valor) as valor, date_part('month',fecha) as mes "
    # sql += "FROM " + tabla + " "
    sql += "FROM mensual_var" + str(variable) + "mensual "
    sql += "WHERE estacion_id=" + str(estacion.est_id) + " and valor!='NaN'::numeric "
    sql += "and date_part('year',fecha)=" + str(periodo) + " "
    sql += (
        "and vacios < (SELECT v.vacios FROM variable_variable v WHERE v.var_id = "
        + str(variable)
        + ") "
    )
    # if variable == 8:
    # sql += " and maximo!=0 and minimo!=0 and valor!=0 "
    if variable == 6 and variable == 8:
        # sql += " and (maximo!=0 or minimo!=0 or valor!=0) "
        sql += " and (max_abs!=0 or min_abs!=0 or valor!=0) "
    sql += "GROUP BY mes ORDER BY mes"
    cursor.execute(sql)
    med_avg = dictfetchall(cursor)

    for item_max, item_min, item_avg in zip(med_max, med_min, med_avg):
        mes = int(item_avg.get("mes"))
        if variable == 6:
            obj_hsu = HumedadSuelo()
            obj_hsu.est_id = estacion
            obj_hsu.hsu_periodo = periodo
            obj_hsu.hsu_mes = mes
            obj_hsu.hsu_maximo = get_maximo(item_max)
            obj_hsu.hsu_minimo = get_minimo(item_min)
            obj_hsu.hsu_promedio = get_promedio(item_avg)
            datos.append(obj_hsu)
        elif variable == 8:
            obj_pat = PresionAtmosferica()
            obj_pat.est_id = estacion
            obj_pat.pat_periodo = periodo
            obj_pat.pat_mes = mes
            obj_pat.pat_maximo = get_maximo(item_max)
            obj_pat.pat_minimo = get_minimo(item_min)
            obj_pat.pat_promedio = get_promedio(item_avg)
            datos.append(obj_pat)
        elif variable == 9:
            obj_tag = TemperaturaAgua()
            obj_tag.est_id = estacion
            obj_tag.tag_periodo = periodo
            obj_tag.tag_mes = mes
            obj_tag.tag_maximo = get_maximo(item_max)
            obj_tag.tag_minimo = get_minimo(item_min)
            obj_tag.tag_promedio = get_promedio(item_avg)
            datos.append(obj_tag)
        elif variable == 10:
            obj_cau = Caudal()
            obj_cau.est_id = estacion
            obj_cau.cau_periodo = periodo
            obj_cau.cau_mes = mes
            obj_cau.cau_maximo = get_maximo(item_max)
            obj_cau.cau_minimo = get_minimo(item_min)
            obj_cau.cau_promedio = get_promedio(item_avg)
            datos.append(obj_cau)
        elif variable == 11:
            obj_nag = NivelAgua()
            obj_nag.est_id = estacion
            obj_nag.nag_periodo = periodo
            obj_nag.nag_mes = mes
            obj_nag.nag_maximo = get_maximo(item_max)
            obj_nag.nag_minimo = get_minimo(item_min)
            obj_nag.nag_promedio = get_promedio(item_avg)
            datos.append(obj_nag)
    cursor.close()
    return datos

def get_maximo(item_max):
    try:
        if isnan(item_max.get("maximo")):
            if isnan(item_max.get("valor")):
                return 0
            else:
                return round(item_max.get("valor"), 2)
        return round(item_max.get("maximo"), 2)
    except TypeError:
        if item_max.get("maximo") is None:
            if item_max.get("valor") is None:
                return 0
            else:
                return round(item_max.get("valor"), 2)
        return round(item_max.get("maximo"), 2)

def get_minimo(item_min):
    try:
        if isnan(item_min.get("minimo")):
            if isnan(item_min.get("valor")):
                return 0
            else:
                return round(item_min.get("valor"), 2)
        return round(item_min.get("minimo"), 2)
    except TypeError:
        if item_min.get("minimo") is None:
            if item_min.get("valor") is None:
                return 0
            else:
                return round(item_min.get("valor"), 2)
        return round(item_min.get("minimo"), 2)

def get_promedio(item_avg):
    if item_avg.get("valor") is None:
        return 0
    return round(item_avg.get("valor"), 2)


import pytest
import requests
import jsonmodels
from tests.service.petstore import PetStore

def test_post_pet():
    json_data = {
        "id": 12,
        "category": {
            "id": 0,
            "name": "string"
        },
        "name": "dog",
        "photoUrls": [
            "string"
        ],
        "tags": [
            {
                "id": 0,
                "name": "string"
            }
        ],
        "status": "available"
    }

    response = PetStore().post_pet(json_data)
    print(response.status_code)
    print(response.json())


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Jun  6 20:41:23 2022

@author: marcelae
"""

import pandas as pd

cat = pd.read_csv('/home/marcelae/Desktop/FACOM/1_Proyectos/2_Estaciones/CSV/existencia_estaciones_catalogo.csv',sep=';')
cat
eng = 'sqlite:////home/marcelae/Desktop/FACOM/2_db/presion.db'

q ='''
SELECT DISTINCT CodigoEstacion FROM presion
'''
df = pd.read_sql(q,con=eng)
pa = []
g=False
for i in cat.Codigo:
    g=False
    for j in df.CodigoEstacion:
        if j == i:
            g=True

    if g==True:
        pa.append(1.0)
    else:
        pa.append(0.0)

cat = cat.assign(pa =pa)

contador = 0
for i in range(len(cat)):
    if cat.pa[i]==1.0 or cat.p[i]==1.0 or cat.pa[i] ==1.0:
        contador +=1
print(contador)
ne = []
for i in df.CodigoEstacion:
    g=False
    for j in cat.Codigo:
        if j == i:
            g = True

    if g==False:
        ne.append(i)

ne = pd.DataFrame(ne)

ne.to_csv(r'/home/marcelae/Desktop/FACOM/1_Proyectos/2_Estaciones/CSV/noexisten_presion.csv', index=None, sep=';')
cat.to_csv(r'/home/marcelae/Desktop/FACOM/1_Proyectos/2_Estaciones/CSV/existencia_estaciones_catalogo.csv', index=None, sep=';')

cat.t[0]


from fastapi import APIRouter, Depends, WebSocket

from app.dependencies.auth import AcaPyAuthVerified
from app.services.websocket import handle_websocket, websocket_auth
from shared.log_config import get_logger

logger = get_logger(__name__)
router = APIRouter()

@router.websocket("/ws/topic/{topic}")
async def websocket_endpoint_topic(
    websocket: WebSocket,
    topic: str,
    auth: AcaPyAuthVerified = Depends(websocket_auth),
):
    logger.info("Received websocket request on topic `{}`", topic)
    await handle_websocket(websocket, wallet_id="", topic=topic, auth=auth)

@router.websocket("/ws/{wallet_id}")
async def websocket_endpoint_wallet(
    websocket: WebSocket,
    wallet_id: str,
    auth: AcaPyAuthVerified = Depends(websocket_auth),
):
    logger.info("Received websocket request on wallet id `{}`", wallet_id)
    await handle_websocket(websocket, wallet_id=wallet_id, topic="", auth=auth)

@router.websocket("/ws/{wallet_id}/{topic}")
async def websocket_endpoint_wallet_topic(
    websocket: WebSocket,
    wallet_id: str,
    topic: str,
    auth: AcaPyAuthVerified = Depends(websocket_auth),
):
    logger.info(
        "Received websocket request on wallet id `{}` and topic `{}`", wallet_id, topic
    )
    await handle_websocket(websocket, wallet_id=wallet_id, topic=topic, auth=auth)


"""
Divisors of 42 are : 1, 2, 3, 6, 7, 14, 21, 42. These divisors squared are: 1, 4, 9, 36, 49, 196, 441, 1764.
The sum of the squared divisors is 2500 which is 50 * 50, a square!

Given two integers m, n (1 <= m <= n) we want to find all integers between m and n whose sum of squared divisors is
itself a square. 42 is such a number.

The result will be an array of arrays or of tuples (in C an array of Pair) or a string, each subarray having two
elements, first the number whose squared divisors is a square and then the sum of the squared divisors.

#Examples:

list_squared(1, 250) --> [[1, 1], [42, 2500], [246, 84100]]
list_squared(42, 250) --> [[42, 2500], [246, 84100]]
"""

import math

def list_squared(m, n):
    result = []
    for number in range(m, n):
        sum_square_divisors = sum(
            [i ** 2 if i == number // i else i ** 2 + (number // i) ** 2 for i in range(1, int(math.sqrt(number)) + 1)
             if not number % i])
        root = math.sqrt(sum_square_divisors)
        if root == math.floor(root):
            result.append([number, sum_square_divisors])
    return result


from __future__ import annotations

from contextlib import ExitStack
from dataclasses import dataclass, field
from itertools import chain
from types import TracebackType
from typing import Any

_DEFAULT_ROWS = 7
_DEFAULT_COLS = 100

@dataclass(frozen=True)
class _Show:
    """Context manager which adjusts the display of NDFrames."""

    dp: int | None = None
    rows: int | None = _DEFAULT_ROWS
    columns: int | None = _DEFAULT_COLS
    stack: ExitStack = field(default_factory=ExitStack)

    def __call__(
        self,
        *,
        dp: int | None = None,
        rows: int | None = _DEFAULT_ROWS,
        columns: int | None = _DEFAULT_COLS,
    ) -> _Show:
        return _Show(dp=dp, rows=rows, columns=columns)

    def __enter__(self) -> None:
        self._enter_pandas()
        self._enter_polars()
        _ = self.stack.__enter__()

    def _enter_pandas(self) -> None:
        try:
            from pandas import option_context
        except ModuleNotFoundError:  # pragma: no cover
            pass
        else:
            kwargs: dict[str, Any] = {}
            if self.dp is not None:
                kwargs["display.precision"] = self.dp
            if self.rows is not None:
                kwargs["display.min_rows"] = kwargs["display.max_rows"] = self.rows
            if self.columns is not None:
                kwargs["display.max_columns"] = self.columns
            if len(kwargs) >= 1:
                context = option_context(*chain(*kwargs.items()))
                self.stack.enter_context(context)

    def _enter_polars(self) -> None:
        try:
            from polars import Config
        except ModuleNotFoundError:  # pragma: no cover
            pass
        else:
            kwargs: dict[str, Any] = {}
            if self.dp is not None:
                kwargs["float_precision"] = self.dp
            if self.rows is not None:
                kwargs["tbl_rows"] = self.rows
            if self.columns is not None:
                kwargs["tbl_cols"] = self.columns
            config = Config(**kwargs)
            _ = self.stack.enter_context(config)

    def __exit__(
        self,
        exc_type: type[BaseException] | None,
        exc_val: BaseException | None,
        exc_tb: TracebackType | None,
    ) -> None:
        _ = self.stack.__exit__(exc_type, exc_val, exc_tb)

show = _Show()

__all__ = ["show"]


import sys
sys.path.insert(0, "../../")
from helpers import utils
# --------------------------------------------------
from typing import List
import copy

def read(filename: str) -> List[str]:
    data = []
    with open(filename, 'r') as f:
        for _, row in enumerate(f):
            data.append([int(i) for i in list(row.split()[0])])
    return data

def part_1_sol(data, verbose=False):
    valid_row = []
    num_rows = len(data)
    num_cols = len(data[0])
    for i in range(num_rows):
        for j in range(num_cols):
            if j == 0:
                if data[i][j+1] > data[i][j]:
                    valid_row.append((i,j))
            elif j == num_cols-1:
                if data[i][j-1] > data[i][j]:
                    valid_row.append((i,j))
            else:
                if data[i][j-1] > data[i][j] and data[i][j+1] > data[i][j]:
                    valid_row.append((i,j))
    valid_col = []
    for j in range(num_cols):
        for i in range(num_rows):
            if i == 0:
                if data[i+1][j] > data[i][j]:
                    valid_col.append((i,j))
            elif i == num_rows-1:
                if data[i-1][j] > data[i][j]:
                    valid_col.append((i,j))
            else:
                if data[i-1][j] > data[i][j] and data[i+1][j] > data[i][j]:
                    valid_col.append((i,j))
    return list(set(valid_row) & set(valid_col))

def part_1(data, verbose=False):
    utils.start_timer()
    valid = part_1_sol(data, verbose=False)
    sum_ = 0
    for i,j in valid:
        sum_ += data[i][j]+1
    utils.pprint(1, sum_)
    return valid

def visited_before(i,j,visited):
    for visited_ in visited:
        if visited_[0] == i and visited_[1] == j:
            return True
    return False

def part_2(data,verbose=False):
    utils.start_timer()
    valid = part_1_sol(data, verbose=False)
    num_rows = len(data)
    num_cols = len(data[0])
    basin_sizes = []
    for i,j in valid:
        queue = []
        visited = []
        queue.append((i,j))
        visited.append((i,j))
        basic_size = 0
        while len(queue) > 0:
            i,j = queue.pop(0)
            basic_size += 1
            # right
            j_ = j+1
            while j_ < num_cols:
                if data[i][j_] != 9 and data[i][j_] > data[i][j_-1] and not visited_before(i,j_,visited):
                    queue.append((i,j_))
                    visited.append((i,j_))
                else:
                    break
                j_ += 1
            # left
            j_ = j-1
            while j_ > -1:
                if data[i][j_] != 9 and data[i][j_] > data[i][j_+1] and not visited_before(i,j_,visited):
                    queue.append((i,j_))
                    visited.append((i,j_))
                else:
                    break
                j_ -= 1
            # top
            i_ = i-1
            while i_ > -1:
                if data[i_][j] != 9 and data[i_][j] > data[i_+1][j] and not visited_before(i_,j,visited):
                    queue.append((i_,j))
                    visited.append((i_,j))
                else:
                    break
                i_ -= 1
            # bottom
            i_ = i+1
            while i_ < num_rows:
                if data[i_][j] != 9 and data[i_][j] > data[i_-1][j] and not visited_before(i_,j,visited):
                    queue.append((i_,j))
                    visited.append((i_,j))
                else:
                    break
                i_ += 1
        basin_sizes.append(basic_size)
    top_3 = sorted(basin_sizes,reverse=True)[:3]
    result = top_3[0]*top_3[1]*top_3[2]
    utils.pprint(2, result)

if __name__ == "__main__":
    print("Advent of Code 2021 - Day 8")
    print("-----------------------------")
    print("Example:")
    data = read('ex.txt')
    part_1(copy.deepcopy(data))
    part_2(copy.deepcopy(data))
    print("-----------------------------")
    print("Test:")
    data = read('input.txt')
    part_1(copy.deepcopy(data))
    part_2(copy.deepcopy(data))


import os
import json
import requests
import random
import redis

from functools import lru_cache
from datetime import timedelta
from dotenv import load_dotenv

load_dotenv()

playlist_ids = [
    "37i9dQZF1DXbYM3nMM0oPk",
    "37i9dQZF1DX4WYpdgoIcn6",
    "37i9dQZF1DWXRqgorJj26U",
    "37i9dQZF1DX62Nfha2yFhL",
    "37i9dQZF1DX4VvfRBFClxm",
    "37i9dQZF1DX9qNs32fujYe",
    "37i9dQZF1DWYnwbYQ5HnZU",
    "37i9dQZF1DWXbttAJcbphz",
    "37i9dQZF1DX9tPFwDMOaN1",
    "37i9dQZEVXbNG2KDcFcKOF",
    "37i9dQZF1DWXtlo6ENS92N",
    "37i9dQZF1DXaKIA8E7WcJj",
    "37i9dQZF1DX2pSTOxoPbx9",
    "37i9dQZF1DWWjGdmeTyeJ6",
    "37i9dQZF1DX4pAtJteyweQ",
]

def get_spotify_token() -> str:
    """
    Retrieve a Spotify access token
    """

    redis_client = redis.Redis()
    access_token = redis_client.get("spotify_access_token")

    if not access_token:
        SPOTIFY_CLIENT_ID = os.environ["SPOTIFY_CLIENT_ID"]
        SPOTIFY_CLIENT_SECRET = os.environ["SPOTIFY_CLIENT_SECRET"]

        payload = {
            "Content-Type": "application/x-www-form-urlencoded",
            "grant_type": "client_credentials",
        }

        END_POINT = "https://accounts.spotify.com/api/token"

        response = requests.post(
            END_POINT, auth=(SPOTIFY_CLIENT_ID, SPOTIFY_CLIENT_SECRET), data=payload
        )

        access_token = response.json()["access_token"]

        redis_client.set("spotify_access_token", access_token)
        redis_client.expire("spotify_access_token", timedelta(minutes=30))
    else:
        access_token = access_token.decode("utf-8")

    return access_token

def get_random_song() -> dict:
    ACCESS_TOKEN = get_spotify_token()
    random.shuffle(playlist_ids)
    random_playlist_id = random.choice(playlist_ids)
    redis_client = redis.Redis()

    random_song_list = redis_client.get(f"playlist:{random_playlist_id}")
    if not random_song_list:
        print("cache miss")
        PLAYIST_URL = (
            f"https://api.spotify.com/v1/playlists/{random_playlist_id}/tracks"
        )

        headers = {
            "Accept": "application/json",
            "Content-Type": "application/json",
            "Authorization": f"Bearer {ACCESS_TOKEN}",
        }
        try:
            response = requests.get(PLAYIST_URL, headers=headers)

            if response.status_code == 200:
                random_song_list = response.json()["items"]

                redis_key = f"playlist:{random_playlist_id}"
                redis_client.set(redis_key, json.dumps(random_song_list))
                redis_client.expire(redis_key, timedelta(days=1))
            else:
                raise Exception("Error retrieving results!")
        except:
            raise Exception("Spotify API unreachable!")
    else:
        print("cache hit")
        random_song_list = json.loads(random_song_list)

    random.shuffle(random_song_list)
    random_song = random.choice(random_song_list)["track"]
    # print(random_song)
    song_details = {
        "track_id": random_song["id"],
        "title": random_song["name"],
        "artist": [artist["name"] for artist in random_song["artists"]],
        "cover_img": random_song["album"]["images"][0]["url"],
        "preview_url": random_song["preview_url"] or "",
        "spotify_url": random_song["external_urls"]["spotify"],
    }

    return song_details


"""A small script to convert binary EDID dumps to text files for
Vivado memory initialisation.

"""

import sys
import os

def main():
    """Writes the converted data from EDID dump.

    """
    edid_path = sys.argv[1]
    txt_path = os.path.splitext(edid_path)[0] + '.txt'

    with open(txt_path, 'w') as txt_file:
        with open(edid_path, 'rb') as edid_file:
            byte = edid_file.read(1)
            while byte != '':
                txt_file.write('{0:08b}\n'.format(ord(byte)))
                byte = edid_file.read(1)

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
"""
Created on Fri Nov  1 07:32:23 2019

@author: Dustin
"""
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import copy
from sklearn import preprocessing
from sklearn.metrics import confusion_matrix
import time

timea = time.time()
times = [time.time()]
###############################################################################
#PART ONE: GENERATE DATA BY SIMULATIONS {random numbers = uniform distribution over the interval [-2, +2]}
###############################################################################

#STEP 1

np.random.seed(seed = 229)
#np.random.seed(seed = 313)
#select 16 random numbers Aij with i= 1 2 3 4 and j = 1 2 3 4
A = np.random.uniform(-2,2,size = (4,4))

#select 4 random numbers Bi with i= 1 2 3 4
B = np.random.uniform(-2,2, size = 4)

#select 1 random number c (c/20)
C = np.random.uniform(-2,2)/20

#display the values of these random numbers

#Define the polynomial of degree 2 in the 4 variables x1 x2 x3 x4 as follows

#STEP 2

#select 10, 000 vectors x1 ... x10,000 in R4
X = np.random.uniform(-2,2, size = (10000, 4))

#for each selected xn compute U(n) = Pol(xn) and y(n) = sign[U(n)]
#  Do a for loop because it was faster than Numpy
U = []
for row in range(0,10000):
    u = np.dot(np.dot(X[row,:].T , A), X[row,:]) + np.dot(X[row,:], B) + C/20
    U += [u]
#Pol(x) = ∑i ∑j Aij xi xj + ∑ i Bi xi + c/20

del row, u

u = pd.Series(U)
print('\npositive:', len(u[u>0]))
print('negative:', len(u[u<0]))

#get y into 1 or -1
y = pd.Series(copy.deepcopy(U))
y[y>0] = 1; y[y<0] = -1

#turn into data frame
data = np.insert(X,4,[y,U],axis = 1)
data = pd.DataFrame(data, columns = [0,1,2,3,'y','U'])

#keep only 2500 cases in CL(1) and 2500 cases in CL(-1),
d = pd.concat([data[data['y']>0].sample(n=2500, axis = 0, random_state = 229),
                    data[data['y']<0].sample(n=2500, axis = 0, random_state = 229)]
                    , axis = 0)
d = d.reset_index(drop = True)
print('\nGreater than 0 for y, U:', len(d[d['y']>0]),len(d[d['U']>0]), '\n',
      'Less than 0 for y, U:', len(d[d['y']<0]),len(d[d['U']<0]))
U_prescale = pd.Series(d['U'])
#d.describe()
#                 0            1  ...          y            U
#count  5000.000000  5000.000000  ...  5000.0000  5000.000000
#mean     -0.032034    -0.012180  ...     0.0000    -0.641322
#std       1.148448     1.148988  ...     1.0001     4.693225
#min      -1.999433    -1.998371  ...    -1.0000   -18.668459
#25%      -1.030560    -0.992777  ...    -1.0000    -3.403314
#50%      -0.031247    -0.021481  ...     0.0000    -0.000209
#75%       0.946375     0.984573  ...     1.0000     2.273674
#max       1.998916     1.998722  ...     1.0000    13.594242

#Center and Rescale  this data set of size 5000  so that the standardized
#data set will have mean = 0 and dispersion =1
#d_scale = pd.DataFrame(preprocessing.scale(d), columns = [0,1,2,3,'y','U'])
d_scale = pd.DataFrame(preprocessing.scale(d), columns = [0,1,2,3,'y','U'])
print('\nGreater than 0 for y, U:', len(d_scale[d_scale['y']>0]),len(d_scale[d_scale['U']>0]), '\n',
      'Less than 0 for y, U:', len(d_scale[d_scale['y']<0]),len(d_scale[d_scale['U']<0]))
#do not carry scaled U because the mean isnt centralized, it is a skewed frequency
d_scale['U'] = U_prescale
x = d_scale[[0,1,2,3]]
y = d_scale['y']
#d_scale['U'] = U_prescale
odescription = d.describe()
description = d_scale.describe()
#                  0             1  ...          y            U
#count  5.000000e+03  5.000000e+03  ...  5000.0000  5000.000000
#mean  -1.980624e-16  3.219647e-17  ...     0.0000    -0.641322
#std    1.000100e+00  1.000100e+00  ...     1.0001     4.693225
#min   -1.713265e+00 -1.728817e+00  ...    -1.0000   -18.668459
#25%   -8.695433e-01 -8.535299e-01  ...    -1.0000    -3.403314
#50%    6.850186e-04 -8.095595e-03  ...     0.0000    -0.000209
#75%    8.520248e-01  8.675927e-01  ...     1.0000     2.273674
#max    1.768606e+00  1.750326e+00  ...     1.0000    13.594242

#Then Split each class into a training set and a test set ,
#using the proportions 80% (2000) and 20% (500)
test = pd.concat([d_scale[d_scale['y']>0].sample(n=500, axis = 0, random_state = 229),
                    d_scale[d_scale['y']<0].sample(n=500, axis = 0, random_state = 229)]
                    , axis = 0)
train = d_scale.drop(index = test.index.tolist())
test = test.reset_index(drop = True); train = train.reset_index(drop = True)

#Plot two elements of the data
#first get highest correlation
d_scale[[0,1,2,3,'y']].corr()['y']

fig, ax = plt.subplots(figsize = (12,8))
plt.scatter(x = d_scale[[0]][d_scale['y']>0], y = d_scale[[2]][d_scale['y']>0],
            color = 'blue')
plt.scatter(x = d_scale[[0]][d_scale['y']<0], y = d_scale[[2]][d_scale['y']<0],
            color = 'red')
plt.title('Two dimensional View of Data')
plt.xlabel('Feature 0')
plt.ylabel('Feature 2')

times += [time.time()-timea]
timea = time.time()
###############################################################################
#PART TWO: SVM classification by linear kernel
###############################################################################

from SVM import SVM

mySVM = SVM()

#SVM on Training set, linear kernel
mySVM.set_kernel(kernel = 'linear')
#run the function
n_sv, sv, y_predict, score, conf_int, dist_hp, model = \
    mySVM.run_svm(train.drop(columns = ['y','U']), train['y'], C=5, random_state = 9989)

#Get density histrograms of accuracy on train
mySVM.get_hist(dist_hp, train['y'])

#plot polynomial vs. svm
mySVM.get_scatter(train['U'], dist_hp)

#get confusion matrix for the training set
train_conf, train_per_conf, train_limits, train_confidence = mySVM.get_confusions(train['y'], y_predict, conf_desired = .95)

#SVM on test set
test_predict, test_score, test_conf_int, test_dist_hp  = \
    mySVM.run_svm(test.drop(columns = ['y','U']), test['y'], C=5, random_state = 9989,
                  train = 'no', model = model)

#plot polynomial vs. svm
mySVM.get_scatter(test['U'], test_dist_hp)

#get confusion matrix on test set and limits
test_conf, test_per_conf, test_limits, test_confidence = mySVM.get_confusions(test['y'], test_predict, conf_desired = .95)

#look at these two values to verify what column means what in confusion matrix
print('\nValue Counts in response:\n', pd.Series(test_predict).value_counts())
print('\nSum of columns in conf matrix:',test_conf.sum(axis = 0))

###############################################################################
#PART THREE: Optimize the parameter cost
###############################################################################

#Tune the cost
mySVM = SVM()

#SVM on Training set, linear kernel
mySVM.set_kernel(kernel = 'linear')

#run through the different tunes: #1
C_scores, C_n_sv = mySVM.tune_svm(train.drop(columns = ['y','U']), \
                                 train['y'], test.drop(columns = ['y','U']), test['y'], \
                                 C=5, random_state = 9989, tune = 'C', \
                                 params = [.00001, .001, 1, 10, 30, 50])

#run through the different tunes
C_scores, C_n_sv = mySVM.tune_svm(train.drop(columns = ['y','U']), \
                                 train['y'], test.drop(columns = ['y','U']), test['y'], \
                                 C=5, random_state = 9989, tune = 'C', \
                                 params = [.001, .01, .1, 1, 1.5, 2])

##print the different scores for each c
#print(max(C_scores.values()), C_scores.keys()[max(C_scores)]))
best_c = .1

mySVM = SVM()

#SVM on Training set, linear kernel
mySVM.set_kernel(kernel = 'linear')
#run the function
n_sv, sv, y_predict, score, conf_int, dist_hp, model = \
            mySVM.run_svm(train.drop(columns = ['y','U']), train['y'], C=best_c,
                          random_state = 9989)

#Get density histrograms of accuracy
mySVM.get_hist(dist_hp, train['y'])

#get confusion matrix for the training set
train_conf, train_per_conf, train_limits, train_confidence = mySVM.get_confusions(train['y'], y_predict, conf_desired = .95)

#plot polynomial vs. svm
mySVM.get_scatter(train['U'], dist_hp)

#Do test run
#SVM on test set
test_predict, test_score, test_conf_int, test_dist_hp  = \
    mySVM.run_svm(test.drop(columns = ['y','U']), test['y'], C=5, random_state = 9989,
                  train = 'no', model = model)

#plot polynomial vs. svm
mySVM.get_scatter(test['U'], test_dist_hp)

#get confusion matrix on test set and limits
test_conf, test_per_conf, test_limits, test_confidence = mySVM.get_confusions(test['y'], test_predict, conf_desired = .95)

times += [time.time()-timea]
timea = time.time()
###############################################################################
#PART FOUR:  SVM classification by radial kernel
###############################################################################

radSVM = SVM()

#SVM on Training set, linear kernel
radSVM.set_kernel(kernel = 'rbf', gamma = 1)

#run the function
n_svr, svr, yr_predict, score_r, conf_r, dist_hp_r, model_r = \
    radSVM.run_svm(train.drop(columns = ['y', 'U']), train['y'], C=best_c,
                  random_state = 9989, train = 'yes')

#Get density histrograms of accuracy
radSVM.get_hist(dist_hp_r, train['y'])

#get confusion matrix for the training set
train_conf_r, train_per_conf_r, train_limits, train_confidencer = radSVM.get_confusions(train['y'], yr_predict, conf_desired = .95)

#plot polynomial vs. svm
radSVM.get_scatter(train['U'], dist_hp_r)

#SVM on test set
test_predict_r, test_score_r, test_conf_r, test_dist_hp_r = radSVM.run_svm(test.drop(columns = ['y', 'U']),
                            test['y'], C=best_c, random_state = 9989, train = 'no',
                            model = model_r)

#get confusion matrix on test set
test_conf_r, test_per_conf_r, test_limitsr, test_confidencer = radSVM.get_confusions(test['y'], test_predict_r, conf_desired = .95)

#plot polynomial vs. svm
radSVM.get_scatter(test['U'], test_dist_hp_r)

###############################################################################
#PART FIVE:  Optimize cost and gamma
###############################################################################

#Tune the radial gamma and cost
tune_radSVM = SVM()

#SVM on Training set, linear kernel
tune_radSVM.set_kernel(kernel = 'rbf')

#run through the different tunes - first iteration
gamma_scores, n_sv_gamma = tune_radSVM.tune_svm(train.drop(columns = ['y', 'U']),
                                    train['y'], test.drop(columns = ['y', 'U']), test['y'],
                                    C=best_c, random_state = 9989, tune = 'gamma',
                                    params = [.00001, .001, .01, .1, .5, 1])

#run through the different tunes - 2nd iteration
gamma_scores, n_sv_gamma = tune_radSVM.tune_svm(train.drop(columns = ['y', 'U']),
                                    train['y'], test.drop(columns = ['y','U']), test['y'],
                                    C=best_c, random_state = 9989, tune = 'gamma',
                                    params = [.05, .075, .1, .25, .45, .65, .75])

"""
looks like gamma of .1 is the best for both Test and Train Set \
Use gamma = 0.45 and iterate through the cost
"""

tune_radSVM = SVM()

#SVM on Training set, linear kernel
tune_radSVM.set_kernel(kernel = 'rbf', gamma = .45)

C_scores, n_sv_gamma = tune_radSVM.tune_svm(train.drop(columns = ['y', 'U']),
                                    train['y'], test.drop(columns = ['y', 'U']), test['y'],
                                    C=5, random_state = 9989, tune = 'C',
                                    params = [.00001, .01, 10, 35, 70, 100])

C_scores, n_sv_gamma = tune_radSVM.tune_svm(train.drop(columns = ['y', 'U']),
                                    train['y'], test.drop(columns = ['y', 'U']), test['y'],
                                    C=5, random_state = 9989, tune = 'C',
                                    params = [.00001, .001, 1, 10, 30, 50])

C_scores, n_sv_gamma = tune_radSVM.tune_svm(train.drop(columns = ['y', 'U']),
                                    train['y'], test.drop(columns = ['y', 'U']), test['y'],
                                    C=5, random_state = 9989, tune = 'C',
                                    params = [15, 20, 25, 30, 35, 40, 45])

"""
Looks like cost of 35 was the highest and closest match between test and train \
We will re-iterate through gamma using a cost of 30, with focus around previous \
gamma.
"""

tune_radSVM = SVM()

#SVM on Training set
tune_radSVM.set_kernel(kernel = 'rbf')

#run through the different tunes with best cost to make sure didnt change
gamma_scores, n_sv_gamma = tune_radSVM.tune_svm(train.drop(columns = ['y', 'U']),
                                    train['y'], test.drop(columns = ['y', 'U']), test['y'],
                                    C=35, random_state = 9989, tune = 'gamma',
                                    params = [.05, .075, .1, .25, .45, .65, .75])

#SVM on Training set, linear kernel
tune_radSVM.set_kernel(kernel = 'rbf', gamma = .1)
C_scores, n_sv_gamma = tune_radSVM.tune_svm(train.drop(columns = ['y', 'U']),
                                    train['y'], test.drop(columns = ['y', 'U']), test['y'],
                                    C=5, random_state = 9989, tune = 'C',
                                    params = [8, 15, 22, 30, 37, 45, 50])

"""
Run: gamma = 0.1, C = 35
"""

#run tuned radial SVM
C=35; gamma = 0.10

tune_radSVM = SVM()

#SVM on Training set
tune_radSVM.set_kernel(kernel = 'rbf', gamma = gamma)

n_svrt, svrt, yrt_predict, score_rt, conf_int_rt, dist_hp_rt, model_rt = \
    tune_radSVM.run_svm(train.drop(columns = ['y', 'U']), train['y'], C=C,
                  random_state = 9989)

#Get density histrograms of accuracy
tune_radSVM.get_hist(dist_hp_rt, train['y'])

#get confusion matrix for the training set
train_conf_rt, train_per_conf_rt, limits_rt, train_confidence_rt = tune_radSVM.get_confusions(train['y'], yrt_predict)

#plot polynomial vs. svm
tune_radSVM.get_scatter(train['U'], dist_hp_rt)

#SVM on test set
test_predict_rt, test_score_rt, test_conf_int_rt, test_dist_hp_rt = tune_radSVM.run_svm(test.drop(columns = ['y', 'U']),
                            test['y'], C=C, random_state = 9989, train = 'no',
                            model = model_rt)

#get confusion matrix on test set
test_conf_rt, test_per_conf_rt, test_limits_rt, test_confidence_rt = tune_radSVM.get_confusions(test['y'], test_predict_rt)

#plot polynomial vs. svm
tune_radSVM.get_scatter(test['U'], test_dist_hp_rt)

times += [time.time()-timea]
timea = time.time()
###############################################################################
#PART SIX:  Run Polynomial with degree = 4, optimize Coeff and Cost
###############################################################################

"""
Since we know degree is 4, lets get an initial look with default parameters
"""

poly_SVM = SVM()

#set polynomial
poly_SVM.set_kernel('poly', degree = 4)

#run poly SVM
n_svp, svp, yp_predict, score_p, conf_p, train_dist_hp_p, model_p = \
    poly_SVM.run_svm(train.drop(columns = ['y', 'U']), train['y'], C = 1,
                  random_state = 9989, train = 'yes')

#Get density histrograms of accuracy
poly_SVM.get_hist(train_dist_hp_p, train['y'])

#get confusion matrix for the training set
train_conf_p, train_per_conf_p, train_limits_p, train_confidence_p = poly_SVM.get_confusions(train['y'], yp_predict)

#plot polynomial vs. svm
poly_SVM.get_scatter(train['U'], train_dist_hp_p)

#SVM on test set
test_predict_p, test_score_p, test_conf_p, test_dist_hp_p = poly_SVM.run_svm(test.drop(columns = ['y', 'U']),
                            test['y'], C=1, random_state = 9989, train = 'no',
                            model = model_p)

#get confusion matrix on test set
test_conf_p, test_per_conf_p, test_limits_p, test_confidence_p = poly_SVM.get_confusions(test['y'], test_predict_p)

poly_SVM.get_scatter(test['U'], test_dist_hp_p)

"""
Tune the parameters Coeff and Cost. Leave Degree = 4
"""
tpoly_SVM = SVM()

#set polynomial
tpoly_SVM.set_kernel('poly', degree = 4)

pt_scores, n_sv_pt = tpoly_SVM.tune_svm(train.drop(columns = ['y', 'U']),
                                    train['y'], test.drop(columns = ['y', 'U']), test['y'],
                                    C=1, random_state = 9989, tune = 'coef0',
                                    params = [.001, .1, 10, 30, 60, 100, 200])

tpoly_SVM = SVM()

#set polynomial
tpoly_SVM.set_kernel('poly', degree = 4)

pt_scores, n_sv_pt = tpoly_SVM.tune_svm(train.drop(columns = ['y', 'U']),
                                    train['y'], test.drop(columns = ['y', 'U']), test['y'],
                                    C=1, random_state = 9989, tune = 'coef0',
                                    params = [15, 20, 25, 30, 35, 40, 45])

"""
Coef of 30 and 50 were equally best on Test. 30 was 100% on Train, 50 was 99.975%.\
Use Coef 30, change params for Cost now.
"""
tpoly_SVM = SVM()

#set polynomial
tpoly_SVM.set_kernel('poly', degree = 4, coef0 = 30)

pt_scores, n_sv_pt = tpoly_SVM.tune_svm(train.drop(columns = ['y', 'U']),
                                    train['y'], test.drop(columns = ['y', 'U']), test['y'],
                                    C=1, random_state = 9989, tune = 'C',
                                    params = [.001, .1, 25, 50, 75, 100, 125])

tpoly_SVM = SVM()

#set polynomial
tpoly_SVM.set_kernel('poly', degree = 4, coef0 = 30)

pt_scores, n_sv_pt = tpoly_SVM.tune_svm(train.drop(columns = ['y', 'U']),
                                    train['y'], test.drop(columns = ['y', 'U']), test['y'],
                                    C=1, random_state = 9989, tune = 'C',
                                    params = [.1, 1, 5, 10, 15, 20, 25])

"""
Looks like best cost is 1, though they are so close it is not really different.\
Use Cost = 1, coef0 = 30, degree = 4 and run_SVM
"""
tpoly_SVM = SVM()

#set polynomial
tpoly_SVM.set_kernel('poly', degree = 4, coef0 = 30)

#run poly SVM
n_svpt, svpt, ypt_predict, score_pt, train_conf_pt, dist_hp_pt, model_pt = \
    tpoly_SVM.run_svm(train.drop(columns = ['y', 'U']), train['y'], C = 1,
                  random_state = 9989, train = 'yes')

#Get density histrograms of accuracy
tpoly_SVM.get_hist(dist_hp_pt, train['y'])

#get confusion matrix for the training set
train_conf_pt, train_per_conf_pt, train_intervals_pt, train_confidence_pt = tpoly_SVM.get_confusions(train['y'], ypt_predict)

#plot polynomial vs. svm
tpoly_SVM.get_scatter(train['U'], dist_hp_pt)

#SVM on test set
test_predict_pt, test_score_pt, test_conf_pt, test_dist_hp_pt = tpoly_SVM.run_svm(test.drop(columns = ['y', 'U']),
                            test['y'], C=1, random_state = 9989, train = 'no',
                            model = model_pt)

#get confusion matrix on test set
test_conf_pt, test_per_conf_pt, test_intervals_pt, test_confidence_pt = poly_SVM.get_confusions(test['y'], test_predict_pt)

#plot polynomial vs. svm
tpoly_SVM.get_scatter(test['U'], test_dist_hp_pt)

times += [time.time()-timea]
timea = time.time()
###############################################################################
#Time
###############################################################################

print(times)

###############################################################################
#NUMPY WAS SLOWER
########################################
#y = copy.deepcopy(U)
#y[y>0] = 1; y[y<0] = -1
#
#time_a = time.time()
#trial = np.dot(X,np.dot(X,A).T).diagonal()
#time_b = time.time()
#check_trial = []
#for row in range(0,10000):
#    s = 0
#    for i in range(0,4):
#        for j in range(0,4):
#            calc = A[i,j]*X[row,i]*X[row,j]
#            s += calc
#    check_trial.append(s)
#
#check_trial = np.array(check_trial)
#time_c = time.time()
#
#print(time_b-time_a, time_c-time_b)
#sum(trial-check_trial)

#from sklearn import svm
#
##set up the train
#lin = svm.SVC(C = 5, kernel = 'linear', probability = True, random_state = 9989)
#
##train with data
#lin.fit(train.drop(columns = 'y'), train['y'])
#
##Predict
#train_predict = lin.predict(train.drop(columns = 'y'))
#
##Score
#score = lin.score(train.drop(columns = 'y'), train['y'])
#
##Support Vectors
#s_vectors = lin.support_vectors_
#n_s = lin.n_support_
#
##probabilities (A,B)
#probabilites = {
#        'A' : lin.probA_,
#        'B' : lin.probB_
#        }
#
##decision function (function values are proportional to the distance of the samples X to the separating hyperplane)
#dist_hp = lin.decision_function(train.drop(columns = 'y'))
#
#pair y actual and distances
#hist = pd.concat([
#        pd.Series(dist_hp2, name = 'Distance to HP'),
#        pd.Series(train['y'])
#        ], axis = 1)

##split into group one and group 2
#hist_1 = hist[hist['y']>0]
#hist_2 = hist[hist['y']<0]
#
##plot histograms
#fig1, ax1 = plt.subplots(1,2, figsize = (12,8))
#hist_1['Distance to HP'].plot.hist(subplots = True, sharey = True, sharex = False, ax = ax1[0], bins = 15, rwidth = .95, title = 'Density Plot')
#hist_2['Distance to HP'].plot.hist(subplots = True, sharey = True, sharex = False, ax = ax1[1], bins = 15, rwidth = .95)
#
##Another way to plot
#fig1, ax1 = plt.subplots(figsize = (12,8))
#ax1.hist.hist(column = 'Distance to HP', by = 'y', layout = (2,1), sharex = True, sharey = True, bins = 15, rwidth = .95)
#
##confusion matrix
#confusion = confusion_matrix(train_predict,train['y'] )
#
##percent confusion matrix
#totals = np.sum(confusion, axis = 1)
#perc_confusion = confusion/totals
#
##print confusion matrix
#print(perc_confusion)


with open('src/conversaciones.txt', 'r') as file1:
    lines = file1.readlines()

opositivos = ['SIN EMBARGO', 'NO OBSTANTE', 'AHORA BIEN', 'AUN ASI']

causativos = ['POR TANTO', 'DADO QUE', 'POR CONSIGUIENTE',
              'ASI PUES','POR ENDE']

for line in lines:
    opositivos_c = 0
    causativos_c = 0
    line = line.strip("\n")
    line = line.strip(',')
    line = line.upper()
    for o in opositivos:
        opositivos_c += line.count(o)
    for c in causativos:
        causativos_c += line.count(c)
    print('Opositivos {} Causativos {}'.format(opositivos_c, causativos_c))


import os
import urllib.request
from urllib.error import HTTPError

if __name__ == '__main__':
    # Create Output folder if not found in system
    if not os.path.exists('CustomSprites'):
        os.mkdir('CustomSprites')

    # Setting Values
    SAVE_DIR = 'CustomSprites/'
    EXT_GEN1 = ['mf', 'md', 'mo', 'fo', 'uk']
    AMT_GEN1_START = 0
    AMT_GEN1_END = 152
    AMT_GEN1_TOTAL = 151

    # Test String
    # https://projectpokemon.org/images/sprites-models/homeimg/poke_capture_0001_000_mf_n_00000000_f_n.png

    # Generation 1 Code
    png_url = ['https://projectpokemon.org/images/sprites-models/homeimg/poke_capture_', '_000_', '_n_00000000_f_n.png']

    dex_num_4 = [str(i).zfill(4) for i in range(AMT_GEN1_START+1, AMT_GEN1_END)]
    dex_num_3 = [str(i).zfill(3) + '.png' for i in range(AMT_GEN1_START+1, AMT_GEN1_END)]

    def base_request(filename, png, extension, dex_num):
        png_url_call = png[0] + dex_num + png[1] + extension + png[2]
        print('Calling: ', png_url_call)
        urllib.request.urlretrieve(png_url_call, filename)

    for i in range(len(dex_num_3)):
        filename_save = SAVE_DIR + dex_num_3[i]
        dex_number = dex_num_4[i]

        # Skip current iteration if file is found in SAVE_DIR
        if os.path.exists(filename_save):
            print('File found for ', dex_num_4[i], '. Skipping.')
            continue

        print(dex_number, ': Starting Function Calls.')
        # Looping through base_request function with different extensions
        for ext in EXT_GEN1:
            try:
                base_request(filename_save, png_url, ext, dex_number)
                print('Function run successfully.')
                break  # Moves to the next file
            except HTTPError:
                continue  # Moves to the next function is Exception is found


"""
Boilerplate that combines Hydra conf and PytorchLightning for supervised training.
"""
import os
import json
import torch
import random
import logging
import pathlib
import pytorch_lightning as pl
from typing import List
from omegaconf import OmegaConf
from torch.utils.data import DataLoader
from .callbacks import WandbTableCallback, S3Callback, GitCallback
from .metrics import Accuracy
from .model import BaseModel
from pytorch_lightning import callbacks as C
from pytorch_lightning.loggers import WandbLogger, CSVLogger

class SupervisedModel(BaseModel):
    """
    Supervised learning Pytorch lightning module.
    You should overload this model as you see fit.
    """

    def get_callbacks(self):
        """
        Returns a list of `pytorch_lightning.callbacks.Callback`s to use for training.
        """
        callbacks = []
        if self.hparams.git.enable:
            callbacks.append(GitCallback())
        if self.hparams.wandb.enable:
            callbacks.append(WandbTableCallback())
        if self.hparams.s3.enable:
            callbacks.append(S3Callback(self.hparams.s3))
        return callbacks

    def featurize(self, batch: list):
        """
        Converts a batch of examples to features.
        By default this returns the batch as is.

        Alternatively you may want to set `collate_fn: "ignore"` in your config and use `featurize` to convert raw examples into features.
        """
        return batch

    def compute_metrics(self, pred: list, gold: list, batch: list) -> dict:
        """
        Computes metrics between predictions and ground truths.
        """
        m = Accuracy()
        return m(pred, gold)

    def compute_loss(self, out, feat, batch) -> torch.Tensor:
        """
        Computes loss from inputs and model output.
        You must implement this function.

        Args:
            out: model output from `wrangl.learn.supervised.Model.forward`.
            feat: featurize output from `wrangl.learn.supervised.Model.featurize`.
            batch: Dataset iterator output.

        """
        raise NotImplementedError()

    def extract_context(self, out, feat, batch) -> List:
        """
        Extract example context from inputs and model output.
        You must implement this function.

        Args:
            out: model output from `wrangl.learn.supervised.Model.forward`.
            feat: featurize output from `wrangl.learn.supervised.Model.featurize`.
            batch: Dataset iterator output.

        """

        raise NotImplementedError()

    def extract_pred(self, out, feat, batch) -> List:
        """
        Extract model predictions from inputs and model output.
        You must implement this function.

        Args:
            out: model output from `wrangl.learn.supervised.Model.forward`.
            feat: featurize output from `wrangl.learn.supervised.Model.featurize`.
            batch: Dataset iterator output.
        """

        raise NotImplementedError()

    def extract_gold(self, feat, batch) -> List:
        """
        Extract ground truths from inputs and model output.
        You must implement this function.

        Args:
            out: model output from `wrangl.learn.supervised.Model.forward`.
            feat: featurize output from `wrangl.learn.supervised.Model.featurize`.
            batch: Dataset iterator output.

        """

        raise NotImplementedError()

    def forward(self, feat, batch):
        """
        Computes model output from inputs.
        You must implement this function.

        Args:
            feat: featurize output from `wrangl.learn.supervised.Model.featurize`.
            batch: Dataset iterator output.

        """
        raise NotImplementedError()

    def infer(self, feat, batch):
        """
        Computes model output from inputs for inference.
        Defaults to `wrangl.learn.supervised.Model.forward`.

        Args:
            feat: featurize output from `wrangl.learn.supervised.Model.featurize`.
            batch: Dataset iterator output.

        """
        return self.forward(feat, batch)

    def infer_batch_size(self, batch):
        """
        Infers batch size from batch of raw examples.
        """
        if isinstance(batch, list):
            return len(batch)
        if isinstance(batch, dict):
            first_key = list(batch.keys())[0]
            return len(batch[first_key])
        return None

    def training_step(self, batch, batch_id):
        """
        One training step in PytorchLightning.
        You should not have to modify this.
        For more information, see `pytorch_lightning.training_step`.
        """
        feat = self.featurize(batch)
        out = self.forward(feat, batch)
        loss = self.compute_loss(out, feat, batch)
        self.log('loss', loss, batch_size=self.infer_batch_size(batch))
        perplexity = torch.exp(loss)
        self.log('ppl', perplexity, batch_size=self.infer_batch_size(batch))
        return loss

    def test_step(self, batch, batch_id, split='test'):
        """
        One test step in PytorchLightning.
        You should not have to modify this.
        For more information, see `pytorch_lightning.validation_step`.
        """
        return self.validation_step(batch, batch_id, split='test')

    def predict_step(self, batch, batch_id):

        """
        One prediction step in PytorchLightning.
        You should not have to modify this.
        For more information, see `pytorch_lightning.prediction_step`.
        """
        feat = self.featurize(batch)
        out = self.infer(feat, batch)
        pred = self.extract_pred(out, feat, batch)
        return pred

    def validation_step(self, batch, batch_id, split='val'):
        """
        One validation step in PytorchLightning.
        You should not have to modify this.
        For more information, see `pytorch_lightning.validation_step`.

        A core difference here is that Wrangl will
        1. compute metrics automatically
        2. extract example contexts, predictions, and ground truths
        3. save predictions to disk (and upload to S3, Wandb etc.) for debugging
        """
        feat = self.featurize(batch)
        context = self.extract_context(feat, batch)
        gold = self.extract_gold(feat, batch)

        out = self.infer(feat, batch)
        pred = self.extract_pred(out, feat, batch)

        metrics = self.compute_metrics(pred, gold, batch)
        for k, v in metrics.items():
            self.log('{}_{}'.format(split, k), v, batch_size=self.infer_batch_size(batch))

        # for logging to wandb
        sample_ids = list(range(len(context)))
        random.shuffle(sample_ids)
        sample_ids = sample_ids[:self.hparams.val_sample_size]

        random.shuffle(self.pred_samples)
        for i in sample_ids:
            self.pred_samples.insert(0, (context[i], pred[i], gold[i]))
        self.pred_samples = self.pred_samples[:self.hparams.val_sample_size]

        with open('pred_samples.json', 'wt') as f:
            data = [dict(context=repr(context), gen=repr(gen), gold=repr(gold)) for context, gen, gold in self.pred_samples]
            json.dump(data, f, indent=2)

    @classmethod
    def run_train_test(cls, cfg: OmegaConf, train_dataset: torch.utils.data.DataLoader, eval_dataset: torch.utils.data.DataLoader, model_kwargs=None):
        """
        Creates a model according to config and trains/evaluates it.

        Args:
            cfg: config to use.
            train_dataset: dataset for training.
            eval_dataset: dataset for validation.
            model_kwargs: key ward arguments for model constructor.

        This will additionally set up according to your config file
            - manual seeds
            - early stopping
            - checkpoint saving
            - gradient clipping
            - callbacks (including an always on CSV logger)
        """
        model_kwargs = model_kwargs or {}
        pl.utilities.seed.seed_everything(seed=cfg.seed, workers=True)
        dout = os.getcwd()

        logger = logging.getLogger(name='{}:train_test'.format(cls.__name__))
        logger.info('Logging to {}'.format(dout))

        checkpoint = C.ModelCheckpoint(
            dirpath=dout,
            monitor=cfg.early_stopping.monitor,
            mode=cfg.early_stopping.mode,
            filename='{epoch:02d}-{' + cfg.early_stopping.monitor + ':.4f}',
            every_n_epochs=1,
            save_top_k=3,
            save_last=True,
            verbose=True,
        )

        train_logger = [CSVLogger(save_dir=dout, name='logs', flush_logs_every_n_steps=cfg.flush_logs_every_n_steps)]
        if cfg.wandb.enable:
            train_logger.append(WandbLogger(project=cfg.wandb.project, name=cfg.wandb.name, entity=cfg.wandb.entity, save_dir=cfg.wandb.dir))

        fconfig = os.path.join(dout, 'config.yaml')
        OmegaConf.save(config=cfg, f=fconfig)

        model = cls(cfg, **model_kwargs)

        logger.info('Loading data')
        if cfg.collate_fn == 'auto':
            collate_fn = None
        elif cfg.collate_fn == 'ignore':
            def collate_fn(batch):
                return batch
        else:
            collate_fn = cls.collate_fn
        train_loader = DataLoader(train_dataset, batch_size=cfg.batch_size, collate_fn=collate_fn)
        eval_loader = DataLoader(eval_dataset, batch_size=cfg.val_batch_size, collate_fn=collate_fn)

        trainer = pl.Trainer(
            precision=cfg.precision,
            strategy=cfg.strategy,
            gpus=cfg.gpus,
            auto_lr_find=False,
            auto_scale_batch_size=False,
            auto_select_gpus=cfg.gpus > 0,
            benchmark=True,
            default_root_dir=dout,
            gradient_clip_val=cfg.grad_clip_norm,
            gradient_clip_algorithm='norm',
            log_every_n_steps=cfg.log_every_n_steps,
            val_check_interval=1 if cfg.debug else cfg.val_check_interval,
            limit_val_batches=cfg.limit_val_batches,
            max_steps=cfg.max_steps,
            logger=train_logger,
            callbacks=[checkpoint] + model.get_callbacks(),
        )
        if not cfg.test_only:
            ckpt_path = None
            if cfg.autoresume:
                ckpt_path = os.path.join(dout, 'last.ckpt')
                if not os.path.isfile(ckpt_path):
                    ckpt_path = None
            trainer.fit(model, train_loader, eval_loader, ckpt_path=ckpt_path)

        model = cls.load_from_checkpoint(os.path.join(dout, 'last.ckpt'))
        result = trainer.test(model, eval_loader, verbose=True, test=False)
        with open(os.path.join(dout, 'eval_pred.json'), 'wt') as f:
            json.dump(result, f, indent=2)
        result = trainer.test(model, eval_loader, verbose=True, test=True)
        with open(os.path.join(dout, 'eval_test.json'), 'wt') as f:
            json.dump(result, f, indent=2)
        logger.info('Finished!')

    @classmethod
    def run_inference(cls, cfg: OmegaConf, fcheckpoint: str, eval_dataset: torch.utils.data.DataLoader, model_kwargs=None, test=False):
        """
        Loads a model according to config and runs inference.

        Args:
            cfg: config to use.
            fcheckpoint: model checkpoint to load.
            eval_dataset: dataset for validation.
            model_kwargs: key ward arguments for model constructor.
            test: whether to evaluate metrics

        Returns:
            if `test==True` then returns a dictionary of scores, otherwise returns a list of model predictions.
        """
        model_kwargs = model_kwargs or {}
        model = cls.load_from_checkpoint(fcheckpoint, **model_kwargs)

        if cfg.collate_fn == 'auto':
            collate_fn = None
        elif cfg.collate_fn == 'ignore':
            def collate_fn(batch):
                return batch
        else:
            collate_fn = cls.collate_fn
        eval_loader = DataLoader(eval_dataset, batch_size=cfg.batch_size, collate_fn=collate_fn)

        trainer = pl.Trainer(
            precision=cfg.precision,
            strategy=cfg.strategy,
            gpus=cfg.gpus,
            auto_lr_find=False,
            auto_scale_batch_size=False,
            auto_select_gpus=cfg.gpus > 0,
            benchmark=True,
        )
        if test:
            result = trainer.test(model, eval_loader, verbose=True)
        else:
            result = []
            for b in trainer.predict(model, eval_loader):
                result.extend(b)
        return result

    @classmethod
    def load_experiment(cls, dexp):
        """
        Loads experiment from disk.
        """
        import pandas as pd
        dexp = pathlib.Path(dexp)
        config = OmegaConf.to_container(OmegaConf.load(dexp.joinpath('config.yaml')), resolve=True)

        # make sure collection exists for project
        df = []
        for flog in dexp.joinpath('logs').glob('*/*.csv'):
            df.append(pd.read_csv(flog))
        if df:
            logs = pd.concat(df).to_dict()
        else:
            logs = {}
        return config, logs


# 정규식 컴파일 옵션
# DOTALL(S) IGNORECASE(I) MULTILINE(M) VERBOSE(X)

import re

p = re.compile('a.b')
m = p.match('a\nb')  # . 문자와 매치되지 않음
print(m)

p = re.compile('a.b', re.DOTALL)  # DOTALL을 통해 \n 문자도 가능하도록 파라미터 추가로 설정
m = p.match('a\nb')
print(m)

p = re.compile('[a-z]', re.I)  # IGNORECASE 를 이용해 대소문자 구분을 하지 않는다
p.match('python')
p.match('Python')
p.match('PYTHON')

p = re.compile("^python\s\w+")

data = """
    life is too short
    python two
    you need python
    python three
"""

print(p.findall(data))

p = re.compile("^python\s\w+", re.MULTILINE)

print(p.findall(data))

# VERBOSE, X
charref = re.compile(r'&[#](0[0-7]+|[0-9]+|x[0-9a-fA-F]);')

# \ 백슬새시는 \section의 경우 \\section으로 해결 가능
# \\\\ RawString 규칙 적용시 \\section 과 같이 표현가능
p = re.compile(r'\\section')


__docformat__ = "restructuredtext en"

from warnings import warn
warn('logservice module is deprecated and will disappear in a near release. \
use logging module instead.',
     DeprecationWarning, stacklevel=2)

from logilab.common.logger import make_logger, LOG_ERR, LOG_WARN, LOG_NOTICE, \
     LOG_INFO, LOG_CRIT, LOG_DEBUG

def init_log(treshold, method='eprint', sid='common-log-service',
             logger=None, output=None):
    """init the logging system and and log methods to builtins"""
    if logger is None:
        logger = make_logger(method, treshold, sid, output=output)
    # add log functions and constants to builtins
    __builtins__.update({'log': logger.log,
                         'log_traceback' : logger.log_traceback,
                         'LOG_CRIT':   LOG_CRIT,
                         'LOG_ERR':    LOG_ERR,
                         'LOG_WARN':   LOG_WARN,
                         'LOG_NOTICE': LOG_NOTICE,
                         'LOG_INFO' :  LOG_INFO,
                         'LOG_DEBUG':  LOG_DEBUG,
                         })

init_log(LOG_ERR)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Sum of Even Numbers
░░░░░░░░░░░░
Given a number N, print sum of all even numbers from 1 to N.
"""

n = int(input())
sum = 0

for i in range(n + 1):
    if(i % 2 == 0):
        sum += i

print(sum)


import pandas as pd
from sklearn import linear_model

df = pd.read_csv("/Users/qozeemodeniran/Projects/python/w3schools/ml/data.csv")

X = df[['Weight', 'Volume']]
y = df['CO2']

regr = linear_model.LinearRegression()
regr.fit(X, y)

print(regr.coef_)


# -*- coding: utf-8 -*-
# Description: logcount python.d module for netdata
# Author: nodiscc (nodiscc@gmail.com)
# SPDX-License-Identifier: GPL-3.0-or-later

import os
import re

from bases.FrameworkServices.SimpleService import SimpleService

priority = 90000
update_every = 10

ORDER = [
    'messages'
]

CHARTS = {
    'messages': {
        'options': [None, 'Messages by level', 'messages', 'messages', 'logcount.messages', 'stacked'],
        'lines': [
            ['info', None, 'absolute'],
            ['error', None, 'absolute'],
            ['warning', None, 'absolute']
        ]
    }
}

RE_error = re.compile(r'error,.*')
RE_warning = re.compile(r'warning,.*')
RE_info = re.compile(r'info,.*')

class Service(SimpleService):
    def __init__(self, configuration=None, name=None):
        SimpleService.__init__(self, configuration=configuration, name=name)
        self.order = ORDER
        self.definitions = CHARTS

        self.data = dict()
        self.path = '/var/cache/logcount'
        self.modtime = ''
        self.data['error'] = 0
        self.data['warning'] = 0
        self.data['info'] = 0

    def check(self):
        return True

    def get_data(self):
        if not is_readable(self.path) or is_empty(self.path):
            self.debug("{0} is unreadable or empty".format(self.path))
            self.data['error'] = 1
            self.data['warning'] = 0
            self.data['info'] = 0
            return self.data

        try:
            if not self.is_changed():
                self.debug("{0} modification time is unchanged, returning previous values".format(self.path))
                return self.data
            file = open(self.path, 'r')
        except:
            self.error("Error while opening {0}".format(self.path))
            self.data['error'] = 1
            self.data['warning'] = 0
            self.data['info'] = 0
            return self.data

        self.modtime = os.path.getmtime(self.path)
        lines = file.read()
        self.debug("LINES: {}".format(lines))

        try:
            self.data['error'] = re.findall(RE_error, lines)[0].split(',')[1]
        except IndexError as e:
            self.data['error'] = 0
        self.debug("ERROR MESSAGES: {}".format(self.data['error']))

        try:
            self.data['warning'] = re.findall(RE_warning, lines)[0].split(',')[1]
        except IndexError as e:
            self.data['warning'] = 0
        self.debug("WARNING MESSAGES: {}".format(self.data['warning']))

        try:
            self.data['info'] = re.findall(RE_info, lines)[0].split(',')[1]
        except IndexError as e:
            self.data['info'] = 0
        self.debug("INFO MESSAGES: {}".format(self.data['info']))

        return self.data

    def is_changed(self):
        return self.modtime != os.path.getmtime(self.path)

def is_readable(path):
    return os.path.isfile(path) and os.access(path, os.R_OK)

def is_empty(path):
    return os.path.getsize(path) == 0


import pandas as pd

class Investment:
    def __init__(self,amount,risk,duration,liquidity):

        '''
            0 <= risk <= 100 *Done
            duration in months *Done
            0 <= liquidity <= 100
        '''
        self.portfolio = {}
        self.amount = amount
        self.risk = risk
        self.duration = duration
        self.liquidity = liquidity
        self.stocksList = pd.read_csv("Investment/getMutualFunds/mfEquityDetails.csv")
        self.bondsList = pd.read_csv("Investment/getMutualFunds/corporateBondsDetails.csv")
        self.classEquityBonds()
        self.segregrateEquityBonds()
        self.getRequirements()
        self.printEquityChoices()

    def classEquityBonds(self):
        self.typeFundsEquity = {}
        self.typeBonds = {}
        st = set()
        for i in list(self.stocksList["Type"]):
            for j in i.split(","):
                st.add(j)

        for i in st:
            self.typeFundsEquity[i] = []

        durationsList = ['Return after 1 Year','Return after 3 Year','Return after 5 Year']

        durationBond = 0

        if self.duration < 12 :
            durationBond = 0
        elif self.duration < 36:
            durationBond = 1
        else:
            durationBond = 2

        for i in range(len(self.bondsList["Name"])):
            cp = 0
            for j in range(durationBond+1):
                if self.bondsList[durationsList[j]][i] != "-":
                    cp = j
            self.typeBonds[self.bondsList["Name"][i]] = [self.bondsList[durationsList[cp]][i],self.bondsList["Minimum SIP Amount"][i]]

    def segregrateEquityBonds(self):
        for i in range(len(self.stocksList["Type"])):
            temp = self.stocksList["Type"][i].split(",")
            for j in temp:
                add = [self.stocksList["Name"][i],self.stocksList["Number"][i],self.stocksList["Price"][i]]
                self.typeFundsEquity[j].append(add)

    def sorDict(self,dic):
        tmp = list(dic.values())
        tmp.sort()
        tmp = tmp[::-1]
        tmpp = {}

        for i in dic:
            tmpp[tuple(dic[i])] = i

        ret = {}
        for i in tmp:
            ret[tmpp[tuple(i)]] = i
        return ret

    def getRequirements(self):

        for i in self.typeBonds:
            if "%" in self.typeBonds[i][0]:
                self.typeBonds[i][0] = self.typeBonds[i][0][:-1]
            self.typeBonds[i][0] = float(self.typeBonds[i][0])

        # print(sorted(self.typeBonds.items(), key=lambda e: e[1][0]))

        self.typeBonds = self.sorDict(self.typeBonds)

        self.investmentTypeDuration = None
        self.riskAcceptedStocks = {
            "Large Cap" : 30,
            "Mid" :20,
            'Small' :20,
        }
        self.bondsPercentage = 10

        self.riskAcceptedStocks['Large Cap'] = 0.003*self.risk*self.risk-0.75*self.risk+60
        self.bondsPercentage = self.riskAcceptedStocks['Large Cap']/3
        self.riskAcceptedStocks["Mid"] = (100-self.riskAcceptedStocks['Large Cap']-self.bondsPercentage)/2
        self.riskAcceptedStocks["Small"] = self.riskAcceptedStocks["Mid"]

        self.bondsPercentage = self.liquidity

        tmp = self.riskAcceptedStocks["Large Cap"]+self.riskAcceptedStocks["Mid"]+self.riskAcceptedStocks['Small']
        self.riskAcceptedStocks["Large Cap"] /= tmp
        self.riskAcceptedStocks["Mid"] /= tmp
        self.riskAcceptedStocks["Small"] /= tmp

        tmp = 100-self.bondsPercentage
        self.riskAcceptedStocks["Large Cap"] *= tmp
        self.riskAcceptedStocks["Mid"] *= tmp
        self.riskAcceptedStocks["Small"] *= tmp

        self.amountLarge = self.amount*self.riskAcceptedStocks["Large Cap"]/100
        self.amountMid = self.amount*self.riskAcceptedStocks["Mid"]/100
        self.amountSmall = self.amount*self.riskAcceptedStocks["Small"]/100
        self.amountBonds =  self.amount*self.bondsPercentage/100

    def printEquityChoices(self):

        self.typeFundsEquity["Large Cap"].sort(key=lambda x: x[1])
        self.typeFundsEquity["Mid Cap"].sort(key=lambda x: x[1])
        self.typeFundsEquity["Focused"].sort(key=lambda x: x[1])
        self.typeFundsEquity["Small Cap"].sort(key=lambda x: x[1])
        self.typeFundsEquity["Value"].sort(key=lambda x: x[1])
        self.typeFundsEquity["Multi Cap"].sort(key=lambda x: x[1])
        self.typeFundsEquity["Contra"].sort(key=lambda x: x[1])
        self.typeFundsEquity["Large and Mid Cap"].sort(key=lambda x: x[1])

        i = 0
        while self.amountLarge > 0 and i < len(self.typeFundsEquity["Large Cap"]):
            if self.amountLarge > self.typeFundsEquity["Large Cap"][i][2]:
                self.portfolio[self.typeFundsEquity["Large Cap"][i][0]] = [self.amountLarge//self.typeFundsEquity["Large Cap"][i][2],self.typeFundsEquity["Large Cap"][i][2]]
                self.amountLarge = self.amountLarge - (self.amountLarge//self.typeFundsEquity["Large Cap"][i][2])*self.typeFundsEquity["Large Cap"][i][2]
            i += 1

        i = 0

        while self.amountMid > 0 and i < len(self.typeFundsEquity["Mid Cap"]):
            if self.amountMid > self.typeFundsEquity["Mid Cap"][i][2]:
                if self.typeFundsEquity["Mid Cap"][i][0] not in self.portfolio:
                    self.portfolio[self.typeFundsEquity["Mid Cap"][i][0]] = [self.amountMid//self.typeFundsEquity["Mid Cap"][i][2],self.typeFundsEquity["Mid Cap"][i][2]]
                    self.amountMid = self.amountMid - (self.amountMid//self.typeFundsEquity["Mid Cap"][i][2])*self.typeFundsEquity["Mid Cap"][i][2]
            i += 1

        i = 0

        while self.amountSmall > 0 and i < len(self.typeFundsEquity["Small Cap"]):
            if self.amountSmall > self.typeFundsEquity["Small Cap"][i][2]:
                if self.typeFundsEquity["Small Cap"][i][0] not in self.portfolio:
                    self.portfolio[self.typeFundsEquity["Small Cap"][i][0]] = [self.amountSmall//self.typeFundsEquity["Small Cap"][i][2],self.typeFundsEquity["Small Cap"][i][2]]
                    self.amountSmall = self.amountSmall - (self.amountSmall//self.typeFundsEquity["Small Cap"][i][2])*self.typeFundsEquity["Small Cap"][i][2]
            i += 1

        i = 0

        while self.amountBonds > 0 and i < len(self.typeBonds):

            if self.amountBonds > self.typeBonds[list(self.typeBonds.keys())[i]][1]:

                if self.typeBonds[list(self.typeBonds.keys())[i]][1] == 0:
                    self.portfolio[list(self.typeBonds.keys())[i]] = [self.amountBonds,self.typeBonds[list(self.typeBonds.keys())[i]][1]]
                    self.amountBonds = 0

                else:
                    self.portfolio[list(self.typeBonds.keys())[i]] = [(self.amountBonds//self.typeBonds[list(self.typeBonds.keys())[i]][1])*self.typeBonds[list(self.typeBonds.keys())[i]][1],self.typeBonds[list(self.typeBonds.keys())[i]][1]]

                    self.amountBonds = self.amountBonds -self.portfolio[list(self.typeBonds.keys())[i]][0]
            i += 1

        return self.portfolio

# for i in range(1000,20000):
#     for j in range(1,101):
#         for k in range(125):
#             for l in range(101):
#                 Investment(i,j,k,l)

class InvestmentMutualFunds:
    def __init__(self,amount,risk,duration,liquidity):
        '''
            If amount < 500, no MF, so suggest individual stocks
            if each individual < 500, join and SIP
        '''
        self.portfolio = {}
        if(amount > 500):
            self.amount = amount
            self.risk = risk
            self.duration = duration
            self.liquidity = liquidity
            # TODO: Liquidity for bonds
            self.mfList = pd.read_csv("Investment/getMutualFunds/mutualFundReturns.csv")
            self.minSIP = 500
            self.classMF()
            self.getPercentageFunds()
            self.getChoices()
        else:
            print("Please invest in individual Stocks, no MF with minimum SIP amount")

    def classMF(self):
        self.nameMF,self.typeMF,self.returnMF = list(self.mfList["Name"]),list(self.mfList["Type"]),list(self.mfList["Average Return"])
        self.typeName = {i : [] for i in set(self.typeMF)}
        for i in range(len(self.nameMF)):
            self.typeName[self.typeMF[i]].append([self.nameMF[i],self.returnMF[i],500])
        for i in self.typeName:
            self.typeName[i].sort(key = lambda x : x[1],reverse = True)
        # print([i for i in self.typeName])

    def modify(self):
        if self.duration <= 6:
            temp = 0.001*self.risk*self.risk + 0.15*self.risk + 15
            self.midCap = temp
            self.smallCap = temp
            self.ultraShort = 4*(100-temp-temp)/5
            self.largeCap = (100-2*temp)/5

        elif self.duration <= 12:
            self.midCap = 0.2*self.risk + 10
            self.smallCap = 0.2*self.risk + 10
            rem = 100-2*self.midCap
            self.lowDuration = 2*rem/3
            self.focused = self.lowDuration/8
            self.largeCap = self.focused*3

        elif self.duration <= 36:
            self.smallCap = 0.002*self.risk*self.risk + 10
            self.midCap = self.smallCap
            rem = 100-2*self.midCap
            self.shortDuration = 4*rem/7
            self.value = self.shortDuration/8
            self.contra = self.value
            self.focused = self.value
            self.largeCap = 3*self.value

        elif self.duration <= 48:
            self.midCap = 0.003*self.risk*self.risk - 0.05*self.risk + 5
            self.smallCap = 0.003*self.risk*self.risk - 0.05*self.risk + 5
            rem = (100-2*self.midCap)
            self.medDuration = rem/2
            self.largeCap = 3*rem/16
            self.focused = rem/16
            self.liquid = rem/4

        elif self.duration <= 84:
            self.smallCap = 0.003*self.risk*self.risk - 0.05*self.risk + 5
            self.midCap = self.smallCap
            rem = 100 - 2*self.midCap
            self.medLongDuration = rem/2
            self.largeCap = self.medLongDuration/2
            self.liquid = self.largeCap/2

        else:
            self.midCap = 0.002*self.risk*self.risk + 0.1*self.risk + 5
            self.smallCap = self.midCap
            rem = 100 - 2*self.smallCap # 70
            self.longDuration = 4*rem/7
            self.liquid = rem/7
            self.focused = self.liquid/2
            self.largeCap = 3*self.focused

    def getPercentageFunds(self):

        self.largeCap = 0
        self.midCap = 0
        self.smallCap = 0
        self.value = 0
        self.contra = 0
        self.focused = 0
        self.longDuration = 0
        self.medDuration = 0
        self.medLongDuration = 0
        self.shortDuration = 0
        self.lowDuration = 0
        self.ultraShort = 0
        self.liquid = 0

        if self.duration <= 6:
            self.ultraShort = 40
            self.largeCap = 10
            self.midCap = 25
            self.smallCap = 25

        elif self.duration <= 12:
            self.lowDuration = 40
            self.largeCap = 15
            self.midCap = 20
            self.smallCap = 20
            self.focused = 5

        elif self.duration <= 36:
            self.shortDuration = 40
            self.largeCap = 15
            self.midCap = 15
            self.smallCap = 15
            self.value = 5
            self.contra = 5
            self.focused = 5

        elif self.duration <= 48:
            self.medDuration = 40
            self.largeCap = 15
            self.midCap = 10
            self.smallCap = 10
            self.focused = 5
            self.liquid = 10

        elif self.duration <= 84:
            self.medLongDuration = 40
            self.largeCap = 20
            self.midCap = 10
            self.smallCap = 10
            self.liquid = 10

        else:
            self.longDuration = 40
            self.largeCap = 15
            self.midCap = 15
            self.smallCap = 15
            self.focused = 5
            self.liquid = 10

        self.modify()

    def getChoices(self):

        self.largeCap *= self.amount/100
        self.midCap *= self.amount/100
        self.smallCap *= self.amount/100
        self.value *= self.amount/100
        self.contra *= self.amount/100
        self.focused *= self.amount/100
        self.longDuration *= self.amount/100
        self.medDuration *= self.amount/100
        self.medLongDuration *= self.amount/100
        self.shortDuration *= self.amount/100
        self.lowDuration *= self.amount/100
        self.ultraShort *= self.amount/100
        self.liquid *= self.amount/100

        self.portfolio = {}
        self.remaining = 0

        def addPortfolio(typeAmount,fundsType):
            if typeAmount < 500:
                self.remaining += typeAmount
                return
            if typeAmount >=3000:
                self.portfolio[fundsType[0][0]] = [fundsType[0][1],typeAmount/2]
                self.portfolio[fundsType[1][0]] = [fundsType[1][1],typeAmount*0.3]
                self.portfolio[fundsType[2][0]] = [fundsType[2][1],typeAmount*0.2]
            elif typeAmount >= 2000:
                self.portfolio[fundsType[0][0]] = [fundsType[0][1],typeAmount*5/8]
                self.portfolio[fundsType[1][0]] = [fundsType[1][1],typeAmount*3/8]
            else:
                self.portfolio[fundsType[0][0]] = [fundsType[0][1],typeAmount]

        addPortfolio(self.largeCap,self.typeName['Large Cap'])
        addPortfolio(self.midCap,self.typeName['Mid Cap'])
        addPortfolio(self.smallCap,self.typeName['Small Cap'])
        addPortfolio(self.value,self.typeName['Value'])
        addPortfolio(self.contra,self.typeName['Contra'])
        addPortfolio(self.focused,self.typeName['Focused'])
        addPortfolio(self.longDuration,self.typeName['Long Duration'])
        addPortfolio(self.medDuration,self.typeName['Medium Duration'])
        addPortfolio(self.medLongDuration,self.typeName['Medium to Long Duration'])
        addPortfolio(self.shortDuration,self.typeName['Short Duration'])
        addPortfolio(self.lowDuration,self.typeName['Low Duration'])
        addPortfolio(self.ultraShort,self.typeName['Ultra short duration'])
        addPortfolio(self.liquid,self.typeName['Liquid'])
#
#         return self.portfolio


# -*- coding: utf-8 -*-
# __author__ = 'xieshichang'

import os
import re
import shutil
from biocluster.core.exceptions import OptionError
from biocluster.agent import Agent
from biocluster.tool import Tool
from mbio.packages.ref_rna.trans_step import step_count

class MetagPredictCombineAgent(Agent):
    def __init__(self, parent):
        super(MetagPredictCombineAgent, self).__init__(parent)
        options = [
            {"name": "fna_path", "type": "infile", "format": "sequence.fasta_dir"},
            {"name": "faa_path", "type": "infile", "format": "sequence.fasta_dir"},
            {"name": "len_range", "type": "string"},
            # 输入文件，预测后的序列路径
            {"name": "sample_stat", "type": "outfile", "format": "sequence.profile_table"},  # 输出文件，对各基因预测结果进行统计
            {"name": "fasta", "type": "outfile", "format": "sequence.fasta"},  # 输出核酸文件，输出序列用于构建非冗余基因集
            {"name": "faa", "type": "outfile", "format": "sequence.fasta"},  # 输出蛋白文件，输出序列用于构建非冗余基因集
            {"name": "fasta_sample", "type": "outfile", "format": "sequence.fasta"},  # 输出核酸文件，输出序列用于构建非冗余基因集
            {"name": "fasta_mix", "type": "outfile", "format": "sequence.fasta"},  # 输出核酸文件，输出序列用于构建非冗余基因集
            {"name": "faa_sample", "type": "outfile", "format": "sequence.fasta"},  # 输出单拼样本蛋白文件，输出序列用于构建非冗余基因集
            {"name": "faa_mix", "type": "outfile", "format": "sequence.fasta"},  # 输出混拼样本蛋白文件，输出序列用于构建非冗余基因集
        ]
        self.add_option(options)
        self.step.add_steps("MetagPredictCombine")
        self.on('start', self.stepstart)
        self.on('end', self.stepfinish)

    def stepstart(self):
        self.step.MetagPredictCombine.start()
        self.step.update()

    def stepfinish(self):
        self.step.MetagPredictCombine.finish()
        self.step.update()

    def check_options(self):
        """
        检查参数
        :return:
        """
        return True

    def set_resource(self):
        """
        设置所需资源，需在之类中重写此方法 self._cpu ,self._memory
        :return:
        """
        self._cpu = 5
        #self._memory = "50G"
        self._memory = str((50 + self._rerun_time * 20)) + 'G' # by xieshichang 20200424

    def end(self):
        result_dir = self.add_upload_dir(self.output_dir)
        result_dir.add_relpath_rules([
            [".", "", "结果输出目录"],
        ])
        result_dir.add_regexp_rules([
            ["", "", ""]
        ])
        super(MetagPredictCombineAgent, self).end()

class MetagPredictCombineTool(Tool):
    def __init__(self, config):
        super(MetagPredictCombineTool, self).__init__(config)
        self._version = "1"
        self.python_path = '/miniconda2/bin/python '
        # self.gene_stat_path = self.config.SOFTWARE_DIR + '/bioinfo/metaGenomic/scripts/gene_stat.py'
        self.gene_stat_path = self.config.PACKAGE_DIR + '/gene_structure/gene_stat.py'
        self.set_environ(LD_LIBRARY_PATH = self.config.SOFTWARE_DIR + '/bioinfo/seq/EMBOSS-6.6.0/lib')
        self.emboss_path = "/bioinfo/seq/EMBOSS-6.6.0/emboss/"

    def run(self):
        """
        运行
        :return:
        """
        super(MetagPredictCombineTool, self).run()
        self.run_stat()
        self.step_count()
        self.combine_faa()
        self.set_output()
        self.end()

    def run_stat(self):
        """
        gene_stat -gene_dir  gene.directory -output_stat  stat_file -output_fa  fasta_file
        :return:
        """
        cmd = self.python_path + '{} -gene_dir {} -output_stat {} -output_fa {} -sp_name {}'
        cmd = cmd.format(self.gene_stat_path, self.option('fna_path').prop['path'],
                         self.work_dir + '/sample.metagene.stat', self.work_dir + '/Total.metagene.fa', 1)
        command = self.add_command("metagenestat", cmd)
        command.run()
        self.wait(command)
        if command.return_code == 0:
            self.logger.info("运行metagenestat的cmd完成")
        else:
            self.set_error("运行metagenestat的cmd运行出错!", code="32201201")

    def step_count(self):
        """
        调用函数，对挑选出来的序列,整理到文件夹汇总，并进行步长统计
        :return:
        """
        self.logger.info("开始组装部分进行步长统计")
        step_list = self.option('len_range').split(',')
        all_files = self.option('fna_path').fastas_full
        all_files.append(self.work_dir + '/Total.metagene.fa')
        if not os.path.exists(os.path.join(self.output_dir, "len_distribute")):
            os.mkdir(os.path.join(self.output_dir, "len_distribute"))
        for file_full in all_files:
            file_name = os.path.basename(file_full)
            sample_name = file_name.split('.')[0]
            for step in step_list:
                output1 = self.work_dir + '/' + sample_name + '_step_' + str(step) + '.txt'
                output2 = self.output_dir + '/len_distribute/' + sample_name + '_step_' + str(step) + '.final.txt'
                if os.path.exists(output2):
                    os.remove(output2)
                step_count(file_full, output1, 20, int(step), output2)
        self.logger.info("序列步长统计结束")

    def combine_faa(self):
        faa_mix = self.work_dir + '/Total.metagene_mix.faa'
        faa_sample = self.work_dir + '/Total.metagene_sample.faa'
        faa = self.work_dir + '/Total.metagene.faa'
        if os.path.exists(self.work_dir + '/Total.metagene.fa_mix'):
            mix = True
        else:
            mix = False
        for f in [faa_mix, faa_sample, faa]:
            if os.path.exists(f):
                os.remove(f)
        for f in os.listdir(self.option('faa_path').path):
            f_path = os.path.join(self.option('faa_path').path, f)
            f_name = f.partition('.')[0]
            cmd1 = ''
            if f_name in ["newbler", "Megahit_Mix"]:
                cmd1 = 'cat {} >> {}'.format(f_path, faa_mix)
            elif mix:
                cmd1 = 'cat {} >> {}'.format(f_path, faa_sample)
            cmd2 = 'cat {} >> {}'.format(f_path, faa)
            if os.system(cmd2) != 0:
                self.set_error('蛋白序列合并失败')
            if mix and os.system(cmd1) != 0:
                self.set_error('蛋白序列合并失败')

    def set_output(self):
        """
        将结果文件复制到output文件夹下面
        :return:
        """
        self.logger.info("设置结果目录")
        if os.path.exists(self.output_dir + '/sample.metagene.stat'):
            os.remove(self.output_dir + '/sample.metagene.stat')
        if os.path.exists(self.output_dir + '/Total.metagene.fa'):
            os.remove(self.output_dir + '/Total.metagene.fa')
        if os.path.exists(self.output_dir + '/Total.metagene.faa'):
            os.remove(self.output_dir + '/Total.metagene.faa')
        if os.path.exists(self.work_dir + '/Total.metagene.faa'):
            os.link(self.work_dir + '/Total.metagene.faa', self.output_dir + '/Total.metagene.faa')
            self.option('faa').set_path(self.output_dir + '/Total.metagene.faa')
        if os.path.exists(self.work_dir + '/sample.metagene.stat'):
            os.link(self.work_dir + '/sample.metagene.stat', self.output_dir + '/sample.metagene.stat')
            self.option('sample_stat').set_path(self.output_dir + '/sample.metagene.stat')
        if os.path.exists(self.work_dir + '/Total.metagene.fa'):
            os.link(self.work_dir + '/Total.metagene.fa', self.output_dir + '/Total.metagene.fa')
            self.option('fasta').set_path(self.output_dir + '/Total.metagene.fa')
        if os.path.exists(self.work_dir + '/Total.metagene_sample.faa'):
            self.option('faa_sample').set_path(self.work_dir + '/Total.metagene_sample.faa')
            self.option('fasta_sample').set_path(self.work_dir + '/Total.metagene.fa_sample')
        if os.path.exists(self.work_dir + '/Total.metagene_mix.faa'):
            self.option('faa_mix').set_path(self.work_dir + '/Total.metagene_mix.faa')
            self.option('fasta_mix').set_path(self.work_dir + '/Total.metagene.fa_mix')
        self.logger.info("设置Metagene分析结果目录成功")


# -*- coding: utf-8 -*-
"""
Created on Wed Nov  4 14:14:59 2020

This script is used to download Point Cloud Library Files.
"""

import requests
from bs4 import BeautifulSoup
import wget #for downloading files
# convert url to normal string
from urllib.parse import unquote
import os

url_base = "https://sourceforge.net/"
url_start = "https://sourceforge.net/projects/pointclouds/files/PCD%20datasets/"

def download_folder(url):
    if url[-1] == '/':
        url = url[:-1]

    folder = unquote(url.split('/')[-1])
    if not os.path.exists(folder):
        os.mkdir(folder)
    os.chdir(folder)
    print("go to", folder)
    response = requests.get(url)
    soup = BeautifulSoup(response.text, features="lxml")
    items = soup.findAll("span", attrs={"class" : "name"})
    items = [item.parent.get("href").replace("/download", "") for item in items]
    print(len(items))

    files = []
    nexturls = []
    for item in items:
        if not item.endswith("/"):
            files.append(item)
        else:
            nexturls.append(url_base + item)

    for file in files:
        print("download ", file)
        # wget.download(file)
        """
        need to install the latest windows wget from
        https://eternallybored.org/misc/wget/
        """
        os.system("wget --continue " + file)

    for nexturl in nexturls:
        download_folder(nexturl)

    os.chdir("..")

if __name__ == "__main__":
    download_folder(url_start)


#necessary imports
import numpy as np
import pandas as pd
import csv
import lucene as ls
from java.io import File

#index imports
from org.apache.lucene.analysis.standard import StandardAnalyzer
from org.apache.lucene.analysis.en import EnglishAnalyzer
from org.apache.lucene.index import IndexWriter, IndexWriterConfig
from org.apache.lucene.store import SimpleFSDirectory, FSDirectory
import org.apache.lucene.document as document

# importing beautiful soup for html scrapping
import os
from bs4 import BeautifulSoup
import requests

# default paths
path = 'adhoc.fire.en/en.docs.2011/'
indexPath='index/'

# taking input from command line
import sys

n=len(sys.argv)
# path in en directory
if sys.argv[1]!='':
	path=sys.argv[1]

# path to index directory path
if sys.argv[2]!='':
	indexPath=sys.argv[2]

#inistialise the VM
ls.initVM()

indexPath=File(indexPath).toPath()
indexDir=FSDirectory.open(indexPath)

# we are using Standard analyser
# stop words removal from english analyzer
writerConfig=IndexWriterConfig(StandardAnalyzer(EnglishAnalyzer.ENGLISH_STOP_WORDS_SET))
writer=IndexWriter(indexDir,writerConfig)

# indexing wrt docno,title and text data
def indexNews(docno,title,text):
    doc=document.Document()
    doc.add(document.Field("DOCNO",docno,document.TextField.TYPE_STORED))
    doc.add(document.Field("TITLE",title,document.TextField.TYPE_STORED))
    doc.add(document.Field("TEXT",text,document.TextField.TYPE_STORED))
    writer.addDocument(doc)

# close the writer
def closeWriter():
    writer.close()

# FINDING THE FILE PATH
# Traverse from the root and find out if if there is any file in that folder store that path

# MAIN FUNCTION

if __name__=="__main__":
	# it goes to each file and parse the content of each tag in the file
	for root, directories, files in os.walk(path, topdown=False):
	    for name in files:
	    	# read the content of the file
	        file=open(os.path.join(root, name),'r').read()
	        # initialise the beauitful soup object
	        soup = BeautifulSoup(file, "lxml")

	        # find titles of news
	        titles=soup.find_all('title')
	        title_text=' '
	        # if some title if found
	        if len(titles)>0:
	            for title in titles:
	                title_text=title.text

	        # find docno of news
	        docnos=soup.find_all('docno')
	        docno_text=' '
	        # if some docno is found
	        if len(docnos)>0:
	            for docs in docnos:
	                docno_text=docs.text

	        # find the text of the news
	        texts=soup.find_all('text')
	        text_text=' '
	        # if some news if found
	        if len(texts)>0:
	            for txt in texts:
	                text_text=txt.text

	        # do the indexing
	        indexNews(docno_text,title_text,text_text.strip('\n').strip())
	# close the writer
	closeWriter()


import math

class Cuadrado:
    def __init__(self,lado):
        self.lado= lado
    def area (self):
        A = self.lado * self.lado
        return A
    def perimetro (self):
        P = self.lado *4
        return P
    def diagonal (self):
        D = self.lado* math.sqrt(2)
        return D

class Triangulo:
    def __init__(self,base,altura):
        self.base=base
        self.altura=altura

    def area (self):
        A = (self.base*self.altura)/2
        return A

    def perimetro (self):
        P = math.sqrt(self.base**2 + self.altura**2)
        return P

class Circulo:
    def __init__(self,radio):
        self.radio=radio

    def area (self):
        A = math.pi * self.radio ** 2
        return A

    def perimetro (self):
        P = 2 * math.pi * self.radio
        return P

print ("Bienvenido")
print ("Opciones disponibles")
print ("1-Cuadrado\n2-Triangulo\n3-Circulo")
opcion = int(input("Elige una opción [1,2,3]:  "))
if opcion == 1:
    print ("Calculos disponibles: ")
    print ("1-Area\n2-Perimetro\n3-Diagonal")
    calculo = int(input("Elige una opción: "))
    lado = float (input("Ingresa el lado del cuadrado:"))

    #crear un cuadrado con el lado especificado
    cuadrado=Cuadrado(lado)
    if calculo == 1:
        A = cuadrado.area()
        print ("El area del cuadrado es ",A)
    elif calculo == 2:
        P = cuadrado.perimetro()
        print (" El perimetro del cuadrado es: ", P)
    elif calculo == 3:
        D = cuadrado.diagonal()
        print (" la diagonal del cuadrado es: ", D)

elif opcion == 2:
    print ("Calculos disponibles: ")
    print ("1-Area\n2-Perimetro")
    calculo = int(input("Elige una opción: "))
    base = float (input("Ingresa la base del triangulo: "))
    altura = float (input("Ingresa la altura del triangulo: "))
    triangulo=Triangulo(base,altura)
    if calculo == 1:
        A = triangulo.area()
        print ("El A del triangulo es ",A)
    elif calculo == 2:
        P = triangulo.perimetro()
        print (" El P del triangulo es: ", P)

elif opcion == 3:
    print ("Calculos disponibles: ")
    print ("1-Area\n2-Perimetro")
    calculo = int(input("Elige una opción: "))
    radio = float (input("Ingresa el radio del circulo: "))
    circulo= Circulo(radio)
    if calculo == 1:
        A = circulo.area()
        print ("El A del circulo es ",A)
    elif calculo == 2:
        P = circulo.perimetro
        print (" El P del circulo es: ", P)


# Задана натуральная степень k. Сформировать случайным образом список коэффициентов
# (значения от 0 до 100) многочлена и записать в файл многочлен степени k.
# Пример:
# если k = 2, то многочлены могут быть => 2*x² + 4*x + 5 = 0 или x² + 5 = 0 или 10*x² = 0
import random

import sympy

def polynomial():
    k = int(input('Введите максимальную степень числа: '))
    my_dict = {}
    for i in range(k, -1, -1):
        y = random.randint(0, 100)
        my_dict[i] = y
    print(my_dict)
    poly = ''
    for n in range(k, -1, -1):
        if my_dict[n] == 0:
            poly += f''
        elif n == 0:
            poly += f'{my_dict[n]}'
        elif n == 1:
            poly += f'{my_dict[n]}*x + '
        else:
            poly += f'{my_dict[n]}*x**{n} + '
    poly += ' = 0'
    return poly

poly_1 = polynomial()
print(poly_1)
with open('file.txt', 'w') as data:
    data.write(f'{poly_1}')
poly_2 = polynomial()
print(poly_2)
with open('file_2.txt', 'w') as data:
    data.write(f'{poly_2}')


import os
import networkx as nx
import numpy as np
import matplotlib.pyplot as plt

path = 'Data/'

dirs = os.listdir(path)

for dir in dirs:
    listing = os.listdir(path+dir+'/')
    if dir == 'Archae':
        s = '^'
        c = 'r'
    else:
        if dir == 'Bacterium':
            s = 's'
            c = 'g'
        else:
            s = 'D'
            c = 'b'
    for infile in listing:
        print("Current file is ",infile)
        fh = open(path+dir+'/'+infile,"rb")
        G = nx.read_edgelist(fh, create_using=nx.DiGraph())
        fh.close()
        nodes = G.nodes()
        nodes = [node for node in nodes if int(node) < 1000000]
        print(len(nodes))
        n = len(nodes)
        li = nx.all_pairs_shortest_path_length(G)
        l = []
        for node in nodes:
            l = l + [li[node][x] for x in nodes if x in li[node]]

        paths = np.array(l)
        diameter = np.mean(paths)/2
        plt.errorbar(n,diameter,yerr = np.std(paths)/2,fmt=s,color=c,ms='12',markeredgecolor=c)
#plt.axis([0,900,1,5])
plt.xlabel('Number of Nodes(N)',fontsize=40)
plt.ylabel('Diameter',fontsize=40)
plt.xticks(fontsize=25)
plt.yticks(fontsize=25)
#plt.tight_layout()
plt.show()