# -*- coding:utf-8 -*-
# @Author : Bo Zhao
# @Time   : 12/1/2022 6:21 PM

import requests
from bs4 import BeautifulSoup
import json

def scrape_country(country,url,day = 'today'):

    print('Scraping...')
    '''
    request the whole website content in url
    '''
    response = requests.post(url).text
    print('Scraping finished!')
    print('--------------------')
    identity = 'main_table_countries_' + day  # id of desired contents
    soup = BeautifulSoup(response, 'html.parser')    # parsing the HTML documents using html.parser
    today = soup.find('table', attrs={'id': identity})  # extract the contents that has covid data
    data_today = today.tbody.find_all('tr')  # extract all data from 'tr' subsection

    data = [] # detailed data of all country
    for index in range(1, len(data_today)):
        r = []
        for tr in data_today[index].find_all('td'):
            r.append(tr.text.replace('\n', '').strip())
        data.append(r)
    data = data[7:]

    head = today.thead.find_all('th') # find all the header of the table
    header = []
    for i in head:
        header.append(i.text.replace('\n', '').strip())
    '''
    use table header as key and detailed covid data as value to creat a dictionary for each country
    put every dictionary into a list
    '''
    data_dic_all = []
    for i in data:
        data_dic = {}
        for j in range(len(header)):
            data_dic[header[j]] = i[j]
            if len(data_dic) == 22:
                data_dic['TotCases/1M pop'] = data_dic['Tot Cases/1M pop']
                del data_dic['Tot Cases/1M pop']
        data_dic_all.append(data_dic)
    '''
    find out the data of the input country
    print out the raw data and normalized data
    '''
    total_num = 0
    total_num_relative = 0
    country_name = ''
    country_data = {}
    for i in data_dic_all:
        if i['Country,Other'].upper().replace(' ','').strip() == country.upper().replace(' ','').strip():
            total_num = i['TotalCases']
            country_name = i['Country,Other']
            #total_num_relative = i['Tot Cases/1M pop']
            total_num_relative = i['TotCases/1M pop']
            country_data = i

    del country_data['#']
    # print(country_name,':')
    # print('Total Cases:',total_num)
    # print('Total Cases / 1M Population:',total_num_relative)
    '''
    save the json file
    '''
    if day == 'today':
        with open('today.json','w') as f:
            f.write(json.dumps(data_dic_all))
    if day == 'yesterday':
        with open('yesterday.json','w') as f:
            f.write(json.dumps(data_dic_all))
    if day == 'yesterday2':
        with open('two_days_ago.json','w') as f:
            f.write(json.dumps(data_dic_all))
    '''return a dictionary of input country'''
    return country_data

if __name__ == '__main__':
    url = 'https://www.worldometers.info/coronavirus/'
    country = 'Turkey'
    day = 'today'  # valid input of day: 'today'(default)/ 'yesterday'/ 'yesterday2'
    data = scrape_country(country, url, day)
    # print(data)


#!/usr/bin/env python3
import spacy
import sqlite3
import ast
import csv

def write_to_csv(fields, data, filename):
    with open(filename, 'w') as csvfile:
        writer = csv.DictWriter(csvfile, fieldnames=fields)
        writer.writeheader()
        writer.writerows(data)

if __name__ == "__main__":
    conn = sqlite3.connect('/home/leander/Desktop/automatic_KB_construction/recipes/old_recipes/recipes1.db')
    c = conn.cursor()
    c.execute("SELECT * FROM Recipes;")
    rows = c.fetchall()
    nlp = spacy.load('en_core_web_trf')
    verbs = {}
    nouns = {}

    for recipe in rows:
        dictionary = ast.literal_eval(recipe[7])

        for key in dictionary:
            step = nlp(dictionary[key])
            sentences = list(step.sents)

            for sentence in sentences:
                for token in sentence:
                    word = token.lemma_.lower()

                    if token.pos_ == "VERB":
                        if word in verbs:
                            verbs[word] += 1
                        else:
                            verbs[word] = 1

                    elif token.pos_ == "NOUN":
                        if word in nouns:
                            nouns[word] += 1
                        else:
                            nouns[word] = 1

    sorted_verbs = dict(sorted(verbs.items(), key=lambda item: item[1], reverse=True))
    csv_verb = []
    for verb_k in sorted_verbs:
        csv_verb.append({'Verb': verb_k, 'Counter': sorted_verbs[verb_k]})

    sorted_nouns = dict(sorted(nouns.items(), key=lambda item: item[1], reverse=True))
    csv_noun = []
    for noun_k in sorted_nouns:
        csv_noun.append({'Noun': noun_k, 'Counter': sorted_nouns[noun_k]})

    write_to_csv(list(csv_verb[0].keys()), csv_verb, 'verbs_in_all_recipes.csv')
    write_to_csv(list(csv_noun[0].keys()), csv_noun, 'nouns_in_all_recipes.csv')


import scipy
import numpy as np
import pandas as pd

from matplotlib.pylab import plt

def testPandas():
    df = pd.DataFrame()
    df['x'] = [1,2,3,4,5]
    xPlot = df['x']  # pandas series

    plt.plot(xPlot)
    plt.show()

def runFilter():

    # make sample data
    xRaw = np.random.normal(size=1000)
    yRaw = np.random.normal(size=1000)

    # make a tmp dataframe (like line annotations)
    df = pd.DataFrame()
    df['x'] = xRaw
    df['y'] = yRaw

    # this is what your code is doing
    xPlot = df['x']
    yPlot = df['y']

    # given two 1D arrays, stack them and transpoe to get
    # two columns corresponding to (x,y)
    xyData = np.vstack((xPlot, yPlot)).T

    print(xyData.shape) # (1000,2)

    width = 5  # box with for median filter (must be odd)
    yFiltered = scipy.ndimage.median_filter(xyData, width)

    plt.plot(xPlot, yPlot, '.-k')  # raw data
    #plt.plot(yFiltered, '.-r')
    plt.show()

def filterOneSegment(segmentID : int, medianFilterWidth : int = 5):
    """

    Args:
        width: box with for median filter (must be odd)
            # width is the number of points (pixels) to consider for each median filer
    """

    import pymapmanager
    path = '../PyMapManager-Data/one-timepoint/rr30a_s0_ch2.tif'
    stack = pymapmanager.stack(path)
    la = stack.getLineAnnotations()

    dfSegment = la.getSegment(segmentID)

    xPlot = dfSegment['x']
    yPlot = dfSegment['y']

    # transpose rotates to the "right"
    # by convention, to the "right" is -90
    #xyData = np.vstack((xPlot, yPlot)).T

    xPlotFiltered = scipy.ndimage.median_filter(xPlot, medianFilterWidth)
    yPlotFiltered = scipy.ndimage.median_filter(yPlot, medianFilterWidth)

    #xPlotFiltered = xyFiltered[:,1]
    #yPlotFiltered = xyFiltered[:,0]

    # now continue your calculations of (xLeft, yLeft) and (xRight, yRight)

    plt.plot(xPlot, yPlot, '.-k')  # raw data
    plt.plot(xPlotFiltered, yPlotFiltered, '.-r')  # filtered data
    plt.show()

if __name__ == '__main__':
    #runFilter()
    #testPandas()

    segmentID = 0
    medianFilterWidth = 9
    filterOneSegment(segmentID, medianFilterWidth)


from django.urls import path
from .views import *

urlpatterns = [
	path('', crud_index, name='crud_index'),
	path('nuevoAlumno', crud_nuevoAlumno, name='crud_nuevoAlumno'),
	path('updateAlumno/<int:rutIn>/', crud_updateAlumno, name='crud_updateAlumno'),
	path('deleteAlumno/<int:rutIn>/', crud_deleteAlumno, name='crud_deleteAlumno'),

	path('nuevoProfesor', crud_nuevoProfesor, name='crud_nuevoProfesor'),
	path('updateProfesor/<int:rutIn>/', crud_updateProfesor, name='crud_updateProfesor'),
	path('deleteProfesor/<int:rutIn>/', crud_deleteProfesor, name='crud_deleteProfesor'),

	path('nuevaPrueba', crud_nuevaPrueba, name='crud_nuevaPrueba'),
	path('updatePrueba/<int:idPruebaIn>/', crud_updatePrueba, name='crud_updatePrueba'),
	path('deletePrueba/<int:idPruebaIn>/', crud_deletePrueba, name='crud_deletePrueba'),

	path('nuevoCurso', crud_nuevoCurso, name='crud_nuevoCurso'),
	path('updateCurso/<int:idCursoIn>/', crud_updateCurso, name='crud_updateCurso'),
	path('deleteCurso/<int:idCursoIn>/', crud_deleteCurso, name='crud_deleteCurso'),

	path('nuevoCursoAlumno', crud_nuevoCursoAlumno, name='crud_nuevoCursoAlumno'),
	path('updateCursoAlumno/<int:idCAIn>/', crud_updateCursoAlumno, name='crud_updateCursoAlumno'),
	path('deleteCursoAlumno/<int:idCAIn>/', crud_deleteCursoAlumno, name='crud_deleteCursoAlumno'),

	path('nuevoCursoPruebaAlumno', crud_nuevoCursoPruebaAlumno, name='crud_nuevoCursoPruebaAlumno'),
	path('updateCursoPruebaAlumno/<int:idCursoAlumnoAIn>/', crud_updateCursoPruebaAlumno, name='crud_updateCursoPruebaAlumno'),
	path('deleteCursoPruebaAlumno/<int:idCursoAlumnoAIn>/', crud_deleteCursoPruebaAlumno, name='crud_deleteCursoPruebaAlumno'),

	path('listarAlumnosPromedio', listaPromedio, name='listaPromedio'),
	path('listarPromedioRojo', listaPromediosRojos, name='listaPromediosRojos'),
]


import random

attackers = ['Montagne', 'Sledge', 'Ash', 'Blackbeard', 'Blitz', 'Buck', 'Capitao', 'Dokkaebi', 'Finka', 'Fuze', 'Glaz', 'Gridlock', 'Hibana', 'Iana', 'Jackal', 'Lion', 'Maverick', 'Nomad', 'Nokk', 'Thatcher', 'Thermite', 'Twitch', 'Ying', 'Zofia', 'Zero', 'Flores', 'Osa', 'Sens', 'Grim', 'Brava', 'Amaru', 'Ace', 'Ram', 'Kali', 'IQ']
defenders = ['Mute', 'Smoke', 'Castle', 'Pulse', 'Doc', 'Rook', 'Kapkan', 'Tachanka', 'Jager', 'Bandit', 'Frost', 'Valkyrie', 'Caveira', 'Echo', 'Mira', 'Lesion', 'Ela', 'Vigil', 'Maestro', 'Alibi', 'Clash', 'Kaid', 'Mozzie', 'Warden', 'Goyo', 'Wamai', 'Oryx', 'Melusi', 'Aruni', 'Thunderbird', 'Thorn', 'Azami', 'Solis', 'Fenrir']

def operators(side, n):
    updating_list = side
    operator_list = []
    for _ in range(n):
        random_operator = random.choice(updating_list)
        operator_list.append(random_operator)
        updating_list.remove(random_operator)

    print(operator_list)

# Call the function
operators(defenders, 3)


"""
두 수의 합 찾기

주어진 정수형 배열에서 2개의 숫자를 선택하여 더한 값이 특정 목표값을 만들 때
그 선택한 2개의 정수가 있는 배열의 인덱스를 반환하는 프로그램 작성,

입력값으로 주어지는 배열에는 정확히 하나의 정답이 존재하며, 같은 요소의 값을 중복해서 사용할 수 없음

입력값 : nums = [2, 7, 10, 19], target = 9
출력값 : [0, 1]
"""

"""
1. Brute Force
nums = [2, 7, 10, 19]
target = 9

for i in range(0, len(nums) - 2):
    for j in range(i + 1, len(nums) - 1):
        if (nums[i] + nums[j] == target):
            print([i, j])
"""

# 2. HashTable

"""
1. 해시테이블을 구성
    - 키값으로는 배열의 요소, 값으로는 요소의 인덱스로 구성
2. 각 요소를 순회하면서
    - 목표값 - 현재 요소 = 다른 요소
    - 해시 테이블에서 다른 요소의 값을 찾는다
    - 만약 다른 요소가 해시 테이블에 있다면, 현재 요소의 인덱스와 해시 테이블의 값(인덱스)을 반환한다.
    - 다른 요소가 없다면, 현재 요소를 해시 테이블의 키값으로 넣고 인덱스를 해시 테이블의 값 항목으로 추가
"""
nums = [2, 7, 10, 19]
target = 9

hashtable_dict = {}
for i in range(len(nums)):
    other_elem = target - nums[i]
    if hashtable_dict.get(other_elem) is not None and hashtable_dict[other_elem] != i:
        print(sorted([i, hashtable_dict[other_elem]]))
    hashtable_dict[nums[i]] = i


from plugins import query, admin

from pyrogram import Client, filters
from pyrogram.enums import ChatAction, ParseMode

@Client.on_message(filters.private & filters.user(admin) & filters.command("help"))
async def help_message(client, message):
    chat_id = message.chat.id

    help_message = """⚡ **اپشن های موجود** ⚡
        \n☔ اضافه کردن پیام به ربات با کامند     /add_msg
        \n☔ حذف کردن پیام از ربات با کامند       /del_msg
        \n☔ نمایش جمسملات موجود با کامند       /show_msg
        \n☔ نمایش وضعیت ربات     /info
        \n☔ چک کردن وضعیت ربات با کامند      /status
        \n☔ فوروارد کردن پیام       /forward
        \b☔ جوین شدن داخل گروه مورد نظر     /join_chat
        \n☔ ست کردن پیام بعد از جوین شدن ربات در گروه       /join_msg
        \n.
    """

    await client.send_chat_action(chat_id=chat_id, action=ChatAction.TYPING)
    await message.reply(text=help_message,parse_mode=ParseMode.HTML)

@Client.on_message(filters.private & filters.user(admin) & filters.command("add_msg"))
async def add_msg(client, message):
    chat_id = message.chat.id
    messages = """⚡ **اضافه کردن پیام** ⚡
        \nلطفا جمله یا جملات خود را ارسال کنید و در انتها کامند زیر را ارسال کنید:
        \n/save
    """

    await client.send_chat_action(chat_id=chat_id, action=ChatAction.TYPING)
    await message.reply(text=messages)

    query.setbit("add:msg", 0, 1)

@Client.on_message(filters.private & filters.user(admin), group=1)
async def get_input(client, message):
    msg = message.text

    if query.getbit("add:msg", 0) == 1 and msg and not msg.startswith("/"):
        msg_counter = query.incr("New:Msg")
        query.sadd("Messages", f"{msg_counter}:{msg}")

    elif msg == "/save":
        chat_id = message.chat.id
        await client.send_chat_action(chat_id=chat_id, action=ChatAction.TYPING)
        await message.reply("""⚡ **اضافه کردن پیام** ⚡
            \n✅ جمله های شما با موفقیت ذخیره شد
            \n.
        """)

        query.setbit("add:msg", 0, 0)

@Client.on_message(filters.private & filters.user(admin) & filters.command("show_msg"))
async def show_msg(client, message):
    chat_id = message.chat.id

    await client.send_chat_action(chat_id=chat_id, action=ChatAction.TYPING)

    if query.exists("Messages"):
        messages = "⚡ **نمایش جمله ها** ⚡\n"
        data = sorted(query.smembers("Messages"), key=lambda x: x.split(":")[0])

        for item in data:
            item = item.split(":", 1)
            separator = "➖" * 12
            messages += f"`{item[0]}`:{item[1]}\n{separator}\n"
        await message.reply(messages)

    else:
        await message.reply("❌جمله ای یافت نشد❌")

# @Client.on_message(filters.private & filters.user(admin) & filters.command("del_msg"))
# async def del_msg(c, m):
#     chat_id = m.chat.id

#     await c.send_chat_action(chat_id=chat_id, action="typing")

#     if r.exists("Messages"):

#         message = """⚡ **حذف کردن جمه ها** ⚡
#             \nلطفا شماره جمله مورد نظر خود ارسال کنید. 🗑
#             \nو یا برای حذف همه جمله ها  `*` را ارسال نمایید
#             \n.
#         """

#         await c.send_message(chat_id=chat_id, text=message)

#         r.setbit("del:msg", 0, 1)

#     else:
#         await c.send_message(chat_id=chat_id, text="❌جمله ای یافت نشد❌")

# @Client.on_message(filters.private & filters.user(admin), group=2)
# async def get_message_id(c, m):

#     msg = m.text

#     if r.getbit("del:msg", 0) == 1 and msg and not msg.startswith("/"):
#         chat_id = m.chat.id
#         await c.send_chat_action(chat_id=chat_id, action="typing")

#         if msg == "*":
#             r.delete("Messages")
#             r.setbit("del:msg", 0, 0)
#             await c.send_message(chat_id=chat_id, text="""⚡ **حذف همه پیام ها** ⚡\n
#                 \nتمامی جملات شما با موفقیت حذف شد ✅"""
#             )

#         else:
#             msg_number = r.sscan("Messages", match=f"{msg}:*")[1]

#             if msg_number:
#                 r.srem("Messages", msg_number[0])
#                 r.setbit("del:msg", 0, 0)
#                 await c.send_message(chat_id=chat_id, text="""⚡ **حذف پیام** ⚡\n
#                     \n جمله شما با موفقیت حذف شد ✅"""
#                 )

#             else:
#                 messages = "⚡ **حذف پیام** ⚡\n\n❌جمله ای یافت نشد❌"

#                 await c.send_message(chat_id=chat_id, text=messages)


# -*- coding: utf-8 -*-
"""
Created on Sat Jan 29 10:09:16 2022

@author: 45414
"""

# Imports
import tensorflow as tf
import pandas as pd
from tensorflow.keras.preprocessing.image import ImageDataGenerator
import splitfolders
from tensorflow.keras.layers import Dropout, Flatten, Dense, BatchNormalization
import matplotlib.pyplot as plt

path = 'C:/Users/45414/Desktop/Anvendt maskinelæring/DIDA/'

df = pd.read_csv(path + 'DIDA_12000_String_Digit_Labels.csv', names=["Index","Label"])

#%%
def append_ext(fn):
    return str(fn) + ".jpg"

df["nr"]=df["Label"].apply(append_ext)

# Create new labels for CC-D-Y models
# 18 century (0) or not (1)
def label_CC(label):

    x = str(label)
    if len(x) == 4:
        if x[0] == "1" and x[1] == "8":
            return str(0)
        else:
            return str(1)
    return str(1)

# Decade or residual (10)
def label_D(label):

    x = str(label)
    if len(x) != 4:
        return str(10)
    else:
        return str(x[2])

# Year or residual (10)
def label_Y(label):

    x = str(label)
    if len(x) != 4:
        return str(10)
    else:
        return str(x[3])

def filename(index):
   return str(index) + '.jpg'

df["CC"]=df["Label"].apply(label_CC)
df["D"]=df["Label"].apply(label_D)
df["Y"]=df["Label"].apply(label_Y)
df["Filename"]=df["Index"].apply(filename)

# Check the ratio of samples in each class:
print(df["CC"].value_counts()) # how many are there in each category
print(df["D"].value_counts()) # how many are there in each category
print(df["Y"].value_counts()) # how many are there in each category

input_folder = path + 'DIDA_12000_String_Digit_Images/'
splitfolders.ratio(input_folder, output= path + 'cell_image',
                   seed=42, ratio=(.7, .2, .1),
                   group_prefix=None) # default values

#%% CC
datagen=ImageDataGenerator(
    rescale=1./255.,
    rotation_range=5,
    zoom_range=0.2,
    horizontal_flip=True,
    vertical_flip=False
    )

datagen_test=ImageDataGenerator(rescale=1./255.)

train_generator_CC=datagen.flow_from_dataframe(
dataframe=df,
directory=path + "cell_image/train/DIDA_1/",
x_col="Filename",
y_col="CC",
batch_size=200,
seed=42,
shuffle=True,
class_mode="binary",
target_size=(32,64))

valid_generator_CC=datagen_test.flow_from_dataframe( #maybe use augmentation on val also?
dataframe=df,
directory= path + "cell_image/val/DIDA_1/",
x_col="Filename",
y_col="CC",
#subset="validation",
batch_size=32,
seed=42,
shuffle=True,
class_mode="binary",
target_size=(32,64))

test_generator_CC=datagen_test.flow_from_dataframe(
dataframe=df,
directory= path + "cell_image/test/DIDA_1/",
x_col="Filename",
y_col="CC",
batch_size=1200,
seed=42,
shuffle=False,
class_mode="binary",
target_size=(32,64))

#%% TRANSFER LEARNING MODEL CC

pre_trained_model = tf.keras.applications.MobileNetV2(
    input_shape=(32, 64, 3), # refers to the shape we transfer from
    include_top=False, # cut off the head
    weights='imagenet', # pretrained on the ImageNet data
)
for layer in pre_trained_model.layers: #freeze the base model to not train it
    layer.trainable=False
pre_trained_model.summary()
#%%

model_CC = tf.keras.models.Sequential([
    pre_trained_model, # the pre-trained part
    tf.keras.layers.Flatten(), # flatten before fully connected part
    tf.keras.layers.Dense(64, activation='relu'),
    BatchNormalization(),
    Dropout(0.5),
    tf.keras.layers.Dense(2, activation='sigmoid'),
    ])
model_CC.compile(
    optimizer='adam',
    loss='sparse_categorical_crossentropy',
    metrics=['accuracy'],
    )
model_CC.summary()
#%%

from tensorflow.keras import layers
STEP_SIZE_TRAIN=train_generator_CC.n//train_generator_CC.batch_size
STEP_SIZE_VALID=valid_generator_CC.n//valid_generator_CC.batch_size
STEP_SIZE_TEST=test_generator_CC.n//test_generator_CC.batch_size

history = model_CC.fit_generator(generator=train_generator_CC,
                    steps_per_epoch=STEP_SIZE_TRAIN,
                    validation_data=valid_generator_CC,
                    validation_steps=STEP_SIZE_VALID,
                    epochs=1
)
#%% D
train_generator_D=datagen.flow_from_dataframe(
dataframe=df,
directory= path + "cell_image/train/DIDA_1/",
x_col="Filename",
y_col="D",
batch_size=200,
seed=42,
shuffle=True,
class_mode="sparse",
target_size=(32,64))

valid_generator_D=datagen_test.flow_from_dataframe(
dataframe=df,
directory= path + "cell_image/val/DIDA_1/",
x_col="Filename",
y_col="D",
batch_size=32,
seed=42,
shuffle=True,
class_mode="sparse",
target_size=(32,64))

test_generator_D=datagen_test.flow_from_dataframe(
dataframe=df,
directory= path + "cell_image/test/DIDA_1/",
x_col="Filename",
y_col="D",
batch_size=1200,
seed=42,
shuffle=False,
class_mode="sparse",
target_size=(32,64))

#%% TRANSFER LEARNING MODEL D

pre_trained_model = tf.keras.applications.MobileNetV2(
    input_shape=(32, 64, 3), # refers to the shape we transfer from
    include_top=False, # cut off the head
    weights='imagenet', # pretrained on the ImageNet data
)
for layer in pre_trained_model.layers: #freeze the base model to not train it
    layer.trainable=False
pre_trained_model.summary()
#%%
import tensorflow as tf
from tensorflow.keras.layers import Dropout, Flatten, Dense, BatchNormalization

model_D = tf.keras.models.Sequential([
    pre_trained_model, # the pre-trained part
    tf.keras.layers.Flatten(), # flatten before fully connected part
    tf.keras.layers.Dense(64, activation='relu'),
    BatchNormalization(),
    Dropout(0.5),
    tf.keras.layers.Dense(11, activation='softmax'),
    ])
model_D.compile(
    optimizer='adam',
    loss='sparse_categorical_crossentropy',
    metrics=['accuracy'],
    )
model_D.summary()
#%%

STEP_SIZE_TRAIN=train_generator_D.n//train_generator_D.batch_size
STEP_SIZE_VALID=valid_generator_D.n//valid_generator_D.batch_size
STEP_SIZE_TEST=test_generator_D.n//test_generator_D.batch_size

historyD = model_D.fit_generator(generator=train_generator_D,
                    steps_per_epoch=STEP_SIZE_TRAIN,
                    validation_data=valid_generator_D,
                    validation_steps=STEP_SIZE_VALID,
                    epochs=1
)

#%% Y
train_generator_Y=datagen.flow_from_dataframe(
dataframe=df,
directory= path + "cell_image/train/DIDA_1/",
x_col="Filename",
y_col="Y",
batch_size=200,
seed=42,
shuffle=True,
class_mode="sparse",
target_size=(32,64)
)

valid_generator_Y=datagen_test.flow_from_dataframe(
dataframe=df,
directory= path + "cell_image/val/DIDA_1/",
x_col="Filename",
y_col="Y",
batch_size=32,
seed=42,
shuffle=True,
class_mode="sparse",
target_size=(32,64)
)

test_generator_Y=datagen_test.flow_from_dataframe(
dataframe=df,
directory= path + "cell_image/test/DIDA_1/",
x_col="Filename",
y_col="Y",
batch_size=1200,
seed=42,
shuffle=False,
class_mode="sparse",
target_size=(32,64)
)
#%% TRANSFER LEARNING MODEL Y

pre_trained_model = tf.keras.applications.MobileNetV2(
    input_shape=(32, 64, 3), # refers to the shape we transfer from
    include_top=False, # cut off the head
    weights='imagenet', # pretrained on the ImageNet data
)
for layer in pre_trained_model.layers: #freeze the base model to not train it
    layer.trainable=False
pre_trained_model.summary()
#%%
import tensorflow as tf
from tensorflow.keras.layers import Dropout, Flatten, Dense, BatchNormalization

model_Y = tf.keras.models.Sequential([
    pre_trained_model, # the pre-trained part
    tf.keras.layers.Flatten(), # flatten before fully connected part
    tf.keras.layers.Dense(64, activation='relu'),
    BatchNormalization(),
    Dropout(0.5),
    tf.keras.layers.Dense(11, activation='softmax'),
    ])
model_Y.compile(
    optimizer='adam',
    loss='sparse_categorical_crossentropy',
    metrics=['accuracy'],
    )
model_Y.summary()
#%%

STEP_SIZE_TRAIN=train_generator_Y.n//train_generator_Y.batch_size
STEP_SIZE_VALID=valid_generator_Y.n//valid_generator_Y.batch_size
STEP_SIZE_TEST=test_generator_Y.n//test_generator_Y.batch_size

historyY = model_Y.fit(train_generator_Y,
                    steps_per_epoch=STEP_SIZE_TRAIN,
                    validation_data=valid_generator_Y,
                    validation_steps=STEP_SIZE_VALID,
                    epochs=1
)

#%%
import numpy as np
# FOR CC
X_test, y_test = next(test_generator_CC)
real_labels_CC = y_test
real_labels_CC=real_labels_CC.astype(int)
predictions = model_CC.predict(X_test)
print(predictions)
predict_labels_CC = np.argmax(predictions, axis=-1)
print('predict for CC',predict_labels_CC)

# FOR D
X_test, y_test = next(test_generator_D)
real_labels_D = y_test
real_labels_D=real_labels_D.astype(int)
predictions = model_D.predict(X_test)
print(predictions)
predict_labels_D = np.argmax(predictions, axis=-1)
print('predict for D',predict_labels_D)

# FOR Y
X_test, y_test = next(test_generator_Y)
real_labels_Y = y_test
real_labels_Y=real_labels_Y.astype(int)
predictions = model_Y.predict(X_test)
print(predictions)
predict_labels_Y = np.argmax(predictions, axis=-1)
print('predict for Y',predict_labels_Y)

#%%

p1=(predict_labels_CC==real_labels_CC)
p2=(predict_labels_D==real_labels_D)
p3=(predict_labels_Y==real_labels_Y)

CNN_p_CC = 0
for i in range(len(predict_labels_CC)):
    if p1[i] == True:
        CNN_p_CC = CNN_p_CC + 0.33
CNN_p_D = 0
for i in range(len(predict_labels_CC)):
    if p2[i] == True:
        CNN_p_D = CNN_p_D + 0.33
CNN_p_Y = 0
for i in range(len(predict_labels_CC)):
    if p3[i] == True:
        CNN_p_Y = CNN_p_Y + 0.3

CNN_p_sequence = 0
for i in range(len(p1)):
    if p1[i] == p2[i] == p3[i] == True:
       CNN_p_sequence = CNN_p_sequence + 1
#%%
print('point for CC-model:', round(CNN_p_CC))
print('point for D-model:', round(CNN_p_D))
print('point for Y-model:', round(CNN_p_Y))
print('point for character acc:', round(CNN_p_CC + CNN_p_D + CNN_p_Y), 'out of', len(real_labels_CC))
print('point for correct sequence:', CNN_p_sequence, 'out of', len(real_labels_CC))

#%% QUESTION 3, 2

'''
2. Investigate if your model’s performance is
particularly good or bad at correctly
classifying certain classes (i.e., it might be very good
at correctly classifying centuries
but not years, or it might be good at correctly classifying
some decades but not certain other decades). Does it mix up
certain classes? If yes, does this surprise you (explain why or why not)?
'''

print(real_labels_CC)
print(real_labels_D)
print(real_labels_Y)
print('predict for CC',predict_labels_CC)
print('predict for Y',predict_labels_Y)
print('predict for D',predict_labels_D)

#%%
n_list = []
for i in range(len(real_labels_CC)):
    if predict_labels_CC[i] != real_labels_CC[i]:
        n_list.append(predict_labels_CC[i])

print(n_list)
#%%
nn_list = []
for i in range(len(real_labels_D)):
    if predict_labels_D[i] != real_labels_D[i]:
        nn_list.append(predict_labels_D[i])

print(nn_list)

#%%
nnn_list = []
for i in range(len(real_labels_Y)):
    if predict_labels_Y[i] != real_labels_Y[i]:
        nnn_list.append(predict_labels_Y[i])

print(nnn_list)

#%%
from collections import Counter
print(Counter(n_list))
print(Counter(nn_list))
print(Counter(nnn_list))


import numpy as np
import pandas as pd
import sys, os, click, glob, tqdm, shutil

@click.command()
@click.option(
    "-u",
    "--run_id",
    "run_id",
    required=True,
    envvar="COVID_RUN_INDEX",
    help="run_id_to convert from",
)
@click.option(
    "-f",
    "--input_folder",
    "input_folder",
    required=True,
    help="Folder where the initial run is synced. There should be a model_ouput in it",
)
def cli(run_id, input_folder):
    return run_id, input_folder

# https://stackoverflow.com/questions/2793789/create-destination-path-for-shutil-copy-files
def copy_path(*, src, dst, dir_mode=0o777, follow_symlinks: bool = True):
    """
    Copy a source filesystem path to a destination path, creating parent
    directories if they don't exist.

    Args:
        src: The source filesystem path to copy. This must exist on the
            filesystem.

        dst: The destination to copy to. If the parent directories for this
            path do not exist, we will create them.

        dir_mode: The Unix permissions to set for any newly created
            directories.

        follow_symlinks: Whether to follow symlinks during the copy.

    Returns:
        Returns the destination path.
    """
    try:
        return shutil.copy2(src=src, dst=dst, follow_symlinks=follow_symlinks)
    except FileNotFoundError as exc:
        if exc.filename == dst and exc.filename2 is None:
            parent = os.path.dirname(dst)
            os.makedirs(name=parent, mode=dir_mode, exist_ok=True)
            return shutil.copy2(
                src=src,
                dst=dst,
                follow_symlinks=follow_symlinks,
            )
        raise

def detect_old_run_id(fp):
    return fp.split("/inference/med/")[-1].split("/")[0]

if __name__ == "__main__":
    # standalone_mode: so click doesn't exit, see
    # https://stackoverflow.com/questions/60319832/how-to-continue-execution-of-python-script-after-evaluating-a-click-cli-function
    run_id, input_folder = cli(standalone_mode=False)
    print(run_id, input_folder)
    if not os.path.isdir(f"{input_folder}/model_output"):
        raise ValueError(
            f"Could not find folder {input_folder}/model_output, are you sure -f option is set correctly ?"
        )
    files = glob.glob(input_folder + "/**/*.parquet", recursive=True)
    os.makedirs("model_output/cont", exist_ok=True)

    fn = files[0]
    old_run_id = detect_old_run_id(fn)
    new_name = (
        fn.replace("seir", "cont").replace(f"{input_folder}/model_output", "model_output").replace(old_run_id, run_id)
    )

    print(f"detected old_run_id: {old_run_id} which will be replaced by user provided run_id: {run_id}")
    empty_str = "°" * len(input_folder)
    print(f"file: \n OLD NAME: {fn}\n NEW NAME: {empty_str}{new_name}")
    for fn in tqdm.tqdm(files):
        old_run_id = detect_old_run_id(fn)
        new_name = (
            fn.replace("seir", "cont")
            .replace(f"{input_folder}/model_output", "model_output")
            .replace(old_run_id, run_id)
        )
        copy_path(src=fn, dst=new_name)

    # os.makedirs(outdir, exist_ok=True)


with open("amh.tsv", encoding = "utf-8") as f:
    f_line = f.readline()
    f_notline = f.read()
l = []
d = {}
for char in f_line:
    if char != "\t" and char != "\n":
        l.append(char)  #получаем лист с гласными
char = f_notline.split() #все символы файла
for i, sym in enumerate(char):
    if i == 0:
        seq_num = i #записываем первую согласную
    elif (i - seq_num) // 8:
        seq_num = i #записываем согласные
    else:
        d[sym] = char[seq_num] + l[i - 1 - seq_num] #потому что гласные смещены на одну
print(d)

with open("ch_am.txt", encoding = "utf-8") as g:
    g = g.read()
    for char in g:
        if char in d:
            char = d[char]
    print(g)


'''
Define game and start execution of game search

Author: Tony Lindgren
'''
from four_in_a_row import FourInARow
from game_node_and_game_search import GameSearch

def ask_ai(state0):
    # gs = GameSearch(state0, depth=3, time=1)
    # move = gs.minimax_search()
    gs = GameSearch(state0, depth=3, time=2)
    move = gs.mcts()
    print("move...", move)
    state1 = state0.result(move)

    # print('--------')
    # print(move, state0.board, state1.board)

    print('--------')
    print('AI moves')
    print('--------')
    state1.pretty_print()
    stop, value = state1.is_terminal()
    if stop == True:
        if value > 0:
            print('AI won')
        elif value < 0:
            print('Human won')
        else:
            print('Draw')
        return state1, True
    return state1, False

def ask_player(state):
    # Tdo: Invalid inputs like >,'str' etc not be accepted
    stop = False
    while not stop:
        try:
            action = int(input('Enter valid empty column from 0 to 6: ') )
        except ValueError:
            print("invalid input")
            continue

        if -1 < action <7:
            if len(state.board[action]) < 6:
                stop = True
                new_state = state.result(action)
                print('--------')
                print('Player moves')
                print('--------')
                new_state.pretty_print()
                stop2, value2 = new_state.is_terminal()
                if stop2 == True:
                    print('Human won')
                    return new_state, True

    return new_state, False # win check is done by ai after player makes move

def main():
    print('Welcome to play for-in-a-row!')
    answer = None
    while answer != 'y' and answer != 'n':
        answer = input('Would you like to start [y/n]: ')
    if(answer == 'y'):
        state0 = FourInARow('human', 'w')  # class state
        stop = False
        while not stop:
        #Ask player
            state1, stop1 = ask_player(state0)
            if stop1:
                break
            else:
        #AI move
                state0, stop2 = ask_ai(state1)
                if stop2:
                    break
    else:
        state0 = FourInARow('ai','w')
        stop = False
        while not stop:
        #AI move
            state1, stop1 = ask_ai(state0)
            if stop1:
                break
            else:
        #Ask player
                state0, stop2 = ask_player(state1)
                if stop2:
                    break

if __name__ == "__main__":
    main()

Game_CSP Problems/Four in a row/run_assignment2.py


# pylint:disable=missing-class-docstring
import angr

from ...sim_type import SimStruct, SimTypeLong, SimTypeFixedSizeArray, SimTypeShort, SimTypeInt

# struct sysinfo {
#    long uptime;             /* Seconds since boot */
#    unsigned long loads[3];  /* 1, 5, and 15 minute load averages */
#    unsigned long totalram;  /* Total usable main memory size */
#    unsigned long freeram;   /* Available memory size */
#    unsigned long sharedram; /* Amount of shared memory */
#    unsigned long bufferram; /* Memory used by buffers */
#    unsigned long totalswap; /* Total swap space size */
#    unsigned long freeswap;  /* Swap space still available */
#    unsigned short procs;    /* Number of current processes */
#    unsigned long totalhigh; /* Total high memory size */
#    unsigned long freehigh;  /* Available high memory size */
#    unsigned int mem_unit;   /* Memory unit size in bytes */
# }
sysinfo_ty = SimStruct(
    {
        "uptime": SimTypeLong(signed=True),
        "loads": SimTypeFixedSizeArray(SimTypeLong(signed=False), 3),
        "totalram": SimTypeLong(signed=False),
        "freeram": SimTypeLong(signed=False),
        "sharedram": SimTypeLong(signed=False),
        "bufferram": SimTypeLong(signed=False),
        "totalswap": SimTypeLong(signed=False),
        "freeswap": SimTypeLong(signed=False),
        "procs": SimTypeShort(signed=False),
        "totalhigh": SimTypeLong(signed=False),
        "freehigh": SimTypeLong(signed=False),
        "mem_unit": SimTypeInt(signed=False),
    },
    name="sysinfo",
    pack=False,
    align=None,
)

class sysinfo(angr.SimProcedure):
    def run(self, info):  # pylint: disable=arguments-differ
        value = {
            "uptime": 1234567,
            "loads": [20100, 22000, 15000],
            "totalram": 1024**2,
            "freeram": 1024**2 // 4,
            "sharedram": 1024**2 // 4,
            "bufferram": 1024**2 // 4,
            "totalswap": 1024**2,
            "freeswap": 1024**2 // 2,
            "procs": 533,
            "totalhigh": 11,
            "freehigh": 12,
            "mem_unit": 13,
        }
        sysinfo_ty.with_arch(self.arch).store(self.state, info, value)
        return 0


from flask import url_for
from flask_testing import TestCase
from application import app, db
from application.models import users, games

class TestBase(TestCase):
    def create_app(self):
        app.config.update(
            SQLALCHEMY_DATABASE_URI = 'sqlite:///test.sqlite3',
            SECRET_KEY = 'testsecretkey',
            DEBUG_MODE = True,
            WTF_CSRF_ENABLED = False
        )
        return app

    def setUp(self):
        db.create_all()
        test_user = users(name="Andrei", email="andrei.chirovici@gmail.com")
        db.session.add(test_user)
        db.session.commit()

def tearDown(self):
        db.session.remove()
        db.drop_all()

class TestHome(TestBase):
    def test_home_get(self):
        response = self.client.get(url_for('home'))
        self.assert200(response)

class TestAddUser(TestBase):
    def test_register_page(self):
        response = self.client.get(url_for('register'))
        self.assert200(response)

    def test_register_user(self):
        response = self.client.post(
            url_for('register'),
            data = dict(name="Andrei", email="andrei.chirovici@gmail.com"),
            follow_redirects = True
        )
        self.assert200(response)


import os

'''
This class regroup either blacklist and whitelist
'''
class Userlist:
    def __init__(self, whitelist, path):
        self.whitelist = whitelist
        self.path = path

        if os.path.isfile(path):
            return path
        else:
            print("[ERR]:", "Whitelist" if self.whitelist else "Blacklist", "file path not valid")


import requests
from bs4 import BeautifulSoup
import csv
import re
import string
import datetime

def get_url(result):
	for title in result.find_all('h3', class_="r"):
		for link in title.find_all('a'):
			url = link.get('href')
	return url

def get_title(result):
	for heading in result.find_all('h3', class_="r"):
		for link in heading.find_all('a'):
			title = link.text
			#title = str(title)
			title = title.encode("utf-8")
	return title

def get_description(result):
	description = result.find_all('span', class_="st")
	description = re.sub('<[^<]+?>','',str(description))
	#description = description.encode("utf-8")
	return description

prod_list_file = '/Users/ascodel/Google Drive/Google Product Scrape/prod_list.csv'

with open(prod_list_file, 'rb') as f:
	reader = csv.reader(f)
	#prod_cats = list(reader)
	prod_cats = map(tuple, reader)

date = datetime.date.today()

terms = []

for product in prod_cats:
	product_search = []

	for word in product:
		word = str(word)
		product_words = word.split()

		for word in product_words:
			word = word + "+"
			product_search.append(word)

	term = ''.join(product_search)
	term_1 = term + "federal+requirement"
	terms.append(term_1)
	#term_2 = term + "federal+purchasing"
	#terms.append(term_2)
	#term_3 = term +"FEMP"
	#terms.append(term_3)

output_file = '/Users/ascodel/Google Drive/SFO Group Files/FEMP EEPP/google_search_results_' + str(date) + '.csv'
dictionary = {'term': '', 'url': '', 'title': '', 'description':'','date': ''}
'''
term = 'femp'
r = requests.get("https://www.google.com/search?q=" + term)
soup = BeautifulSoup(r.text)
for result in soup.find_all('div',class_='g'):
	print result
'''

for term in terms:
	r = requests.get("https://www.google.com/search?q=" + term)
	soup = BeautifulSoup(r.text)

	#print term
	#url = "http://www.google.com/" + term
	#print url

	for result in soup.find_all('div', class_='g'):
		dictionary['date'] = date
		dictionary['term'] = term
		dictionary['url'] = get_url(result)
		dictionary['title'] = get_title(result)
		dictionary['description'] = get_description(result)

		with open(output_file, 'a') as csvfile:
			writer = csv.writer(csvfile)
			writer.writerow(dictionary.values())

		#with open(output_file, 'a') as csvfile:
		#	writer = csv.writer(csvfile)
		#	writer.writerow(dictionary.values())
			#writer = UnicodeWriter(csvfile)
			#writer.writerow(dictionary.values())
'''

	for result in soup.find_all('li', class_="_g"):
		dictionary['date'] = date
		dictionary['term'] = term
		dictionary['url'] = get_url(result)
		dictionary['title'] = get_title(result)
		dictionary['description'] = get_description(result)

		with open(output_file, 'a') as csvfile:
			writer = csv.writer(csvfile)
			writer.writerow(dictionary.values())

#print terms
	#product_words = str(product)
	#print product_words
	#product_words = product_words.split()
	#print product_words

	#product_search = []

	#for word in product_words:
		#print word
	#	word = word + "+"
	#	#print word
	#	product_search.append(word)

		#print product_search
		#print ''.join(product_search)
	#terms.append(''.join(product_search))
'''


import numpy as np

from matplotlib import pyplot as plt
from scipy.optimize import curve_fit, root_scalar

def linear_fit(x, m, h):
    return m * x + h

data = np.loadtxt("exercises/scipy/data.txt")

x, linear, exponential = data.T

linear_params, linear_cov = curve_fit(linear_fit, x, linear)
log_params, log_cov = curve_fit(linear_fit, x, np.log(exponential))

def root_func(x):
    return linear_fit(x, *linear_params) - np.exp(linear_fit(x, *log_params))

res = root_scalar(root_func, x0=3.5, x1=4)

plt.plot(x, linear,
         linestyle="none",
         marker="o",
         label="Linear data")

plt.plot(x, exponential,
         linestyle="none",
         marker="s",
         label="Exponential data")

plt.plot(x, linear_fit(x, *linear_params),
         label="Linear fit")

plt.plot(x, np.exp(linear_fit(x, *log_params)),
         label="Exponential fit")

plt.annotate("Intersection",
             xy=[res.root, linear_fit(res.root, *linear_params)],
             xytext=[2, 8],
             arrowprops=dict(facecolor='black'))

plt.legend()
plt.show()


from selenium import webdriver
from selenium.webdriver.common.by import By

browser = webdriver.Chrome()
url = 'https://spa2.scrape.center'
browser.get(url)
logo = browser.find_element(By.CLASS_NAME, 'logo-image')
print(logo)
print(logo.get_attribute('src'))

JavaScript动态渲染页面爬取/Selenium/获取节点信息/获取属性信息.py


#!/usr/bin/env python
import vtk

# Enable printing for debugging
debugPrint = 0

# Test the ability to pack segmentation labels into
# contiguous runs. This is useful for discrete isocontouring
# filters such as vtkSurfaceNets3D, vtkDiscreteMarchingCubes,
# and vtkDiscreteFlyingEdges.

# Create a small volume with non-packed voxel values. The output
# data should be packed into unsigned char since there are less
# than 256 labels.
VTK_UCHAR = 3
VTK_INT = 6
xDim = 3
yDim = 3
zDim = 3

numVoxels = xDim * yDim * zDim
image = vtk.vtkImageData()
image.SetDimensions(xDim,yDim,zDim)
image.AllocateScalars(VTK_INT,1)

scalars = image.GetPointData().GetScalars()

# Populate the volume. Create a weird ordering,
# with label gaps and non-contiguous values.
for i in range(0,numVoxels,3):
    idx = numVoxels - i - 1;
    val = 100 + idx
    scalars.SetTuple1(idx,val)
    scalars.SetTuple1(idx-1,val)
    scalars.SetTuple1(idx-2,val)

scalars.SetTuple1(26,0) # make non-contiguous and non-increasing
scalars.SetTuple1(25,0)
scalars.SetTuple1(24,0)

# Print out data as constructed
if debugPrint:
    print("Input to pack labels:")
    for i in range(0,numVoxels):
        print(i,scalars.GetTuple1(i))

# Pack the labels. This should automatically pack into unsigned char.
pack = vtk.vtkPackLabels()
pack.SetInputData(image)
pack.Update()

# See what kind of label we have
labels = pack.GetLabels()
numLabels = labels.GetNumberOfTuples()
if debugPrint:
    print("\nNumber of resulting labels: ",labels.GetNumberOfTuples())
    for i in range(0,numLabels):
        print("Label: ", labels.GetTuple1(i))

# Look at the resulting volume
outScalars = pack.GetOutput().GetPointData().GetScalars()
dataType = outScalars.GetDataType()
if debugPrint:
    print("\nPoint scalars: ")
    for i in range(0,numVoxels):
        print(i,outScalars.GetTuple1(i))

# Now check for proper output
assert(dataType == VTK_UCHAR)
assert(numLabels == 9)
assert(labels.GetTuple1(0) == 0)
assert(labels.GetTuple1(numLabels-1) == 123)
assert(outScalars.GetTuple1(0) == 1)
assert(outScalars.GetTuple1(numVoxels-1) == 0)

# Now test sorting of packed labels by frequency count,
# and make sure that packing works for cell data as well.
numVoxelCells = (xDim-1) * (yDim-1) * (zDim-1)

cellScalars = vtk.vtkShortArray()
cellScalars.SetName("cellScalars")
cellScalars.SetNumberOfTuples(numVoxelCells)
cellScalars.Fill(0)
cellScalars.SetTuple1(7,1)
cellScalars.SetTuple1(6,2)
cellScalars.SetTuple1(5,2)
cellScalars.SetTuple1(3,3)
cellScalars.SetTuple1(2,3)
cellScalars.SetTuple1(1,3)

image.GetCellData().SetScalars(cellScalars)

if debugPrint:
    print("\nCell scalars input to pack labels:")
    for i in range(0,numVoxelCells):
        print(i,cellScalars.GetTuple1(i))

pack.SetInputArrayToProcess(0,0,0,vtk.vtkDataObject.FIELD_ASSOCIATION_CELLS,"cellScalars")
pack.SortByLabelCount()
pack.Update()
outCellScalars = pack.GetOutput().GetCellData().GetScalars()
dataType = outCellScalars.GetDataType()

labels = pack.GetLabels()
counts = pack.GetLabelsCount()
numLabels = labels.GetNumberOfTuples()

if debugPrint:
    print("\nNumber of sorted cell labels with counts: ",labels.GetNumberOfTuples())
    for i in range(0,numLabels):
        print("Label: ", labels.GetTuple1(i), ", count: ", counts.GetTuple1(i))

if debugPrint:
    print("\nOutput cell scalars:")
    for i in range(0,numVoxelCells):
        print(i,outCellScalars.GetTuple1(i))

assert(dataType == VTK_UCHAR)
assert(numLabels == 4)
assert(counts.GetTuple1(0) == 3)
assert(counts.GetTuple1(3) == 1)
assert(labels.GetTuple1(0) == 3)
assert(labels.GetTuple1(3) == 1)


"""
Given the head of a sorted linked list, delete all duplicates such that
each element appears only once. Return the linked list sorted as well

Example 1:

Input: head = [1,1,2]
Output: [1,2]
Example 2:

Input: head = [1,1,2,3,3]
Output: [1,2,3]
"""

# Definition for singly-linked list.
# class ListNode:
#     def __init__(self, val=0, next=None):
#         self.val = val
#         self.next = next

class Solution:
    def deleteDuplicates(self, head: ListNode) -> ListNode:
        l = []
        while (head):
            l.append(head.val)
            head = head.next
        l = list(set(l))
        l.sort()
        i = ListNode()
        temp = i
        for j in l:
            i.next = ListNode(j)
            i = i.next
        return temp.next

result = Solution()

head = [-1,0,0,0,0,3,3]

print(result.deleteDuplicates(head))


obrazki = []
nierek=0
pierw = True
with open('dane_obrazki.txt') as plik:
    for i in range(200):
        obrazek = []

        for j in range(21):
            obrazek.append(plik.readline().strip())
        plik.readline()
        obrazki.append(obrazek)

for obrazek in obrazki:

    for i in range(10):
        for j in range(10):
            if obrazek[i][j] != obrazek[i+10][j] or obrazek[i][j] != obrazek[i][j+10] or obrazek[i][j] != obrazek[i+10][j+10]:
                break
        else:
            if pierw == True:
                for k in range(20):
                    for l in  range(20):
                        print(obrazek[k][l], end='')
                    print()
                pierw = False
            continue
        nierek += 1
        break

print(200-nierek)


from setuptools import setup

with open('README.md', 'r') as f:
    long_description = f.read()

setup(
  name='nu',
  version='0.2.0',
  packages=['nu'],
  author='Nyan Of The Moon',
  author_email='nyanofthemoon@gmail.com',
  description='Framework for creating Cozmo skills using multiple sensory inputs.',
  long_description=long_description,
  url='https://hotchiwawa.com',
  license='MIT',
  entry_points={
      'console_scripts': [
          'nu = nu.__main__:main'
      ]
  },
  install_requires=[
      'argparse',
      'configparser',
      'redis',
      'hiredis',
      'sense-hat',
      'cozmo[camera]',
      'SpeechRecognition',
      'PyAudio',
      'PocketSphinx',
      'python-Levenshtein',
      'google-api-python-client',
      'textblob',
      'flask',
      'pyopenssl'
  ],
  zip_safe=False
)


import copy
import itertools

def count_live_neighbors(w, x, y, z, layout):
    return sum(live(w+dw,x+dx,y+dy,z+dz,layout) if (dw != 0 or dx != 0 or dy != 0 or dz != 0) else 0 for dw,dx,dy,dz in itertools.product([-1,0,1], repeat=4))

def live(w, x,y,z,layout):
    if w >= len(layout) or w < 0:
        return 0
    if x >= len(layout[w]) or x < 0:
        return 0
    if y >= len(layout[w][x]) or y < 0:
        return 0
    if z >= len(layout[w][x][y]) or z < 0:
        return 0
    return 1 if layout[w][x][y][z] == '#' else 0

def prettyprint(layout):
    for layer in layout:
        print("LAYER")
        for sublayer in layer:
            print("SUBLAYER")
            for string in sublayer:
                print(string)

with open("input.txt") as inp:
    layout = [[[line.strip() for line in inp.readlines()]]]

for n in range(6):
    print(n)
    #prettyprint(layout)
    empty_layer = []
    for _ in range(len(layout)):
        empty_layer.append(["."*len(layout[0][0][0])]*len(layout[0][0]))
    layout.insert(0,copy.deepcopy(empty_layer))
    layout.append(copy.deepcopy(empty_layer))
    for layer in layout:
        layer.insert(0,copy.deepcopy(empty_layer[0]))
        layer.append(copy.deepcopy(empty_layer[0]))
    for layer in layout:
        for sublayer in layer:
            sublayer.insert(0,copy.deepcopy(empty_layer[0][0]))
            sublayer.append(copy.deepcopy(empty_layer[0][0]))
    for i,layer in enumerate(layout):
        for j,sublayer in enumerate(layer):
            for k,row in enumerate(sublayer):
                layout[i][j][k] = '.' + row + '.'

    new_layout = copy.deepcopy(layout)
    #prettyprint(layout)
    for i,w in enumerate(new_layout):
        #print("w {}", i)
        for j,x in enumerate(w):
            #print ("x {}", j)
            for k,y in enumerate(x):
                #print ("y {}", k)
                for l,z in enumerate(y):
                    neighbors = count_live_neighbors(i,j,k,l,layout)
                    if z == '#' and neighbors != 2 and neighbors != 3:
                        new_layout[i][j][k] = new_layout[i][j][k][:l] + '.' + new_layout[i][j][k][l+1:]
                    elif z != '#' and neighbors == 3:
                        new_layout[i][j][k] = new_layout[i][j][k][:l] + '#' + new_layout[i][j][k][l+1:]

    layout = new_layout

print(sum(sum(sum(y.count('#') for y in x) for x in w) for w in layout))


#coding: utf-8

import argparse
from readDataset import readDataset
from individual  import Individual
from population  import Population
from select      import Select
from crossover   import Crossover
from mutation    import Mutation
import six
if (six.PY2):
    from ga4py2  import GA
if (six.PY3):
    from ga4py3  import GA
from copy        import deepcopy

class Onemax(GA):
    def evaluate(self, ind, args):
        fitness = sum(ind)
        return (fitness)

    def revolution(self, args):
        #config
        if   (args.crossmode=="one"):
            cross_func = Crossover.onePoint
        elif (args.crossmode=="two"):
            cross_func = Crossover.twoPoints
        else:
            cross_func = Crossover.randomPoints

        # Prepare dataset
        dataset = readDataset(args.dpath)

        # Create individuals & population
        ppl = Population()
        for i in range(args.individuals):
            individual = Individual()
            individual.createGene(dataset, args.gene)
            ppl.addInd(individual)

        # Evolution
        for i in range(args.revolution):
            # Evaluation population in individuals
            ppl.calcFitness(self.evaluate)
            if ((i%10)==0):
                ppl.show()

            # Select parents
            if (args.elite):
                parents = Select.Elite(ppl, args.esize, args.mode)
                parents.extend(Select.Tournament(ppl, args.individuals-args.esize, args.tornsize, args.mode))
            else:
                parents = Select.Tournament(ppl, args.individuals, args.tornsize, args.mode)

            # Clossover
            children = cross_func(parents, args.individuals, args.gene, args.crossrate)

            # Mutation
            children = Mutation.mutation(children, args.mutaterate, dataset)
            # Swap children
            ppl.setPpl(children)

        # show result
        ppl.calcFitness(self.evaluate)
        ppl.show()

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("-d",        "--dpath",       type=str, default="./dataset/binary.txt", help="dataset path")
    parser.add_argument("-ind",      "--individuals", type=int, default=50,                     help="Number of individuals")
    parser.add_argument("-gene",     "--gene",        type=int, default=10,                     help="Length of gene")
    parser.add_argument("-revo" ,    "--revolution",  type=int, default=100,                    help="Number of revolution")
    parser.add_argument("-elite",    "--elite",                 default=False,                  help="Use elite selection", action="store_true")
    parser.add_argument("-esize",    "--esize",       type=int, default=2,                      help="Size of elite")
    parser.add_argument("-tornsize", "--tornsize",    type=int, default=3,                      help="Size of tournament")
    parser.add_argument("-mode",     "--mode",        type=str, default="max",                  help="Max or min")
    parser.add_argument("-crate",    "--crossrate",   type=float , default=0.7,                 help="probability of crossover")
    parser.add_argument("-cmode",    "--crossmode",   type=str, default="one",                  help="one or two or random")
    parser.add_argument("-mrate",    "--mutaterate",  type=float,  default=0.05,                 help="probability of mutation")
    args   = parser.parse_args()
    print("[argments list]")
    print(args)
    onemax = Onemax()
    onemax.revolution(args)


import os
import shutil
import subprocess as sp
import tempfile as tp
import random
import re
import pandas as pd
from itertools import cycle
from functools import reduce
from contextlib import contextmanager
from circmimi.seq import Seq

class Miranda:
    RESULT_TITLE = (
        'query_id',
        'reference_id',
        'score',
        'energy',
        'query_start',
        'query_end',
        'ref_start',
        'ref_end',
        'aln_length',
        'identity',
        'similarity',
        'aln_mirna',
        'aln_map',
        'aln_utr'
    )

    def __init__(self, ref_file, work_dir='.',
                 bin_path='miranda', options=None):
        self.ref_file = ref_file
        self.work_dir = work_dir
        self.bin_path = bin_path

        if options:
            self.options = options
        else:
            self.options = []
        self.options += ['-keyval']
        self.options = list(map(str, self.options))

    def _generate_cmd(self, seq_file, out_file=None):
        cmd = [self.bin_path, self.ref_file, seq_file] + self.options

        if out_file:
            cmd += ['-out', out_file]

        return cmd

    @staticmethod
    def _rand_id(n=5):
        return '{{:0>{}}}'.format(n).format(random.randrange(10**n))

    @contextmanager
    def _split_file(self, seq_file, num):
        tmp_dir = os.path.join(
            self.work_dir,
            'miranda_tmp_{}'.format(self._rand_id())
        )
        os.makedirs(tmp_dir)

        filenames = ['seq_file.part{}.fa'.format(i) for i in range(1, num + 1)]
        file_paths = [os.path.join(tmp_dir, fname) for fname in filenames]
        tmp_files = [open(path, 'w') for path in file_paths]

        tmp_files_iter = cycle(tmp_files)
        with open(seq_file) as seq_in:
            for line in seq_in:
                if line.startswith('>'):
                    tmp_f = next(tmp_files_iter)
                tmp_f.write(line)

        for tmp_f in tmp_files:
            tmp_f.close()

        try:
            yield file_paths
        finally:
            shutil.rmtree(tmp_dir)

    def run(self, seq_file, num_proc=1):
        if num_proc == 1:
            cmd = self._generate_cmd(seq_file)
            res = sp.run(cmd, stdout=sp.PIPE, encoding='utf-8')
            return res.stdout
        elif num_proc > 1:
            with self._split_file(seq_file, num=num_proc) as tmp_files:
                out_files = ["{}.result".format(tmp_f) for tmp_f in tmp_files]

                cmds = [self._generate_cmd(tmp_f, out_f)
                        for tmp_f, out_f in zip(tmp_files, out_files)]
                procs = [sp.Popen(cmd) for cmd in cmds]

                for p in procs:
                    p.wait()

                result = ''
                for out_f in out_files:
                    with open(out_f) as res_in:
                        result += res_in.read()

            return result

    @staticmethod
    def _get_value(res_line):
        return [kv.split('=')[1] for kv in res_line.split('\t')]

    @classmethod
    def parse_result(cls, raw_result):
        aln_res_pat = re.compile(r'^//hit_info\t(.*)', flags=re.M)
        for m in re.finditer(aln_res_pat, raw_result):
            yield cls._get_value(m.group(1))

def get_binding_sites(seq_df,
                      mir_ref_file,
                      work_dir='.',
                      num_proc=1,
                      miranda_options=None):

    if miranda_options is None:
        miranda_options = []

    miranda = Miranda(
        mir_ref_file,
        work_dir=work_dir,
        options=['-quiet'] + miranda_options
    )

    with tp.NamedTemporaryFile(dir=work_dir) as tmp_fa_file:
        with open(tmp_fa_file.name, 'w') as fa_out:
            fa_txt = Seq.to_fasta(seq_df)
            fa_out.write(fa_txt)

        raw_result = miranda.run(tmp_fa_file.name, num_proc=num_proc)
        miranda_df = pd.DataFrame(
            Miranda.parse_result(raw_result),
            columns=Miranda.RESULT_TITLE
        ).astype({
            'score': 'float',
            'energy': 'float',
            'query_start': 'int',
            'query_end': 'int',
            'ref_start': 'int',
            'ref_end': 'int',
            'aln_length': 'int',
            'identity': 'float',
            'similarity': 'float'
        })

        return miranda_df

class MirandaUtils:
    @staticmethod
    def append_exons_len(miranda_df, exons_len_df):
        miranda_df_with_total_len = miranda_df.merge(
            exons_len_df,
            left_on='reference_id',
            right_on='exons_id',
            how='left'
        ).drop('exons_id', axis=1)

        return miranda_df_with_total_len

    @classmethod
    def remove_redundant_result(cls, miranda_df):
        filtered_df = miranda_df.query('ref_start <= total_len')

        dropped_index = filtered_df.query(
            'ref_start == 1'
        ).assign(
            ext_ref_end=lambda df: df['ref_end'] + df['total_len']
        ).loc[:, [
            'query_id',
            'reference_id',
            'ext_ref_end'
        ]].reset_index(
        ).merge(
            filtered_df,
            left_on=['query_id', 'reference_id', 'ext_ref_end'],
            right_on=['query_id', 'reference_id', 'ref_end'],
            how='inner'
        ).loc[:, 'index']

        filtered_df = filtered_df.drop(dropped_index).reset_index(drop=True)

        return filtered_df

    @staticmethod
    def _is_cross_boundary(s):
        exons_len = s['total_len']
        ref_start = s['ref_start']
        ref_end = s['ref_end']

        if (ref_start <= exons_len) and (exons_len < ref_end):
            return '1'
        else:
            return '0'

    @classmethod
    def append_cross_boundary(cls, miranda_df_with_len):
        if miranda_df_with_len.empty:
            cross_boundary_res = pd.Series()
        else:
            cross_boundary_res = miranda_df_with_len.apply(
                cls._is_cross_boundary,
                axis=1
            )

        appended_res_df = miranda_df_with_len.assign(
            cross_boundary=cross_boundary_res
        )

        return appended_res_df

    @staticmethod
    def append_ev_id(df, exons_ev_id_df):
        df_with_ev_id = df.merge(
            exons_ev_id_df,
            left_on='reference_id',
            right_on='exons_id',
            how='left'
        ).drop(
            columns='exons_id'
        ).sort_values(
            ['ev_id', 'query_id', 'score'],
            ascending=[True, True, False]
        )

        return df_with_ev_id

    @staticmethod
    def append_merged_aln(miranda_df):
        miranda_df_with_merged_aln = miranda_df.assign(
            aln=lambda df: df['aln_mirna'] + df['aln_map'] + df['aln_utr']
        )

        return miranda_df_with_merged_aln

    @classmethod
    def generate_uniq_id(cls, miranda_df, column_name):
        all_uniq_items = pd.Series(
            miranda_df[column_name].unique(),
            name=column_name
        ).reset_index(
        ).astype(
            {
                'index': str
            }
        ).rename(
            {
                'index': f'{column_name}_id'
            },
            axis=1
        )

        miranda_df_with_id = miranda_df.merge(
            all_uniq_items,
            on=column_name,
            how='left'
        )

        return miranda_df_with_id

    @staticmethod
    def get_grouped_results(miranda_df_with_aln_id, with_AGO=False):
        if with_AGO:
            grouped_res_df = miranda_df_with_aln_id[[
                'ev_id',
                'query_id',
                'score',
                'aln_id',
                'cross_boundary',
                'AGO_support',
                'AGO_support_yn'
            ]].drop_duplicates(
            ).groupby([
                'ev_id',
                'query_id'
            ]).agg({
                'score': 'max',
                'aln_id': 'nunique',
                'cross_boundary': 'max',
                'AGO_support': 'max',
                'AGO_support_yn': 'sum'
            }).reset_index(
            ).rename(
                {
                    'query_id': 'mirna',
                    'score': 'max_score',
                    'aln_id': 'num_binding_sites',
                    'AGO_support': 'MaxAgoExpNum',
                    'AGO_support_yn': 'num_AGO_supported_binding_sites'
                },
                axis=1
            ).astype(
                {
                    'ev_id': 'object',
                    'num_binding_sites': 'Int64',
                    'MaxAgoExpNum': 'Int64',
                    'num_AGO_supported_binding_sites': 'Int64'
                }
            )
        else:
            grouped_res_df = miranda_df_with_aln_id[[
                'ev_id',
                'query_id',
                'score',
                'aln_id',
                'cross_boundary'
            ]].drop_duplicates(
            ).groupby([
                'ev_id',
                'query_id'
            ]).agg({
                'score': 'max',
                'aln_id': 'nunique',
                'cross_boundary': 'max'
            }).reset_index(
            ).rename(
                {
                    'query_id': 'mirna',
                    'score': 'max_score',
                    'aln_id': 'num_binding_sites'
                },
                axis=1
            ).astype(
                {
                    'ev_id': 'object',
                    'num_binding_sites': 'Int64'
                }
            )

        return grouped_res_df

    @staticmethod
    def get_genomic_position(miranda_df, pos_map_db):
        genomic_regions = miranda_df.apply(
            lambda s: pos_map_db[s['reference_id']].get_real_blocks(
                s['ref_start'],
                s['ref_end']
            ),
            axis=1
        )

        if genomic_regions.empty:
            miranda_df_with_genomic_position = miranda_df.assign(
                genomic_regions='NA'
            )
        else:
            miranda_df_with_genomic_position = miranda_df.assign(
                genomic_regions=genomic_regions
            )

        return miranda_df_with_genomic_position


import timeit

setup = '''
inputs = [i for i in range(100000)]
power_lambda = lambda a,b: a*b
def power(a,b):
    return a*b
'''

first_method = '''
for i in inputs:
    power_lambda(1,4)
'''

second_method = '''
for i in inputs:
    power(1,4)
'''

first_method_del = '''
inputs.pop(0)
'''

second_method_del = '''
del inputs[0]
'''

iterations = 100000
time1 = timeit.timeit(first_method_del, setup=setup, number=iterations)
time2 = timeit.timeit(second_method_del, setup=setup, number=iterations)

print(f'Difference: {((time2-time1)/time2)*100}%')


import requests
import sys
sys.path.extend(['./app/db'])

import schemas_pydantic
host = 'localhost'
port = '13371'

def authenticate(user,pw):
    token_response = requests.post(f"http://{host}:{port}/token", data={"username": user, "password": pw, "grant_type": "password"},
                                   headers={"content-type": "application/x-www-form-urlencoded"})

    token_response = token_response.json()
    token = token_response['access_token']

    auth_header = {'Authorization': 'Bearer {}'.format(token)}

    return auth_header

auth_header = authenticate("kit", "KIT_huipuischui_23")

#test chemical
dmc = schemas_pydantic.Chemical(smiles='COC(=O)OC', name='DMC', reference='DMC_Elyte_2020')
ans_ = requests.post(f"http://{host}:{port}/api/broker/post/chemical",
                     data=dmc.json(),headers=auth_header).json()

#test compound
dmc_compound = schemas_pydantic.Compound(chemicals=[schemas_pydantic.Chemical(smiles='COC(=O)OC', name='DMC', reference='DMC_Elyte_2020')],
    amounts=[schemas_pydantic.Amount(value=1.0, unit='mol')],name='DMC')
ans_ = requests.post(f"http://{host}:{port}/api/broker/post/compound",data=dmc_compound.json(),headers=auth_header).json()

#test compound complex
mixed_compound = schemas_pydantic.Compound(chemicals=[
    schemas_pydantic.Chemical(smiles='COC(=O)OC', name='DMC', reference='DMC_Elyte_2020'),
    schemas_pydantic.Chemical(smiles='[Li+].F[P-](F)(F)(F)(F)F', name='LiPF6', reference='LiPF6_Elyte_2020')],
    amounts=[schemas_pydantic.Amount(value=1.0, unit='mol'), schemas_pydantic.Amount(value=0.2, unit='mol')],
    name='LDMC_LIPF6')

ans_ = requests.post(f"http://{host}:{port}/api/broker/post/compound",data=mixed_compound.json(),headers=auth_header).json()

#test measurement
form = schemas_pydantic.Formulation(chemicals=[schemas_pydantic.Chemical(smiles='COC(=O)OC', name='DMC', reference='DMC_Elyte_2020'),
                                                schemas_pydantic.Chemical(smiles='[Li+].F[P-](F)(F)(F)(F)F', name='LiPF6', reference='LiPF6_Elyte_2020')],
    amounts=[schemas_pydantic.Amount(value=1.0, unit='mol'),schemas_pydantic.Amount(value=0.1, unit='mol')],
    ratio_method='molal')
orig = schemas_pydantic.Origin(origin='simulation', what='density')
data = schemas_pydantic.FomData(values=[1.23],unit='g/cm**2',dim=1,
                                name='density',origin=orig,internalReference='aTest',
                                fail=False, message='My Message', rating=1)
data_nd = schemas_pydantic.FomData(values=[1.23,234,12,13,56,26],unit='g/cm**2',dim=6,
                                name='vectorial',origin=orig,internalReference='aTest',
                                fail=False, message='My Message', rating=1)
temp = schemas_pydantic.Temperature(value=300,unit='K')
meas = schemas_pydantic.Measurement(formulation=form,temperature=temp,pending=False,fom_data=[data_nd],kind=orig)

ans_ = requests.post(f"http://{host}:{port}/api/broker/post/measurement",data=meas.json(),headers=auth_header)
ans_.json()


#!/usr/bin/env python3
# -*- coding:utf-8 -*-

import os

def main():
	print('Process [%s] start...'% (os.getpid()));
	pid = os.fork();
	if( pid == 0):
		print('child process [%s], parent is [%s]'%(os.getpid(), os.getppid()));
	else:
		print('parent process [%s], child is [%s]'%(os.getpid(), pid));

if __name__ == '__main__':
	main();


import time

start_time = time.time()  # just to time runtime
data = 'data/input9.txt'  # input filename

tmp = [10]                #tmp will be help array to read each line and they should all start with 10
input = []                #will hold oure line arrays

with open(data) as f:           #read in Data. the idea is to surround our original input
  while True:                   #with 10s so we don't need to watch out for the edge cases
    c = f.read(1)               #and can just compare evry original input with each neigbough
                                #since the edge cases are sourrounded with numbers that are higher in any case
    if not c:                   #read in char by char there arn't any left
        tmp.append(10)          #put a 10 in the end
        input.append(tmp)       #put the line araay in input input=[...,[10,...,10]]
        break                   #break since input ends
    if c == '\n':               #if line ends
        tmp.append(10)          # put a 10 at the end
        input.append(tmp)       #put the line araay in input input=[...,[10,...,10]]
        tmp = [10]              #put a 10in the begin of the next line
    else:                       #else make the char to int and add to line list
        tmp.append(int(c))

tmp1 = [10 for e in range(0,len(input[0]))]     #our goal is to get (x=input int):
input.append(tmp1)                              #[[10,10,...,10,10],
tmp2 = input                                    # [10,x,.....,x,10],
input= [tmp1]                                   # [10,x,.....,x,10],
                                                # [10,10,...,10,10]]
for e in tmp2:                                  #this part only puts the arrays of 10 in front
    input.append(e)                             #and at the end so we get our wished form

risk_lvl=[]

for i in range(1,len(input)-1):                     #no we itterate over our input ignoren the
    for j in range(1,len(input[i])-1):              #first and last row and colone
        x=input[i][j]                               #for each point we look if hes smaler than all his neighbours
        if input[i-1][j] > x and input[i+1][j] > x and  input[i][j-1] > x and input[i][j+1] > x :
            risk_lvl.append(x+1)                    #if s we add one to get the risk and add it to our risk list

print("risk_lvl = ",risk_lvl)
print("result= ",sum(risk_lvl))                     #print sum of the risk list to get result
print("-------- %s seconds --------" % (time.time() - start_time),"\n") #print runtime


#! /usr/bin/env python3
"""From http://stackoverflow.com/a/12260597/400691"""
import logging
import sys
from optparse import make_option, OptionParser

import dj_database_url
import django
from colour_runner.django_runner import ColourRunnerMixin
from django.conf import settings
from django.test.runner import DiscoverRunner

logging.getLogger().addHandler(logging.NullHandler())

settings.configure(
    DATABASES={
        'default': dj_database_url.config(
            default='postgres://localhost/rest_framework_push_notifications',
        ),
    },
    INSTALLED_APPS=(
        'push_notifications',

        # Put contenttypes before auth to work around test issue.
        # See: https://code.djangoproject.com/ticket/10827#comment:12
        'django.contrib.contenttypes',
        'django.contrib.auth',
        'django.contrib.sessions',
        'django.contrib.sites',
        'django.contrib.admin',
    ),
    MIDDLEWARE_CLASSES=(),
    PASSWORD_HASHERS=('django.contrib.auth.hashers.MD5PasswordHasher',),
    ROOT_URLCONF='tests.urls',
    SITE_ID=1,
    REST_FRAMEWORK={
        'DEFAULT_PERMISSION_CLASSES': ('rest_framework.permissions.IsAuthenticated',),
        'DEFAULT_RENDERER_CLASSES': ('rest_framework.renderers.JSONRenderer',),
    },

    USE_TZ=True,
)

# must be called after the settings were configured
django.setup()

class Runner(ColourRunnerMixin, DiscoverRunner):
    pass

option_list = (
    make_option(
        '-v', '--verbosity', action='store', dest='verbosity', default='1',
        type='choice', choices=['0', '1', '2', '3'],
        help=('Verbosity level; 0=minimal output, 1=normal output, ' +
              '2=verbose output, 3=very verbose output'),
    ),
)

parser = OptionParser(option_list=option_list)
options, args = parser.parse_args()

test_runner = Runner(verbosity=int(options.verbosity))
failures = test_runner.run_tests(args)
if failures:
    sys.exit(1)


x = int(input())

now = 0
i = 1
ans = 0

# 超えた段階でもうそこにいくことができる
while now < x:
    now += i
    i += 1
    ans += 1
print(ans)


import neural_network as nn
import preprocessing as pp
import k_fold_cross_validation as kfx
import sys
import numpy as np
import matplotlib.pyplot as plt
import scipy.io as spio

def main(args):
    message = """\
    There are three ways of running my neural network. Please select one.
    5 - Train 2 models, one with a ReLU activation function and the other with a Sigmoid activation
        function. Both with 1 hidden layer and 50 neurons.
        Example: python3 run_network.py 5
    6 - Train 10 different models, each with a random number between 1-200 of hidden layer neurons.
        Example: python3 run_network.py 6
    7 - Train 1 model with 2 hidden layers each with an optional number of neurons.
        Example: python3 run_network.py 7 50 25
        (This would train a model wit type h 50 and 25 neurons on the first and second hidden layers respectively)"""
    if not args or not (args[0] == '5' or args[0] == '6' or args[0] == '7'):
        print(message)
        sys.exit()

    x_train, train_labels = import_training_data()
    x_train = pp.minmax_initialisation(x_train)
    pca_matrix = pp.get_pca_matrix(x_train)
    x_train = pp.reduce_dimensions(x_train, pca_matrix)

    x_test, test_labels = import_testing_data()
    x_test = pp.minmax_initialisation(x_test)
    x_test = pp.reduce_dimensions(x_test, pca_matrix)

    if args[0] == '5':
        method5(x_train, train_labels, x_test, test_labels)
    elif args[0] == '6':
        method6(x_train, train_labels, x_test, test_labels)
    elif args[0] == '7':
        method7(x_train, train_labels, x_test, test_labels, (int(args[1]), int(args[2])))

def method5(train_data, train_labels, test_data, test_labels):
    kfx.train_test_plot_model(train_data, train_labels, test_data, test_labels)
    kfx.train_test_plot_model(train_data, train_labels, test_data, test_labels,
                              activation=nn.sigmoid, activation_derivative=nn.sigmoid_derivative)

def method6(train_data, train_labels, test_data, test_labels, n_trials=5):
    hidden_neurons = np.arange(20, 201, 20)  # [20, 40, 60, 80, 100, 120, 140, 160, 180, 200]
    accuracies = np.empty((n_trials, hidden_neurons.size))
    for i in range(n_trials):
        print("Trial", i + 1)
        for j in range(hidden_neurons.size):
            print("Initialising model with", hidden_neurons[j], "hidden layer neurons.")
            accuracies[i, j] = kfx.train_test_plot_model(train_data, train_labels, test_data, test_labels,
                                                         hiddens=(hidden_neurons[j], 0)).accuracies[-1]
    print(accuracies)
    avg_accuracies = np.mean(accuracies, axis=0)
    print(avg_accuracies)
    print(np.std(accuracies, axis=0))
    plt.errorbar(x=hidden_neurons, y=avg_accuracies, yerr=np.std(accuracies, axis=0))
    plt.xlabel('Number of hidden layer neurons')
    plt.ylabel('Accuracy (%)')
    plt.title('Average accuracy against hidden layer neurons')
    plt.show()

def method7(train_data, train_labels, test_data, test_labels, hiddens):
    kfx.train_test_plot_model(train_data, train_labels, test_data, test_labels, hiddens=hiddens)

def import_training_data():
    mat = spio.loadmat('../train_data', squeeze_me=True)
    train_data = mat['x_train']
    train_labels = mat['x_train_labs']
    return train_data, np.array(train_labels) - 1

def import_testing_data():
    mat2 = spio.loadmat('../test_data', squeeze_me=True)
    test_data = mat2['x_test']
    test_labels = mat2['x_test_labs']
    return test_data, np.array(test_labels) - 1

if __name__ == '__main__':
    main(sys.argv[1:])


from RPIO import PWM
from mpu import *
import serial
import RPi.GPIO as GPIO
import time
import math
import datetime
#GPIO.setmode(GPIO.BCM)
leftPin=17
rightPin=18
upPin=22
max=2000
min=1000
servo=PWM.Servo()
PWM.set_loglevel(PWM.LOG_LEVEL_ERRORS)
baseSpeed=1000
lastTime=None
pidX=0
pidY=0
angleX=0
lastAngleX=0
lastAngleY=0
angleY=0
kpX=5
kiX=5
kdX=5
kpY=10
kiY=10
kdY=10
msg=0
# Read message from the serial port
ser=serial.Serial("/dev/ttyAMA0",timeout=0)
def run():
	global baseSpeed
	init()
	while True:
		try:
			if ser.inWaiting():
				msg=ser.read(ser.inWaiting())
				if(msg=='u'):
					baseSpeed+=20
				elif(msg=='d'):
					baseSpeed-=20
				print(msg)
				print(baseSpeed)
		except Exception as e:
			print(e)
		pidX,pidY=calculate(0,0)
		print("PID-Y:"+str(pidY))
		print("Base Power:"+str(baseSpeed))
		#ySpeed=getTransY()*10
		setSpeed(leftPin,baseSpeed-transform(pidY))
		setSpeed(upPin,baseSpeed+transform(pidX))
		setSpeed(rightPin,baseSpeed+transform(pidY))
		time.sleep(0.05)
def calculate(goalX,goalY):
	global lastTime
	global angleX
	global angleY
	global lastAngleX
	global lastAngleY
	if not lastTime:
		lastTime=time.time()
		return 0,0
	interval=time.time()-lastTime
	gx,gy,gz=get_gyro_rate()
	acX,acY=get_acc()
	angleX=0.98*(angleX+gx*interval)+0.02*acX
	angleY=0.98*(angleY+gy*interval)+0.02*acY
	#angleY+=gy*(interval/1000.0)
	print("Angle Y:"+str(transform(angleY)))

	#pidX=kpX*(goalX-acX)+kiX*(lastX-acX)*interval+kdX*(lastX-acX)/interval
	#pidY=kpY*(goalY-acY)+kiY*(lastY-acY)*interval+kdY*(lastY-acY)/interval
	pidY=kpY*(goalY-angleY)+kiY*(lastAngleY-angleY)*interval+kdY*(lastAngleY-angleY)/interval
	pidX=kpX*(goalX-angleX)+kiX*(lastAngleX-angleX)*interval+kdX*(lastAngleX-angleX)/interval
	lastAngleX=angleX
	lastAngleY=angleY
	lastTime=time.time()
	return pidX,pidY
def transform(value):
	return value/math.cos(math.radians(45))
def init():
	setSpeed(leftPin,min)
	setSpeed(rightPin,min)
	setSpeed(upPin,min)
	time.sleep(2)
def setSpeed(pin,speed):
	speed=int(speed)
	if speed>max:
		speed=max
	elif speed<min:
		speed=min
	speed=speed/10*10
	servo.set_servo(pin,int(speed))
if __name__=="__main__":
	#calibrate()
	run()


#!/usr/bin/env python
# coding: utf-8

# ## Research Question 1: Causal Effect between Party Support and candidates being on the ballot.

# In[1]:

get_ipython().run_line_magic('matplotlib', 'inline')
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from ipywidgets import interact, interactive
import itertools
import hashlib
from scipy.stats import poisson, norm, gamma
import statsmodels.api as sm

sns.set(style="dark")
plt.style.use("ggplot")

from pymc3 import *
import pymc3 as pm

import arviz as az

# In[2]:

dem = pd.read_csv("dem_candidates.csv")

# In[3]:

dem = dem.dropna(subset=['Party Support?'])
print(dem.shape)
dem.head()

# In[4]:

supported = dem[dem['Party Support?'] == 'Yes']
supported[supported['Primary Status'] == 'Advanced'].head()

# ### EDA

# Visualization of the number of cadidates being on the ballot and not being on the ballot given their party support status.

# In[5]:

dem_short = dem[['Party Support?', 'General Status']]

# In[6]:

support_and_status = dem_short.groupby(by=['Party Support?', 'General Status'], as_index=False).size()
support_and_status = support_and_status.rename(columns={'size': 'Number of candidates'})
support_and_status

# In[7]:

sns.barplot(x='Party Support?', y='Number of candidates', hue='General Status', data=support_and_status)
plt.ylabel('Number of candidates');

# In[8]:

support_and_status['Percentage of candidates'] = support_and_status['Number of candidates'] / support_and_status.groupby('Party Support?')['Number of candidates'].transform('sum') * 100

# In[9]:

support_and_status

# In[10]:

sns.barplot(x='Party Support?', y='Percentage of candidates', hue='General Status', data=support_and_status)
plt.ylabel('Percentage of candidates %');

# From the bar chart above, we can see that candidates with party support have higher chance being on the ballot, and candidates without party support have higher chance not being on the ballot. Thus, there might be some correlation between `Party Support?` and `General Status`. This visualization is relevant to our research question because it suggests that there might be a causation between `Party Support?` and `General Status` of a candidate.

# ### Research Question Results

# In[11]:

dem['General Status'] = dem['General Status'].map({'None': 0, 'On the Ballot': 1})
dem['Veteran?'] = dem['Veteran?'].map({'No': 0, 'Yes': 1})
dem['LGBTQ?'] = dem['LGBTQ?'].map({'No': 0, 'Yes': 1})
dem['Elected Official?'] = dem['Elected Official?'].map({'No': 0, 'Yes': 1})
dem['STEM?'] = dem['STEM?'].map({'No': 0, 'Yes': 1})
dem['Obama Alum?'] = dem['Obama Alum?'].map({'No': 0, 'Yes': 1})
dem['Self-Funder?'] = dem['Self-Funder?'].map({'No': 0, 'Yes': 1})
dem['Race'] = dem['Race'].map({'Nonwhite': 0, 'White': 1})
dem['Won Primary'] = dem['Won Primary'].map({'No': 0, 'Yes': 1})
dem['Party Support?'] = dem['Party Support?'].map({'No': 0, 'Yes': 1})
dem['Biden Endorsed?'] = dem['Biden Endorsed?'].map({'No': 0, 'Yes': 1})
dem['Guns Sense Candidate?'] = dem['Guns Sense Candidate?'].map({'No': 0, 'Yes': 1})

# #### Causal Inference:

# ##### Methods:
#
# • Treatment: `Party Support?`
#
# • Outcome: `General Status`
#
# • (Possible) Confounders: `Race`, `LGBTQ?`, `STEM?`,`Obama Alum?`
#
# (other possible confounders are omitted due to the lack of data.)
#
# We use outcome regression to conduct our research. There is no colliders given the `General Status` is the final result.

# In[12]:

def fit_LR_model(df, target_variable, explanatory_variables, intercept = False):
    """
    Fits an Logistic Regression model from data.

    Inputs:
        df: pandas DataFrame
        target_variable: string, name of the target variable
        explanatory_variables: list of strings, names of the explanatory variables
        intercept: bool, if True add intercept term
    Outputs:
        fitted_model: model containing OLS regression results
    """

    target = df[target_variable]
    inputs = df[explanatory_variables]
    if intercept:
        inputs = sm.add_constant(inputs)

    fitted_model = sm.Logit(target, inputs).fit()
    return(fitted_model)

# In[13]:

dem_clean = dem.dropna(subset=['General Status', 'Party Support?', 'Race', 'LGBTQ?', 'STEM?','Obama Alum?'])
print(dem_clean.shape)

# In[14]:

linear_model = fit_LR_model(dem_clean, 'General Status', ['Party Support?'])
print(linear_model.summary())

# ##### Our models:

# In[15]:

linear_model = fit_LR_model(dem_clean, 'General Status', ['Party Support?', 'Race', 'LGBTQ?', 'STEM?','Obama Alum?'])
print(linear_model.summary())

# ##### Results:
#
# From the model above, being supported by the party increases the log odds ratio of the candidate being on the ballot by between 4.1 to 9.8. The causation is solidified by significantly small p-value. If the model is misspecified, then the interpretation is only valid with respect to the projected model.
#
# For all factors other than `Party Support?`, the p-values are significant for `Race` and `STEM`.

# Our model's Log-Likelihood is also closer to 0 than the model not considering confounders. Thus, it is more accurate.

# ## Research Question 2: Predicting Democratic Primary Election Results From Personal Information on Candidates using Logistic Regression vs. Decision Tree Classifier

# In[16]:

dem = pd.read_csv("dem_candidates.csv")
dems = dem[['Candidate', 'State', 'Veteran?', 'LGBTQ?', 'Elected Official?', 'STEM?', 'Obama Alum?', 'Self-Funder?']]
dems['win'] = dem['Primary Status'].map({'Lost': 0, 'Advanced': 1})
dems['Veteran?'] = dem['Veteran?'].map({'No': 0, 'Yes': 1})
dems['LGBTQ?'] = dem['LGBTQ?'].map({'No': 0, 'Yes': 1})
dems['Elected Official?'] = dem['Elected Official?'].map({'No': 0, 'Yes': 1})
dems['STEM?'] = dem['STEM?'].map({'No': 0, 'Yes': 1})
dems['Obama Alum?'] = dem['Obama Alum?'].map({'No': 0, 'Yes': 1})
dems['Self-Funder?'] = dem['Self-Funder?'].map({'No': 0, 'Yes': 1})

# In[17]:

dems.head(1)

# ### EDA

# In[18]:

xnoise, ynoise = np.random.normal(0, 0.05, size = 811), np.random.normal(0, 0.05, size = 811)
plt.scatter(x = dems['Veteran?'] + xnoise, y = dems['win'] + ynoise, alpha = 0.3)
plt.xlabel("Veteran?")
plt.ylabel("Advanced?")
f = np.mean(dems[dems['Veteran?'] == 0]['win'])
t = np.mean(dems[dems['Veteran?'] == 1]['win'])
f, t;

# On average, being a veteran marginally reduces chances of advancing without controling for confounders, this might be worth a further analysis.

# In[19]:

xnoise, ynoise = np.random.normal(0, 0.05, size = 811), np.random.normal(0, 0.05, size = 811)
plt.scatter(x = dems['LGBTQ?'] + xnoise, y = dems['win'] + ynoise, alpha = 0.3)
plt.xlabel("LGBTQ?")
plt.ylabel("Advanced?")
f = np.mean(dems[dems['LGBTQ?'] == 0]['win'])
t = np.mean(dems[dems['LGBTQ?'] == 1]['win'])
f, t;

# On average, being an LGBTQ member marginally reduces chances of advancing without controling for confounders, this might be worth further analyzing.

# In[20]:

xnoise, ynoise = np.random.normal(0, 0.05, size = 811), np.random.normal(0, 0.05, size = 811)
plt.scatter(x = dems['Elected Official?'] + xnoise, y = dems['win'] + ynoise, alpha = 0.3)
plt.xlabel("Elected official?")
plt.ylabel("Advanced?")
f = np.mean(dems[dems['Elected Official?'] == 0]['win'])
t = np.mean(dems[dems['Elected Official?'] == 1]['win'])
f, t;

# On average being an Elected official increases the chances of advancement without controling for confounders, this is worth a further exploration in phase 2.

# In[21]:

xnoise, ynoise = np.random.normal(0, 0.05, size = 811), np.random.normal(0, 0.05, size = 811)
plt.scatter(x = dems['STEM?'] + xnoise, y = dems['win'] + ynoise, alpha = 0.3)
plt.xlabel("STEM?")
plt.ylabel("Advanced?")
f = np.mean(dems[dems['STEM?'] == 0]['win'])
t = np.mean(dems[dems['STEM?'] == 1]['win'])
f, t;

# On average, having a STEM background significantly reduces the chances of advancing without controling for confounders, this is surely worth exploring in phase 2. Maybe non-STEM degrees builds skills required for politicians.

# In[22]:

xnoise, ynoise = np.random.normal(0, 0.05, size = 811), np.random.normal(0, 0.05, size = 811)
plt.scatter(x = dems['Obama Alum?'] + xnoise, y = dems['win'] + ynoise, alpha = 0.3)
plt.xlabel("Obama Alum?")
plt.ylabel("Advanced?")
f = np.mean(dems[dems['Obama Alum?'] == 0]['win'])
t = np.mean(dems[dems['Obama Alum?'] == 1]['win'])
f, t;

# On average, being an Obama Alum significantly increases the chances of advancing without controling for confounders, this is surely worth exploring in phase 2. Maybe Obama alum have skills required for successful advancement.

# ### Methods

# We will be predicting whether a Democratic candidate wins the Democratic primaries or not using demographic/personal information about the candidate. The features we chose are "Obama Alum?", "Veteran?", "STEM?", "LGBTQ?", "Elected Official?". We will compare the performance of logistic regression vs decision tree classification when predicting primary election outcomes for Democratic candidates from personal information.

# In[23]:

#clean the dataset utilizing only variables of value.
df = dems[['Candidate', 'Veteran?', 'LGBTQ?', 'Elected Official?', 'STEM?', 'Obama Alum?', 'win']].dropna()
df.head(1)

# In[24]:

#Split the data set into train and test sets
from sklearn.model_selection import train_test_split
data_tr, data_te = train_test_split(df, test_size=0.10, random_state=42)
print("Training Data Size: ", data_tr.shape)
print("Test Data Size: ", len(data_te))

# X, Y are training data
X = data_tr[['Veteran?', 'LGBTQ?', 'Elected Official?', 'STEM?', 'Obama Alum?']]
Y = data_tr['win']

# In[25]:

X.head(1)

# ### Logistic Regression

# In[26]:

# Fit Logistic GLM model on training set
freq_model = sm.GLM(Y, sm.add_constant(X), family=sm.families.Binomial())
freq_res = freq_model.fit()
print(freq_res.summary())

# In[27]:

# Fit Logistic GLM model on training set excluding LGBTQ data since it's parameter is centered around 0.
X = data_tr[['Veteran?', 'Elected Official?', 'STEM?', 'Obama Alum?']]
freq_model = sm.GLM(Y, sm.add_constant(X), family=sm.families.Binomial())
freq_res = freq_model.fit()
print(freq_res.summary())

# In[28]:

# use parameters to predict Y values for test set
X_test = data_te[['Veteran?', 'Elected Official?', 'STEM?', 'Obama Alum?']]
Y_prob = 1 / (1 + (1/np.exp(X_test['Veteran?']*(-0.2156) + X_test['Elected Official?']*0.6369 + X_test['STEM?']*(-0.3501)
                   + X_test['Obama Alum?']*0.1124 - 0.7382)))

# ### Logistic Regression Results

# In[29]:

#accuracy of training dataset
Y_prob_train = 1 / (1 + (1/np.exp(X['Veteran?']*(-0.2156) + X['Elected Official?']*0.6369 + X['STEM?']*(-0.3501)
                   + X['Obama Alum?']*0.1124 - 0.7382)))
Y_preb_train = (Y_prob_train>= 0.5)
a = np.mean(data_tr["win"] == Y_preb_train)
print(f"Accuracy on training data set for parametric case: {a}")

# In[30]:

#accuracy of parametric model on test set at 0.5 threshold
Y_test = data_te['win']
Y_pred = (Y_prob >= 0.5)
accuracy = np.mean(Y_test == Y_pred)
print(f"Accuracy on test set for parametric case: {accuracy}")

# In[31]:

#calculating logistic regression cross entropy loss
ce_loss = -(np.average((Y_test * np.log(Y_prob)) + (1 - Y_test) * np.log(Y_prob)))
ce_loss

# ### Decision Tree Classification

# In[32]:

#non parametric classifier - decision tree

X = data_tr[['Veteran?', 'LGBTQ?', 'Elected Official?', 'STEM?', 'Obama Alum?']]
X_test = data_te[['Veteran?', 'LGBTQ?', 'Elected Official?', 'STEM?', 'Obama Alum?']]

from sklearn.tree import DecisionTreeClassifier

model_tree = DecisionTreeClassifier()
model_tree.fit(X, Y)
probs = model_tree.predict_proba(X_test)[:, 1]
y_hat_tree = (probs > 0.5).astype(np.int64)

# ### Decision Tree Classification Results

# In[33]:

accuracy = np.mean(Y_test == y_hat_tree)
print(f"Accuracy on test set for non-parametric case: {accuracy}")

# In[34]:

probs_null = model_tree.predict_proba(X)[:, 1]
y_null = (probs_null > 0.5).astype(np.int64)

accuracy = np.mean(Y == y_null)
print(f"Accuracy on train set for non-parametric case: {accuracy}")

#explanation for not 100% accuracy - since we have binary RV, we can only split once on
#each feature within each vertical path.

# In[35]:

#interpretability and explanability of why model makes choices, best attempt below.
#It appears classifier splitting down the middle for each train feature and
#rounding up the average of outcome variables to train itself, applying the same rule to test set.

from sklearn.tree import plot_tree

plt.figure(figsize=(16, 12))
plot_tree(model_tree, fontsize=12, filled=True);

# We can conclude that the logistic regression model and the decision tree classification perform similarly.

# In[ ]:


import numpy as np
from sklearn.linear_model import LinearRegression
import pandas as pd
import matplotlib.pyplot as plt
train=pd.read_csv("./california_housing_test.csv")
pd.set_option('display.max_columns', None)
bed=train.loc[:,'total_bedrooms']
x=np.linspace(0,10,100)
y=2+3*x + np.random.normal(-3, 3, 100)
model = LinearRegression()
x = x[:, np.newaxis]
y = y[:, np.newaxis]
model.fit(x, y)
y_pred = model.predict(x)
plt.scatter(x, y, s=10)
plt.plot(x, y_pred, color='r')
print(y_pred)
plt.show()


# helpers.py
# Isabelle Bouchard
# Carolyne Pelletier
# 2019-12-14
# IFT-6390

from scikit_classifiers import DATASETS
from scikit_classifiers import CLASSIFIERS
import plotly.graph_objects as go
import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.metrics import confusion_matrix
import numpy as np

def print_models():
    _MODELS = list(CLASSIFIERS.keys())
    print(f"Models")
    for i, m in enumerate(_MODELS):
        print(f"{i}: {m}")

def print_datasets():
    _DATASETS = list(DATASETS.keys())
    print(f"Datasets")
    for i, m in enumerate(_DATASETS):
        print(f"{i}: {m}")

def plot_grid_search(cv_results, model_name, dataset_name, balance=True):
    """
    Helper to plot grid search using plotly.
    """
    params = cv_results["params"]
    param_names = list(params[0].keys())
    if len(param_names) == 0:
        print("No parameters to show!")
        return None
    # Special case for MLP hidden layers
    elif len(param_names) == 1:
        x_title = "1st hidden layer"
        y_title = "2nd hidden layer"
        x, y = [], []
        for p in params:
            x.append(
                p[param_names[0]][0]
                if isinstance(p[param_names[0]], tuple)
                else p[param_names[0]]
            )
            y.append(
                p[param_names[0]][0]
                if isinstance(p[param_names[0]], tuple)
                else 0
            )

    elif len(param_names) == 2:
        x_title = list(params[0].keys())[0]
        y_title = list(params[0].keys())[1]
        x = [p[x_title] for p in params]
        y = [p[y_title] for p in params]

    fig = go.Figure()

    fig.add_trace(go.Scatter(
        x=x,
        y=y,
        hovertext=cv_results["mean_test_score"],
        marker=dict(
            size=10,
            color=cv_results["mean_test_score"],
            colorbar=dict(
                title="mean_f1_score"
            ),
            colorscale="Magma"
        ),
        mode="markers")
    )
    fig.update_layout(
        title=f"Grid Search for {model_name} \
                on {dataset_name} dataset\
                {' balanced' if balance else ''}",
        xaxis_title=x_title,
        yaxis_title=repr(y_title),
    )

    return fig

def separate_by_gender(x,y):
    """
    female 0
    male 1
    """
    male_x = []
    male_y = []
    female_x = []
    female_y = []
    print(len(x))
    print()

    for i,j in zip(x,y):
        # print(i)
        print(i[1])
        if i[1] == 1:
            male_x.append(i)
            male_y.append(j)
        elif i[1] == 0:
            female_x.append(i)
            female_y.append(j)
    print("len of male  : ", len(male_y))
    print("len of female: ", len(female_y))
    return male_x, male_y, female_x, female_y

def save_conf_matrix_gender_specific(cm_male, cm_female, model_type, SMOTE):

    sns.set_style("whitegrid")

    plt.figure(figsize=(12, 5))

    title = "Confusion Matrices for Heart Disease using " + model_type +" "+ SMOTE

    plt.suptitle(title, fontsize=24)
    plt.subplots_adjust(wspace=0.4, hspace=0.4)

    plt.subplot(1, 2, 1)
    plt.title("Male")

    sns.heatmap(cm_male, annot=True, cmap="Blues", fmt="d", cbar=False, annot_kws={"size": 24})
    plt.xlabel("Pred")
    plt.ylabel("True")

    b, t = plt.ylim()  # discover the values for bottom and top
    b += 0.5  # Add 0.5 to the bottom
    t -= 0.5  # Subtract 0.5 from the top
    plt.ylim(b, t)  # update the ylim(bottom, top) values

    plt.subplot(1, 2, 2)
    plt.title("Female")

    sns.heatmap(cm_female, annot=True, cmap="Blues", fmt="d", cbar=False, annot_kws={"size": 24})
    plt.xlabel("Pred")
    plt.ylabel("True")

    b, t = plt.ylim()  # discover the values for bottom and top
    b += 0.5  # Add 0.5 to the bottom
    t -= 0.5  # Subtract 0.5 from the top
    plt.ylim(b, t)  # update the ylim(bottom, top) values

    plt.savefig(title+".png")
    print("saved CM plot")

def plot_target_distribution(target):
    sns.set(style="whitegrid")
    sns.set_palette(sns.color_palette("Set2", n_colors=5))

    plt.figure(figsize=(15, 12))
    plt.subplots_adjust(wspace=0.4, hspace=0.4)
    y_df = pd.DataFrame({'target': target})
    plt.subplot(2, 2, 1)
    plt.title("Distribution of diseased and not diseased patients after SMOTE.")
    sns.countplot(data=y_df, x='target')
    plt.xlabel("Target (0 = no cervical cancer, 1= cervical cancer)")
    # plt.savefig('after_smote_{}.png'.format(dataset_name))
    print("saved figure")

def plot_conf_matrices(X, y, model, model_name, balance):
    male_x, male_y, female_x, female_y = separate_by_gender(X, y)

    male_prediction = model.predict(male_x)
    male_accuracy = np.mean(male_prediction == male_y)
    print("Male Acc  : ", male_accuracy)
    cm_male = confusion_matrix(male_y, male_prediction)

    female_prediction = model.predict(female_x)
    female_accuracy = np.mean(female_prediction == female_y)
    print("Female Acc: ", female_accuracy)
    print(female_prediction)
    cm_female = confusion_matrix(female_y, female_prediction)

    save_conf_matrix_gender_specific(cm_male, cm_female, model_type=model_name, SMOTE=balance)