import telebot
import flask
import conf
from telebot import types
import re
from work_with_db import add_to_data, collect_vacancies_data, check_experience, \
    get_calendar, add_to_calendar, db
from work_with_answers import process_values
import logging
logging.basicConfig(level=logging.ERROR,
                    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

WEBHOOK_URL_BASE = "https://{}:{}".format(conf.WEBHOOK_HOST, conf.WEBHOOK_PORT)
WEBHOOK_URL_PATH = "/{}/".format(conf.TOKEN)
bot = telebot.TeleBot(conf.TOKEN, threaded=False)
bot.remove_webhook()
bot.set_webhook(url=WEBHOOK_URL_BASE+WEBHOOK_URL_PATH)
app = flask.Flask(__name__)

app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///database.db'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
db.app = app
db.init_app(app)

applicant_dict = {}  # Словарь с пользователями
vacancies_data = collect_vacancies_data()  # информация о вакансиях из базы данных

# класс пользователя
class Applicant:
    def __init__(self, name):
        self.name = name
        self.city = None
        self.age = None
        self.sex = None
        self.number = None
        self.email = None
        self.social = None
        self.vacancies = []  # сюда будем складывать вакансии, которыми заинтересовался пользователь

@bot.message_handler(commands=['start'])
def send_welcome(message):
    """
    Функция, которой бот приветствует пользователя
    """
    bot.send_message(message.chat.id, 'Привет!\n'
                                      'Меня зовут Бушенка. Я чат-бот компании "Буше". '
                                      'Буду рад рассказать о нашей компании, наших вакансиях, '
                                      'а также помочь тебе попасть именно к нам! '
                                      'У нас обалденные вензели с вишней! :)\n'
                                      'Чтобы узнать, что я умею нажми /help')

@bot.message_handler(commands=['help'])
def help_func(message):
    """
    Функция, которая выводит все полезные кнопки бота
    """
    bot.send_message(message.chat.id, "Я супер-умный и всегда готов тебе помочь. Надеюсь, мы с тобой подружимся)\n"
                                      "У меня ты можешь:\n"
                                      "-узнать все о компании /about\n"
                                      "-найти контакты HR /contacts (Ну вдруг я тебе не понравлюсь:()\n"
                                      "-получить ответ на интересующий вопрос /Q_A\n"
                                      "-посмотреть актуальные вакансии и выбрать то, что тебе подходит /vacancies\n"
                                      "-вспомнить, что я умею /help\n"
                                      "-познакомиться со мной поближе /intro")

@bot.message_handler(commands=['about'])
def about_func(message):
    """
    Функция, которая выводит информацию о компании
    """
    bot.send_message(message.chat.id, "Наш сайт - https://bushe.ru/\n"
                                      "Наши соц. сети:\n"
                                      "Вконтакте - https://vk.com/bushe.bakery\n"
                                      "Инстаграм - https://www.instagram.com/bushe.bakery/\n"
                                      "Одноклассники - https://ok.ru/bushe.bakery\n"
                                      "Facebook - https://www.facebook.com/bushe.bakery/\n"
                                      "YouTube - https://www.youtube.com/c/BusheBakery\n\n"
                                      "Мы - 'Буше'. Меняем вкус повседневной жизни, "
                                      "создавая новые традиции для еды, общения и вдохновения.\n"
                                      "Работать с нами, значит быть живым в любой момент времени!")

@bot.message_handler(commands=['contacts'])
def contacts_func(message):
    """
    Функция, которая выводит контакты HR
    """
    bot.send_message(message.chat.id, "Позвони в HR: +7 (812) 640-01-10\n"
                                      "Напиши: personal@bushe.ru\n"
                                      "Посети наш сайт с вакансиями - https://careers.bushe.ru/\n"
                                      "Или на hh.ru: "
                                      "https://hh.ru/search/"
                                      "vacancy?st=searchVacancy&from=employerPage&employer_id=114223")

@bot.message_handler(commands=['Q_A'])
def qa_func(message):
    """
    Функция для ответов на вопросы пользователя
    """
    msg = bot.send_message(message.chat.id, 'Можешь задать мне любой вопрос!')
    bot.register_next_step_handler(msg, no_answer)

def no_answer(message):
    """
    Функция, которая генерирует ответ на вопрос. Сейчас бот отвечает только на один вопрос :)
    """
    if message.text == 'Какие этапы отбора?':
        bot.send_message(message.chat.id, 'Сначала ты пообщаешься со мной. А дальше если на твою вакансию '
                                          'нужны hard-skills, то мы тебя позовем на прохождение конкурса. '
                                          'И последний этап - это краткое интервью с hr')
    else:
        bot.send_message(message.chat.id, 'Упс! Такое мне не рассказывали. '
                                          'Можешь связаться со знающими людьми /contacts')

@bot.message_handler(commands=['intro'])
def intro_func(message):
    """
    Функция, с которой начинается заполнение информации о пользователе
    """
    if message.chat.id in applicant_dict:  # проверяем, нет ли пользователя уже в БД,
        # если нет, то спрашиваем имя
        bot.send_message(message.chat.id, "Мы с тобой уже знакомы! :)\n"
                                          "Если хочешь посмотреть вакансии, нажми /vacancies")
    else:
        msg = bot.send_message(message.chat.id, 'Как тебя зовут? (Лучше полностью ФИО)')
        bot.register_next_step_handler(msg, process_name_step)  # переходим к следующему вопросу

def process_name_step(message):
    """
    Функция, которая добавляет пользователя в словарь и записывает его имя, затем спрашиваем возраст
    """
    applicant = Applicant(message.text)  # создаем объект класса Applicant и передаем name
    applicant_dict[message.chat.id] = applicant
    msg = bot.send_message(message.chat.id, 'Сколько тебе лет?')
    bot.register_next_step_handler(msg, process_age_step)

def process_age_step(message):
    """
    Функция, которая записывает возраст пользователя. Если возраст указан не цифрой, то просим ответить корректно.
    Потом спрашиваем пол
    """
    age = message.text
    if not age.isdigit():  # проверяем цифрой ли ответил пользователь
        msg = bot.send_message(message.chat.id, 'Пж, укажи возраст одной цифрой')
        bot.register_next_step_handler(msg, process_age_step)
        return
    applicant_dict[message.chat.id].age = int(age)
    markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
    markup.add('М', 'Ж')
    msg = bot.send_message(message.chat.id, 'Укажи свой пол', reply_markup=markup)
    bot.register_next_step_handler(msg, process_sex_step)

def process_sex_step(message):
    """
    Функция, которая записывает пол пользователя. Затем спрашиваем город
    """
    applicant_dict[message.chat.id].sex = message.text
    markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
    markup.add('Москва', 'Санкт-Петербург')
    msg = bot.send_message(message.chat.id, 'Из какого ты города?', reply_markup=markup)
    bot.register_next_step_handler(msg, process_city_step)

def process_city_step(message):
    """
    Функция, которая записывает город пользователя. Затем спрашиваем номер телефона
    """
    applicant_dict[message.chat.id].city = message.text
    msg = bot.send_message(message.chat.id, 'Дашь номерочек? Вдруг мне захочется тебе позвонить в 5 утра в вс :)')
    bot.register_next_step_handler(msg, take_number)

def take_number(message):
    """
    Функция, которая записывает номер пользователя. Затем спрашиваем email
    """
    applicant_dict[message.chat.id].number = message.text
    msg = bot.send_message(message.chat.id, 'И email тоже (постараюсь не спамить)')
    bot.register_next_step_handler(msg, take_email)

def take_email(message):
    """
    Функция, которая записывает email пользователя. Затем спрашиваем соц. сети
    """
    applicant_dict[message.chat.id].email = message.text
    msg = bot.send_message(message.chat.id, 'Можешь еще оставить ссылку на какую-нибудь соц. сеть')
    bot.register_next_step_handler(msg, take_social)

def take_social(message):
    """
    Функция, которая записывает соц.сети пользователя, выводит все введенные пользователем данные
    и спрашивает, верны ли они
    """
    applicant_dict[message.chat.id].social = message.text
    bot.send_message(message.chat.id, f'Проверь, все ли я правильно записал:\n'
                                      f'Имя: {applicant_dict[message.chat.id].name}\n'
                                      f'Возраст: {applicant_dict[message.chat.id].age}\n'
                                      f'Пол: {applicant_dict[message.chat.id].sex}\n'
                                      f'Город: {applicant_dict[message.chat.id].city}\n'
                                      f'Телефон: {applicant_dict[message.chat.id].number}\n'
                                      f'Email: {applicant_dict[message.chat.id].email}\n'
                                      f'Соц. сети: {applicant_dict[message.chat.id].social}')
    markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
    markup.add('Да', 'Нет')
    msg = bot.send_message(message.chat.id, 'Все верно?', reply_markup=markup)
    bot.register_next_step_handler(msg, check_correctness_step)

def check_correctness_step(message):
    """
    Функция, которая записывает все данные в базу данных, если они верны, а если нет, то идем по второму кругу
    """
    if message.text == 'Да':
        user = applicant_dict[message.chat.id]  # получаем информацию о пользователе из словаря
        data_app = [(message.chat.id, user.name, user.city, user.age, user.sex, user.number, user.email, user.social)]
        # подготавливаем данные
        add_to_data(data_app)  # записываем данные в БД
        bot.send_message(message.chat.id, 'Приятно познакомиться! Теперь можешь посмотреть свежие вакансии /vacancies')
    else:
        applicant_dict.pop(message.chat.id, None)
        bot.send_message(message.chat.id, 'Ой-ой, ну ты чего, теперь придется проходить все заново... Нажми /intro')

@bot.message_handler(commands=['vacancies'])
def vacancies_func(message):
    """
    Функция, которая проверяет, есть ли данные о пользователе в БД. Если нет, то спрашиваем,
    если есть, то показываем вакансии
    """
    if message.chat.id not in applicant_dict:  # проверяем, есть ли id пользователя в словаре
        bot.send_message(message.chat.id, "Кажется, мы с тобой еще не знакомы, давай знакомиться!")
        intro_func(message)  # отправляем к функции, которая собирает информацию о пользователе
    else:
        bot.send_message(message.chat.id, "Привет! Сейчас посмотрю, что появилось из новеньких вакансий")
        choose_city(message)

def choose_city(message):
    """
    Функция для выбора города, по которому будут фильтрованы вакансии
    """
    markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
    markup.add('Москва', 'Санкт-Петербург')
    msg = bot.send_message(message.chat.id, 'В каком городе тебе бы хотелось работать?', reply_markup=markup)
    bot.register_next_step_handler(msg, buttons_vac)

def buttons_vac(message):
    """
    Функция, которая фильтрует вакансии по городу и выводит все актуальные вакансии в виде кнопок с названиями
    """
    city = message.text
    vacancies_city = []
    for vacancy in vacancies_data:
        if vacancy.city == city:
            vacancies_city.append((vacancy.id, vacancy.name))
    keyboard = types.InlineKeyboardMarkup(row_width=1)
    keyboard.add(*[types.InlineKeyboardButton(text=vac[1], callback_data=f"button_{int(vac[0])}")
                   for vac in vacancies_city])
    bot.send_message(message.chat.id, 'Можешь выбрать из этих вакансий:', reply_markup=keyboard)

@bot.callback_query_handler(func=lambda call: True)
def callback_inline(call):
    """
    Функция, которая выводит условия и обязанности по выбранной должности. Дальше спрашиваем про опыт работы
    """
    if call.message:
        _, i = call.data.split('_')  # получаем id вакансии
        vacancy_id = int(i)
        applicant_dict[call.message.chat.id].vacancies.append(vacancy_id)  # добавляем вакансию в список всех вакансий,
        # которыми интересовался пользователь
        for vacancy in vacancies_data:
            if vacancy.id == vacancy_id:
                conditions = vacancy.conditions
                responsibilities = vacancy.responsibilities
        bot.send_message(call.message.chat.id, 'Вот, что мы предлагаем:\n'
                                               f'{conditions}')
        bot.send_message(call.message.chat.id, 'А вот, что тебе предстоит делать:\n '
                                               f'{responsibilities}')
        msg = bot.send_message(call.message.chat.id, 'Если тебя все устраивает, то укажи свой опыт работы на схожей '
                                                     'должности (просто цифрой сколько лет)')
        bot.register_next_step_handler(msg, process_exp_step)

def process_exp_step(message):
    """
    Функция, которая записывает id пользователя, id вакансии и опыт работы в БД
    """
    if re.search('\d+', message.text):  # если в ответе пользователя есть цифра, то записываем ее
        experience = int(re.findall('\d+', message.text)[0])
    else:
        experience = 0  # иначе записываем 0
    vac_id = applicant_dict[message.chat.id].vacancies[-1]  # получаем id вакансии
    if check_experience(vac_id, experience):  # если кандидат подходит по опыту работы, то спрашиваем про ценности
        msg = bot.send_message(message.chat.id, 'Отлично! Теперь мы хотели бы узнать тебя поближе. '
                                                'Пожалуйста, опиши кратко, какой философии по жизни ты придерживаешься,'
                                                'почему хочешь попасть в нашу команду')
        bot.register_next_step_handler(msg, values_step)
    else:
        if applicant_dict[message.chat.id].sex == 'M':  # подберем слово по полу пользователя
            word = 'готов'
        else:
            word = 'готова'
        bot.send_message(message.chat.id, 'Друг, к сожалению, ты не подходишь для данной вакансии из-за нехватки опыта.'
                                          f' Мы с удовольствием примем тебя в нашу команду, когда ты будешь {word}.'
                                          'Или можешь посмотреть другие вакансии /vacancies')

def values_step(message):
    """
    Функция, которая проверяет кандидата на совпадение ценностей.
    Если все ценности совпадают, то выводит требования к кандидату
    """
    answer = message.text
    if process_values(answer):  # проверяем на совпадение ценностей
        vac_id = applicant_dict[message.chat.id].vacancies[-1]
        for vacancy in vacancies_data:
            if vacancy.id == vac_id:
                requirements = vacancy.requirements.split('\n')
        bot.send_message(message.chat.id, 'Теперь тебе надо будет отметить те пункты, по которым ты подходишь')
        ask_requirements(requirements, message)
    else:
        bot.send_message(message.chat.id, 'Прости, друг, но я считаю, что наши мировоззрения не сходятся. '
                                          'Удачи тебе в поисках. Кто ищет, тот найдет!')

def ask_requirements(requirements, message):
    """
    Функция, которая выводит по очереди пункты с требованиями и спрашивает, относятся ли они к кандидату
    Если кандидат подошел по всем требованиям, то предлагает выбрать дату и время для собеседования
    :param requirements: список пунктов с требованиями
    """
    if not requirements:
        dates = get_calendar()  # получаем расписание
        markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
        markup.add(*[x.day + ', ' + x.time for x in dates if not x.applicant_id])  # выводим свободные слоты в виде кнопок
        msg = bot.send_message(message.chat.id, 'Мы друг другу подходим! Выбери дату и время, когда ты можешь прийти '
                                                'на собес', reply_markup=markup)
        bot.register_next_step_handler(msg, choose_date)
    else:
        requirement = requirements[0]  # получаем пункт с требованием
        requirements.remove(requirement)  # убираем этот пункт из общего списка требований
        markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
        markup.add('Да', 'Нет')
        msg = bot.send_message(message.chat.id, f'{requirement}', reply_markup=markup)
        bot.register_next_step_handler(msg, lambda m: check(m, requirements))

def check(message, requirements):
    """
    Функция, которая обрабатывает ответ пользователя по каждому пункту
    Если ответ отрицательный, то кандидат не прошел отбор
    :param requirements: список пунктов с требованиями
    """
    if message.text == 'Да':
        ask_requirements(requirements, message)
    else:
        bot.send_message(message.chat.id, 'Это печально, но по результатам тестового задания мы вынуждены '
                                          'расстаться... Но твои контакты будут добавлены в нашу базу данных, '
                                          'к которым мы обязательно обратимся при необходимости в найме.\n'
                                          'Успехов тебе и побольше вензелей с вишней!')

def choose_date(message):
    """
    Функция, которая записывает пользователя в БД с расписанием
    """
    date = message.text
    day, time = date.split(', ')
    add_to_calendar(day, time, message.chat.id)  # добавляем в БД
    bot.send_message(message.chat.id, f'Супер! Ждем тебя {day} в {time}')

@bot.message_handler(func=lambda m: True)
def send_common_msg(message):
    """
    Функция, которая выводит сообщение, если пользователь ввел что-то не то
    """
    bot.send_message(message.chat.id, 'Дружок, ты что-то потерялся. Нажми на /help, чтобы найти то, что тебе нужно')

@app.route('/', methods=['GET', 'HEAD'])
def index():
    return 'ok'

@app.route(WEBHOOK_URL_PATH, methods=['POST'])
def webhook():
    if flask.request.headers.get('content-type') == 'application/json':
        json_string = flask.request.get_data().decode('utf-8')
        update = telebot.types.Update.de_json(json_string)
        bot.process_new_updates([update])
        return ''
    else:
        flask.abort(403)


from skfuzzy.cluster import cmeans
import matplotlib.pyplot as plt
from sklearn import datasets
import numpy as np
import pandas as pd
import seaborn as sns

def process_data(data, centers):
    cntr, u, u0, d, jm, p, fpc = cmeans(data, centers, 2, error=0.005, maxiter=10000, init=None)
    cluster_membership = np.argmax(u, axis=0)

    data = pd.DataFrame(data=data)
    data['target'] = cluster_membership

    g = sns.FacetGrid(data, hue='target', palette='tab20', size=5)
    g.map(plt.scatter, 0, 1, s=100, linewidth=.5, edgecolor='white')
    g.add_legend()
    # for i in range(centers):
    #     plt.scatter(data[cluster_membership == i])
    # for pt in cntr:
    #     plt.plot(pt[0], pt[1], 'rs')
    plt.show()

def main():
    n_samples = 10000
    random_state = 170
    transformation = [[0.6, -0.6], [-0.4, 0.8]]

    models = [
        {
            'name': 'Far Blobs',
            'X':
                datasets.make_blobs(n_samples=n_samples, centers=25, random_state=0, center_box=(-10000, 10000),
                                    cluster_std=50)[0],
            'k': 25
        },
        {
            'name': 'Noisy Circles',
            'X':
                datasets.make_circles(n_samples=n_samples, factor=.5, noise=.05)[0],
            'k': 2
        },
        {
            'name': 'Noisy Moons',
            'X': datasets.make_moons(n_samples=n_samples, noise=.05)[0],
            'k': 2
        },
        {
            'name': 'Blobs',
            'X': datasets.make_blobs(n_samples=n_samples, random_state=8)[0],
            'k': 3
        },
        {
            'name': 'No structure',
            'X': np.random.rand(n_samples, 2),
            'k': 2
        },
        {
            'name': 'Anisotropicly distributed data',
            'X': np.dot(datasets.make_blobs(n_samples=n_samples, random_state=random_state)[0], transformation),
            'k': 2
        },
        {
            'name': 'Blobs with varied variances',
            'X': datasets.make_blobs(n_samples=n_samples,
                                     cluster_std=[1.0, 2.5, 0.5],
                                     random_state=random_state)[0],
            'k': 2
        },
        {
            'name': '2 features, 1 informative, 1 cluster',
            'X': datasets.make_classification(n_samples=500, n_features=2, n_redundant=0, n_informative=1,
                                              n_clusters_per_class=1)[0],
            'k': 2
        },
        {
            'name': '2 features, 2 informative, 1 cluster',
            'X': datasets.make_classification(n_samples=500, n_features=2, n_redundant=0, n_informative=2,
                                              n_clusters_per_class=1)[0],
            'k': 3
        },
        {
            'name': '2 features, 2 informative',
            'X': datasets.make_classification(n_samples=500, n_features=2, n_redundant=0, n_informative=2)[0],
            'k': 2
        },
        {
            'name': '2 features, 2 informative, 2 cluster, 3 classes',
            'X': datasets.make_classification(n_samples=500, n_features=2, n_redundant=0, n_informative=2,
                                              n_clusters_per_class=1, n_classes=3)[0],
            'k': 3
        },
        {
            'name': '2 features, 5 centers',
            'X': datasets.make_blobs(n_samples=500, n_features=2, centers=5)[0],
            'k': 4,
            'dist_metric': 'manhattan'
        },
        {
            'name': '2 features, 6 classes',
            'X': datasets.make_gaussian_quantiles(n_samples=500, n_features=2, n_classes=6)[0],
            'k': 6
        },
        {
            'name': 'Circles',
            'X': datasets.make_circles(n_samples=500, factor=0.5)[0],
            'k': 4,
            'dist_metric': 'manhattan'
        },
    ]

    for model in models:
        process_data(model['X'], model['k'])

if __name__ == '__main__':
    main()


import tensorflow as tf
import numpy as np
import random
import time

#step = 1
a = 49000
batch_size = 49000
# = 63816
#atch_size = 31908
# dropout (keep_prob) rate  0.7~0.5 on training, but should be 1 for testing
#keep_prob = tf.placeholder(tf.float32)

train = np.loadtxt('train_PF.csv', delimiter = ',', dtype = np.float32)
test = np.loadtxt('test_PF.csv', delimiter = ',', dtype = np.float32)
P = np.loadtxt('merge_print_PF.csv', delimiter = ',', dtype = np.float32)

x_data = train[1:49001,1:-1]
x_data = x_data.reshape([-1,8,8,4])
#x_data = x_data.reshape(-1,8,8,4)
max_x_0 = np.max(x_data[:,:,:,0]); min_x_0 = np.min(x_data[:,:,:,0])
max_x_1 = np.max(x_data[:,:,:,1]); min_x_1 = np.min(x_data[:,:,:,1])
max_x_2 = np.max(x_data[:,:,:,2]); min_x_2 = np.min(x_data[:,:,:,2])
max_x_3 = np.max(x_data[:,:,:,3]); min_x_3 = np.min(x_data[:,:,:,3])

norm0=(x_data[:,:,:,0] - min_x_0)/(max_x_0 - min_x_0)
norm1=(x_data[:,:,:,1] - min_x_1)/(max_x_1 - min_x_1)
norm2=(x_data[:,:,:,2] - min_x_2)/(max_x_2 - min_x_2)
norm3=(x_data[:,:,:,3] - min_x_3)/(max_x_3 - min_x_3)
x_data[:,:,:,0]=norm0
x_data[:,:,:,1]=norm1
x_data[:,:,:,2]=norm2
x_data[:,:,:,3]=norm3
#x_data=x_data.reshape([-1,192])
x_data = x_data.reshape([-1,256])

x_test = test[1:7001,1:-1]
x_test = x_test.reshape([-1,8,8,4])
#x_test = x_test.reshape(-1,8,8,4)
norm0=(x_test[:,:,:,0] - min_x_0)/(max_x_0 - min_x_0)
norm1=(x_test[:,:,:,1] - min_x_1)/(max_x_1 - min_x_1)
norm2=(x_test[:,:,:,2] - min_x_2)/(max_x_2 - min_x_2)
norm3=(x_test[:,:,:,3] - min_x_3)/(max_x_3 - min_x_3)
x_test[:,:,:,0]=norm0
x_test[:,:,:,1]=norm1
x_test[:,:,:,2]=norm2
x_test[:,:,:,3]=norm3
x_test=x_test.reshape([-1,256])

x_P = P[1:14001,1:-1]
x_P = x_P.reshape([-1,8,8,4])
norm0=(x_P[:,:,:,0] - min_x_0)/(max_x_0 - min_x_0)
norm1=(x_P[:,:,:,1] - min_x_1)/(max_x_1 - min_x_1)
norm2=(x_P[:,:,:,2] - min_x_2)/(max_x_2 - min_x_2)
norm3=(x_P[:,:,:,3] - min_x_3)/(max_x_3 - min_x_3)
x_P[:,:,:,0]=norm0
x_P[:,:,:,1]=norm1
x_P[:,:,:,2]=norm2
x_P[:,:,:,3]=norm3
x_P=x_P.reshape([-1,256])
#x_data_tf=tf.reshape(x_data, [-1,192])
y_data = train[1:49001,[-1]]
y_data = y_data.reshape([-1,1])
max_y = np.max(y_data)
min_y = np.min(y_data)
y_data = (y_data - min_y)/(max_y - min_y)

y_test = test[1:7001,[-1]]

y_test = y_test.reshape([-1,1])
y_test = (y_test - min_y)/(max_y - min_y)

y_P = P[1:14001,[-1]]
y_P = y_P.reshape([-1,1])
y_P = (y_P - min_y)/(max_y - min_y)
# input place holders
X = tf.placeholder(tf.float32, [None, 256])
X_img = tf.reshape(X, [-1, 8, 8, 4])
Y = tf.placeholder(tf.float32, [None, 1])
#Max_Y = tf.placeholder(tf.float32)
#Min_Y = tf.placeholder(tf.float32)

# L1 ImgIn shape=(?, 8, 8, 3)
W1 = tf.Variable(tf.random_normal([3, 3, 4, 64], stddev=0.01), name = 'pf_w1')
#    Conv     -> (?, 8, 8, 10)
#    Pool     -> (?, 4, 4, 10)
L1 = tf.nn.conv2d(X_img, W1, strides=[1, 1, 1, 1], padding='SAME', name = 'pf_conv1')
L1 = tf.nn.relu(L1, name = 'pf_relu1')

W2 = tf.Variable(tf.random_normal([3, 3, 64, 64], stddev=0.01), name = 'pf_w2')
L2 = tf.nn.conv2d(L1, W2, strides=[1, 1, 1, 1], padding='SAME', name = 'pf_conv2')
L2 = tf.nn.relu(L2, name = 'pf_relu2')

W3 = tf.Variable(tf.random_normal([3, 3, 64, 64], stddev=0.01), name = 'pf_w3')
L3 = tf.nn.conv2d(L2, W3, strides=[1, 1, 1, 1], padding='SAME', name = 'pf_conv3')
L3 = tf.nn.relu(L3, name = 'pf_relu3')

#L3 = tf.nn.max_pool(L3, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
#L3 = tf.nn.dropout(L3, keep_prob=keep_prob)

W4 = tf.Variable(tf.random_normal([3, 3, 64, 64], stddev=0.01), name = 'pf_w4')
#    Conv     -> (?, 8, 8, 10)
#    Pool     -> (?, 4, 4, 10)
L4 = tf.nn.conv2d(L3, W4, strides=[1, 1, 1, 1], padding='SAME', name = 'pf_conv4')
L4 = tf.nn.relu(L4, name = 'pf_relu4')

L4 = tf.nn.max_pool(L4, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name = 'pf_pool1')

W5 = tf.Variable(tf.random_normal([3, 3, 64,128], stddev=0.01), name = 'pf_w5')
L5 = tf.nn.conv2d(L4, W5, strides=[1, 1, 1, 1], padding='SAME', name = 'pf_conv5')
L5 = tf.nn.relu(L5, name = 'pf_relu5')

W6 = tf.Variable(tf.random_normal([3, 3,128,128], stddev=0.01), name = 'pf_w6')
L6 = tf.nn.conv2d(L5, W6, strides=[1, 1, 1, 1], padding='SAME', name = 'pf_conv6')
L6 = tf.nn.relu(L6, name = 'pf_relu6')

W7 = tf.Variable(tf.random_normal([3, 3,128,128], stddev=0.01), name = 'pf_w7')
L7 = tf.nn.conv2d(L6, W7, strides=[1, 1, 1, 1], padding='SAME', name = 'pf_conv7')
L7 = tf.nn.relu(L7, name = 'pf_relu7')

W8 = tf.Variable(tf.random_normal([3, 3,128,128], stddev=0.01), name = 'pf_w8')
L8 = tf.nn.conv2d(L7, W8, strides=[1, 1, 1, 1], padding='SAME', name = 'pf_conv8')
L8 = tf.nn.relu(L8, name = 'pf_relu8')

W9 = tf.Variable(tf.random_normal([3, 3,128,128], stddev=0.01), name = 'pf_w9')
L9 = tf.nn.conv2d(L8, W9, strides=[1, 1, 1, 1], padding='SAME', name = 'pf_conv9')
L9 = tf.nn.relu(L9, name = 'pf_relu9')

L9 = tf.nn.max_pool(L9, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name = 'pf_pool2')
#L6 = tf.nn.dropout(L6, keep_prob=keep_prob)
L9_flat = tf.reshape(L9, [-1,2*2*128], name = 'pf_flat')

# Final FC 2x2x90 inputs -> 1 outputs
W10 = tf.get_variable("pf_w10", shape=[2*2*128,256], initializer=tf.contrib.layers.xavier_initializer())
b10 = tf.Variable(tf.random_normal([256]), name = 'pf_b10')
L10 = tf.nn.relu(tf.matmul(L9_flat, W10) + b10, name = 'pf_relu10')
#L7 = tf.nn.dropout(L7, keep_prob=keep_prob)

W11 = tf.get_variable("pf_w11", shape=[256, 128], initializer=tf.contrib.layers.xavier_initializer())
b11 = tf.Variable(tf.random_normal([128]), name = 'pf_b11')
L11 = tf.nn.relu(tf.matmul(L10, W11) + b11, name = 'pf_relu11')
#L8 = tf.nn.dropout(L8, keep_prob=keep_prob)

W12 = tf.get_variable("pf_w12", shape=[128, 64], initializer=tf.contrib.layers.xavier_initializer())
b12 = tf.Variable(tf.random_normal([64]), name = 'pf_b12')
L12 = tf.nn.relu(tf.matmul(L11, W12) + b12, name = 'pf_relu12')
#L9 = tf.nn.dropout(L9, keep_prob=keep_prob)

W13 = tf.get_variable("pf_w13", shape=[64,1], initializer=tf.contrib.layers.xavier_initializer())
b13 = tf.Variable(tf.random_normal([1]), name = 'pf_b13')

hypothesis = tf.matmul(L12, W13) + b13
regularizers = tf.nn.l2_loss(W1) + tf.nn.l2_loss(W2) + tf.nn.l2_loss(W3) + tf.nn.l2_loss(W4) + tf.nn.l2_loss(W5) +\
               tf.nn.l2_loss(W6) + tf.nn.l2_loss(W7) + tf.nn.l2_loss(W8) + tf.nn.l2_loss(W9) + tf.nn.l2_loss(W10) +\
               tf.nn.l2_loss(W11) + tf.nn.l2_loss(W12) + tf.nn.l2_loss(W13)
cost = tf.reduce_mean(tf.square(hypothesis - Y))

# define cost/loss & optimizer
optimizer = tf.train.AdamOptimizer(learning_rate=0.0001).minimize(cost)
#train = optimizer.minimize(cost)
r=random.randint(0,154)
diff = (hypothesis - Y)*100000
rms = tf.sqrt(tf.reduce_mean(tf.square(diff)))
diff_max = tf.reduce_max(diff)
diff_min = tf.reduce_min(diff)

per = (hypothesis - Y)/(Y + min_y/(max_y - min_y))*100
rms_per = tf.sqrt(tf.reduce_mean(tf.square(per)))
per_max = tf.reduce_max(per)
per_min = tf.reduce_min(per)

# Launch the graph in a session.
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 1
sess = tf.Session()#config=config)
# Initializes global variables in the graph.
start_time = time.time()
sess.run(tf.global_variables_initializer())
# Class used to save and/or restore Tensor Variables
saver = tf.train.Saver(max_to_keep=30)

out = open('M1NR_1batch_0.0001_9L_64W_reg_off_FC_rev.out','w')
#out=open('ML_9layer.out','a')
#out.write('#-----------------------------------------------------------------------------------\n')
#out.write('#-----------------------------------------------------------------------------------\n')
#out.write('#-----------------------------------------------------------------------------------\n')
out.close ()

a=np.logspace(1, 4, 100, endpoint=True, dtype=int)
b=np.logspace(4, 6, 200, endpoint=True, dtype=int)
c=np.logspace(6, 6.30103, 41, endpoint=True, dtype=int)

stop_count = 0
stop_cost = 20

for epoch in range(50000):
    step = epoch

    p = np.random.permutation(len(x_data))
    x_data=x_data[p]
    y_data=y_data[p]
    total_batch = 1
    avg_cost_test = avg_cost_train = 0

    for i in range(total_batch):
        cost_val, hy_val, _ = sess.run(
        [cost, hypothesis, optimizer], feed_dict = {X: x_data[i*batch_size:(i+1)*batch_size], Y: y_data[i*batch_size:(i+1)*batch_size]})
    if epoch % 10 == 0:
        rms_per_train = sess.run(rms_per, feed_dict={X: x_data, Y: y_data})

        cost_test, rms_test, diff_max_test, diff_min_test, rms_per_test, per_max_test, per_min_test = sess.run(
        [cost, rms, diff_max, diff_min, rms_per, per_max, per_min], feed_dict={X: x_test, Y: y_test})

        avg_cost_train += cost_val / total_batch
        avg_cost_test += cost_test / total_batch

        out = open('M1NR_1batch_0.0001_9L_64W_reg_off_FC_rev.out','a')
        print(epoch,'epoch' , ' cost_train', avg_cost_train, "cost_test ", avg_cost_test, per_max_test, per_min_test, rms_per_test, rms_per_train)
        out.write('%d Cost_train %e  Cost_test %e %e %e %e %e \n' %(epoch,cost_val,cost_test,per_max_test,per_min_test,rms_per_test, rms_per_train))
        out.close ()
        saver.save(sess, 'model/M1NR_1batch_0.0001_9L_64W_reg_off_FC_rev/M1NR_1batch_0.0001_9L_64W_reg_off_FC_rev', global_step=epoch, write_meta_graph=False)

        if stop_cost >= cost_test:
            stop_cost = cost_test
            stop_count = 0
        else:
            stop_count += 1

        if stop_count == 20:
            break

print('Trained Model Saved.')
print("train: cost    per_max     per_min     rms_per    \n    ", sess.run([cost, per_max, per_min, rms_per], feed_dict={X: x_data, Y: y_data}))

print("test : cost    per_max     per_min     rms_per    \n    ", sess.run([cost, per_max, per_min, rms_per], feed_dict={X: x_test, Y: y_test}))
print("time elapsed: {:.2f}s".format(time.time() - start_time))
print_hypo = hypothesis * (max_y - min_y) + min_y
print_Y = Y * (max_y - min_y) + min_y

saver.restore(sess, 'model/M1NR_1batch_0.0001_9L_64W_reg_off_FC_rev/M1NR_1batch_0.0001_9L_64W_reg_off_FC_rev-%d' %(step))
out = open('M1NR_1batch_0.0001_9L_64W_reg_off_FC_rev_diff_%d.out' %(step),'w')

cost_test, hypo_, Y_, rms_per_test, per_max_test, per_min_test = sess.run(
[cost, print_hypo, print_Y, rms_per, per_max, per_min], feed_dict={X: x_P, Y: y_P})

cost_val, hy_val, rms_per_train, per_max_train, per_min_train = sess.run(
[cost, print_hypo, rms_per, per_max, per_min], feed_dict = {X: x_data, Y: y_data})

for i in range(1):
       out = open('M1NR_1batch_0.0001_9L_64W_reg_off_FC_rev_diff_%d.out' %(step) ,'a')
       out.write('hypo %d : %e PF %d : %e\n'%(i,hypo_[i],i,Y_[i]))
       out.close()

for i in range(1):
       print('hypo %d : %e PF %d : %e\n'%(i,hypo_[i],i,Y_[i]))

print(' cost_train', cost_val, "cost_test ", cost_test, per_max_test, per_min_test, rms_per_test)
print(' hy_train :', hy_val)
print(' hy_test :', hypo_)
print("train: cost    per_max     per_min     rms_per    \n    [", cost_val, per_max_train, per_min_train, rms_per_train,"]")
print("test : cost    per_max     per_min     rms_per    \n    [", cost_test, per_max_test, per_min_test, rms_per_test,"]")
#out = open('CNN_opt_test_2batch_0.0001.out','a')
#out.write('time elapsed:  {:.2f}s \n'.format(time.time() - start_time))
#out.close ()


from setuptools import setup, Extension
from torch.utils import cpp_extension

torch_library_paths = cpp_extension.library_paths(cuda=False)

setup(name='rephine_mt',
      ext_modules=[cpp_extension.CppExtension('rephine_mt', ['ph/rephine_mt_cpu.cpp'], extra_link_args=[
                    '-Wl,-rpath,' + library_path
                    for library_path in torch_library_paths])],
      cmdclass={'build_ext': cpp_extension.BuildExtension})


from bs4 import BeautifulSoup
from urllib.request import urlopen
import re
import ssl
import os
import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
import codecs

def checkChar(token):
    for char in token:
        if (0 <= ord(char) and ord(char) <= 64) or (91 <= ord(char) and ord(char) <= 96) or (123 <= ord(char)):
           return False
        else:
            continue
    return True

def cleanMe(html):
    soup = BeautifulSoup(html, "html.parser")                                                       # create a new bs4 object from the html data loaded
    for script in soup(["script", "style"]):                                                        # remove all javascript and stylesheet code
        script.extract()
    # get text
    text = soup.get_text()
    # break into lines and remove leading and trailing space on each
    lines = (line.strip() for line in text.splitlines())
    # break multi-headlines into a line each
    chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
    # drop blank lines
    text = '\n'.join(chunk for chunk in chunks if chunk)
    return text

path = 'crawled_html_pages/'
index = {}
docNum = 0
stop_words = set(stopwords.words('english'))                                                            #collection of stopwords to avoid
for filename in os.listdir(path):                                                                       #for every doc in folder...
    collection = {}                                                                                #all words collected from this html file
    docNum += 1                                                                                #Start with doc 1, then 2..3..4..500
    file = codecs.open('crawled_html_pages/'+ filename,'r', 'utf-8')                                  #open the html file
    page_text = cleanMe(file)                                                                         #collect page text from html file
    tokens = nltk.word_tokenize(page_text)                                                              #tokenize page text
    filtered_sentence = [w for w in tokens if not w in stop_words]                                      #remove stopwords from page text
    filtered_sentence = []

    breakWord = ''

    for w in tokens:                                                                                #place non-stopwords in token list
        if w not in stop_words:
            filtered_sentence.append(w.lower())

    for token in filtered_sentence:
        if len(token) == 1 or token == 'by':
            continue
        if checkChar(token) == False:
            continue
        if token == 'australia':
            breakWord = token
            continue
        if token == 'bangladesh' and breakWord == 'australia':
            break
        if token not in collection:
            collection[token] = 0
        collection[token] += 1

    for token in collection:
        if token not in index:
            index[token] = ''
        index[token] = index[token] + '['+ str(docNum) + ', ' + str(collection[token]) + ']'

f = open('index.txt', 'w')
vocab = open('uniqueWords.txt', 'w')
for term in index:
    f.write(term + ' => ' + index[term])
    vocab.write(term + '\n')
    f.write('\n')
f.close()
vocab.close()

print('Finished...')


from celery.task import task
from numpy import mean, std, sqrt, fabs
from pandas import DataFrame

@task
def task_cohen(dataOne, dataTwo):

    labelOne = 'column1'
    labelTwo = 'column2'

    # Create input
    df = DataFrame.from_items([(labelOne, dataOne), (labelTwo, dataTwo)])

    meanx = mean(df[labelOne])
    meany = mean(df[labelTwo])
    sdx = std(df[labelOne])
    sdy = std(df[labelTwo])
    s = sqrt((sdx**2 + sdy**2)/2)
    d = round(fabs(meanx-meany)/s, 4)

    return d


def mod_exp(a: int, b: int, m: int) -> int:
    """
    高速なmod演算を用いてa^b(mod m)を計算します。
    Args:
        a: 底数。
        b: 指数。
        m: 除数。

    Returns:
        a^b (mod m) の結果。
    """

    result = 1
    while b > 0:
        if b % 2 == 1:
            result = (result * a) % m
        a = (a * a) % m
        b = b // 2
    return result


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import mpl_toolkits.mplot3d.axes3d as p3
import matplotlib.animation as animation

#Reading the input instructions and names of the objects.
obj_names=[]
f = open("../run/input.dat","r")
obj_count=int(f.readline())
for obj in range(obj_count):
    obj_names.append(f.readline().split()[0])
step=float(f.readline())
sim_length=float(f.readline())
step_count=float(f.readline())
if step==0:
    step=sim_length/step_count
elif sim_length==0:
    sim_length=step*step_count
else:
    step_count=sim_length/step
f.close()
obj_names=sorted(obj_names)

#Reading the results and sorting them first by name and then by index (time)
sim_results=pd.read_csv("../run/output.dat")
sim_results["indCol"]=sim_results.index
sim_results=sim_results.sort_values(["Object","indCol"])

#Number of iterations
iters=int((sim_results.count()/obj_count)["indCol"])

#Visualization initialization
fig=plt.figure()
ax=p3.Axes3D(fig)
ax.set_xlim3d([sim_results["X"].min(),sim_results["X"].max()])
ax.set_xlabel("X")
ax.set_ylim3d([sim_results["Y"].min(),sim_results["Y"].max()])
ax.set_ylabel("Y")
ax.set_zlim3d([sim_results["Z"].min(),sim_results["Z"].max()])
ax.set_zlabel("Z")
ax.set_title("SC2 Planetary Motion Simulation")
ax.view_init(40,10)

#Initial locations
scatters= [ax.scatter(sim_results["X"].iloc[i*iters],sim_results["Y"].iloc[i*iters],sim_results["Z"].iloc[i*iters],label=str(obj_names[i])) for i in range(obj_count)]

#Change of location with each iteration
def animate_scatters(iteration, data, scatters):
    for i in range(obj_count):
        scatters[i]._offsets3d = ([data["X"].iloc[i*iters+iteration]],
                                    [data["Y"].iloc[i*iters+iteration]],
                                    [data["Z"].iloc[i*iters+iteration]])
    return scatters

#Animation
ani=animation.FuncAnimation(fig,animate_scatters,iters,fargs=(sim_results,scatters),interval=10,blit=False,repeat=False)

#Option for saving the animation.
#ani.save('../run/animation.mp4', fps=30, extra_args=['-vcodec', 'libx264'])

#Drawing the animation
plt.legend()
plt.show()


from django.db.models.signals import post_save
from django.dispatch import receiver
from channels.layers import get_channel_layer
from asgiref.sync import async_to_sync

from .models import Module, ProjectRespond

channel_layer = get_channel_layer()

@receiver(post_save, sender=Module)
def activities(sender, instance, *args, **kwargs):
    async_to_sync(channel_layer.group_send)(instance.profession.group_name, {
        "type": "prochan.message",
        "profession_id": instance.profession.id,
        "username": instance.project.customer.username,
        "message": instance.title,
        "module_id": instance.id,
    })

@receiver(post_save, sender=ProjectRespond)
def activities(sender, instance, *args, **kwargs):
    async_to_sync(channel_layer.group_send)(instance.module.group_name, {
        "type": "module.respond",
        "username": instance.responded.username,
        "message": str(instance.cost),
        "module_id": instance.module.id,
    })


import numpy as np

class Normalizer:

    @staticmethod
    def normalize_vector_points(vector: np.ndarray, r_to: float = 1, r_from: float = 0) -> np.ndarray:
        min_val = np.min(vector)
        max_val = np.max(vector)
        normalized_vector = np.interp(vector, (min_val, max_val), (r_from, r_to))
        return normalized_vector


from __future__ import annotations
from PIL import Image
import torch
from torchvision import datasets
from torchvision import transforms
from typing import Tuple
from torch.utils.data import Dataset, DataLoader
from torch import Tensor
import matplotlib.pyplot as plt
from torch import optim
from torch import nn
import torch.nn.functional as F
import numpy as np
from tqdm import tqdm
import pandas as pd
from typing import Optional
import sys

def main() -> None:
    (train_set, test_set) = get_dataset()
    check_data(train_set)
    train_loader, test_loader = get_dataloader(train_set, test_set)
    print(len(train_loader))
    print(len(test_loader))
    images, labels = iter(train_loader).next()
    print(labels.shape)
    print(torch.max(labels, 0)[0])
    print(torch.min(labels, 0)[0])

    device = get_device()
    print(device)

    net = Net(100).to(device)
    criterion = nn.CrossEntropyLoss()
    optimizer = optim.Adam(net.parameters(), lr=0.001)

    history = train(train_loader, test_loader, optimizer,
                    criterion, net, device)
    print(history.shape)
    torch.save(net.state_dict(), "param.pt")
    pd.to_pickle(history, "history.pkl")

Loader = DataLoader[Tuple[Tensor, Tensor]]

def train(train_loader: Loader, test_loader: Loader, optimizer: optim.Adam, criterion: nn.CrossEntropyLoss, net: Net, device, num_epoch: int = 20) -> np.ndarray:
    batch_size: int = iter(train_loader).next()[0].shape[0]
    history = np.zeros(5)
    for epoch in range(num_epoch):
        train_acc, train_loss = 0.0, 0.0  # train data
        val_acc, val_loss = 0.0, 0.0  # validation data
        num_train, num_test = 0, 0  # the number of data

        inputs: Tensor
        labels: Tensor
        for inputs, labels in tqdm(train_loader):
            num_train += len(labels)

            inputs = inputs.to(device)
            labels = labels.to(device)

            optimizer.zero_grad()
            outputs = net(inputs).to(device)
            loss: Tensor = criterion(outputs, labels)
            loss.backward()

            optimizer.step()
            _, predicted = torch.max(outputs, 1)

            train_loss += loss.item()
            train_acc += (predicted == labels).sum().item()

        inputs_test: Tensor
        labels_test: Tensor
        for inputs_test, labels_test in test_loader:
            num_test += len(labels_test)
            inputs_test = inputs_test.to(device)
            labels_test = labels_test.to(device)

            outputs_test = net(inputs_test)
            loss_test: Tensor = criterion(outputs_test, labels_test)
            _, predicted_test = torch.max(outputs_test, 1)

            val_loss += loss_test.item()
            val_acc += (predicted_test == labels_test).sum().item()

        train_acc = train_acc / num_train
        val_acc = val_acc / num_test
        train_loss = train_loss * batch_size / num_train
        val_loss = val_loss * batch_size / num_test

        print(
            f"Epoch [{epoch + 1}/{num_epoch}], loss: {train_loss:.5f}, acc: {train_acc:.5f}, val_loss: {val_loss:.5f}, val_acc: {val_acc:.5f}"
        )
        items = np.array(
            [epoch + 1, train_loss, train_acc, val_loss, val_acc]
        )
        history = np.vstack((history, items))
    # [[index, train_loss, train_acc, val_loss, val_acc]], len(history) == num_epoch + 1, len(history[0]) == 5
    return history

class Net(nn.Module):
    def __init__(self, kind:int = 10) -> None:
        super().__init__()
        self.conv1 = nn.Conv2d(
            in_channels=3, out_channels=32, kernel_size=(3, 3), padding="same"
        )
        self.conv2 = nn.Conv2d(
            in_channels=32, out_channels=32, kernel_size=(3, 3), padding="same"
        )
        self.pool = nn.MaxPool2d(kernel_size=2, stride=2)
        self.drop1 = nn.Dropout2d(p=0.25)

        self.conv3 = nn.Conv2d(
            in_channels=32, out_channels=64, kernel_size=(3, 3), padding="same"
        )
        self.conv4 = nn.Conv2d(
            in_channels=64, out_channels=64, kernel_size=(3, 3), padding="same"
        )
        self.drop2 = nn.Dropout2d(p=0.25)

        self.fc1 = nn.Linear(in_features=64*32*32, out_features=512)
        self.drop3 = nn.Dropout(p=0.5)
        self.fc2 = nn.Linear(in_features=512, out_features=kind)

    def forward(self, x: Tensor) -> Tensor:
        x = F.relu(self.conv1(x))
        x = F.relu(self.conv2(x))
        x = self.drop1(x)

        x = F.relu(self.conv3(x))
        x = F.relu(self.conv4(x))
        x = self.drop2(x)
        x = torch.flatten(x, 1)

        x = F.relu(self.fc1(x))
        x = self.drop3(x)
        x = self.fc2(x)

        return x

def get_dataset() -> Tuple[Dataset[Tuple[Tensor, int]],
                           Dataset[Tuple[Tensor, int]]]:
    data_root = "./data"

    transform: transforms.Compose = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize(0.5, 0.5),
    ])

    train_set: Dataset[Tuple[Tensor, int]] = datasets.CIFAR100(
        root=data_root,
        train=True,
        download=True,
        transform=transform
    )

    test_set = datasets.CIFAR100(
        root=data_root,
        train=True,
        download=False,
        transform=transform
    )

    return (train_set, test_set)

def get_dataloader(train_set: Dataset[Tuple[Tensor, int]], test_set: Dataset[Tuple[Tensor, int]],
                   batch_size=128) -> Tuple[DataLoader[Tuple[Tensor, Tensor]], DataLoader[Tuple[Tensor, Tensor]]]:
    train_loader: DataLoader[Tuple[Tensor, Tensor]] = DataLoader(
        train_set, batch_size=batch_size, shuffle=True)  # type: ignore

    test_loader: DataLoader[Tuple[Tensor, Tensor]] = DataLoader(
        test_set, batch_size=batch_size, shuffle=False)  # type: ignore

    return (train_loader, test_loader)

def check_data(data: Dataset[Tuple[Tensor, int]]):
    for i in range(10):
        image, _ = data[i]
        plt.subplot(2, 5, i + 1)
        # (rgb, height, length) -> (length, height, rgb) -> (height, length, rgb)
        plt.imshow(image.transpose(0, 2).transpose(0, 1))
    plt.savefig("pic.png")

def get_device() -> torch.device:
    return torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    # return torch.device("cuda:0" if False else "cpu")

def check_history():
    # [[index, train_loss, train_acc, val_loss, val_acc]], len(history) == num_epoch + 1, len(history[0]) == 5
    history: np.ndarray = pd.read_pickle("history.pkl")
    plt.plot(history[1:, 0], history[1:, 1], "b", label="train")
    plt.plot(history[1:, 0], history[1:, 3], "k", label="test")
    plt.xlabel("epoch")
    plt.ylabel("loss")
    plt.title("loss curve")
    plt.legend()
    plt.savefig("loss_curve.png")
    plt.close()

    plt.plot(history[:, 0], history[:, 2], "b", label="train")
    plt.plot(history[:, 0], history[:, 4], "k", label="test")
    plt.xlabel("epoch")
    plt.ylabel("acc")
    plt.title("accuracy")
    plt.legend()
    plt.savefig("acc_curve.png")

    print(f"initial: loss: {history[1, 3]}, acc: {history[1, 4]}")
    print(f"last: loss: {history[-1, 3]}, acc: {history[-1, 4]}")
    print(f"eval mode acc: {check_acc()}")

def check_acc() -> float:
    device = get_device()
    net = load_model(device)
    _, test_loader = get_dataloader(*get_dataset())
    acc: float = 0.0
    num_test: int = 0
    images: Tensor
    labels: Tensor
    for images, labels in test_loader:
        num_test += len(labels)
        images = images.to(device)
        labels = labels.to(device)
        outputs = net(images)
        _, predicted = torch.max(outputs, 1)
        acc += (predicted == labels).sum().item()
    return acc / num_test

def check_model(test_loader: Optional[Loader] = None, net: Optional[Net] = None, index: int = 0):
    device = get_device()
    if net is None:
        net = load_model(device)
    if test_loader is None:
        _, test_loader = get_dataloader(*get_dataset())
    images, labels = list(iter(test_loader))[index]
    print(images.shape)
    images: Tensor = images.to(device)
    labels: Tensor = labels.to(device)
    outputs: Tensor = net(images)
    predicteds: Tensor = torch.max(outputs, 1)[1]

    acc = (labels == predicteds).sum().item() / len(labels)
    print(acc)
    if acc == 1:
        check_model(test_loader, net, index + 1)
        sys.exit(0)

    plt.figure(figsize=(30, 24))
    acc: float = 0
    for i in range(50):
        ax = plt.subplot(5, 10, i + 1)

        image = images[i].cpu()
        label = labels[i].cpu()
        predicted = predicteds[i].cpu()
        if (predicted == label):
            c = "k"
            acc += 1
        else:
            c = "b"

        plt.imshow(image.reshape(3, 32, 32).transpose(0, 2).transpose(0, 1))
        label_name = ["airplane", "automobile", "bird", "cat",
                      "deer", "dog", "frog", "horse", "ship", "truck"]
        ax.set_title(f"{label_name[label]}:{label_name[predicted]}", c=c)
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)
    acc = acc / 50
    print(f"acc: {acc}")
    plt.savefig("pred.png")

def check_my_image(path="my.png"):
    img = Image.open(path)
    img = img.resize((32, 32))
    print(np.array(img).shape)
    img = transforms.Normalize(0.5, 0.5)(transforms.ToTensor()(img))
    input = Tensor(img)
    device = get_device()
    print(input.shape)
    # (height, length, rgb) -> (rgb, height, length)
    # input = input.transpose(0, 2).transpose(1, 2)
    input = input[:3, ]  # for image that have 4 dim color
    plt.imshow(input.transpose(0, 2).transpose(0, 1))
    plt.savefig("conversion.png")
    print(input.shape)
    input = input.reshape(1, 3, 32, 32).to(device)
    print(input.shape)
    label_name = ["airplane", "automobile", "bird", "cat",
                  "deer", "dog", "frog", "horse", "ship", "truck"]
    net = load_model(device)
    output = net(input)
    pred = torch.max(output, 1)
    print(pred)
    print(label_name[pred[1]])

def load_model(device: torch.device) -> Net:
    net = Net(100).to(device)
    net.load_state_dict(torch.load("param.pt"))
    net.eval()
    return net

if __name__ == "__main__":

    # predict image in conv/`path`
    # check_my_image(my.jpg)

    # check acc in eval mode
    # check_model()

    # create png which was generated while train phase
    check_history()

    # train
    # main()


import sys
l=[]
for i in range(int(input())):
    n=float(sys.stdin.readline())
    if i>6:
        if l[-1]>n:
            l[-1]=n
            l.sort()
    else:
        l.append(n)
        l.sort()
for i in l:
    print('%.3f' % i)


from api.views import (APIFavorite, APIShoppingList, APIShoppingListDownload,
                       APIUserFollow, GetFollowViewSet, IngredientViewSet,
                       RecipeViewSet, TagViewSet)
from django.urls import include, path
from rest_framework.routers import DefaultRouter

router = DefaultRouter()

router.register('tags', TagViewSet, basename='tags')
router.register('ingredients', IngredientViewSet, basename='ingredients')
router.register('recipes', RecipeViewSet, basename='recipes')

urlpatterns = [
    path('users/subscriptions/', GetFollowViewSet.as_view({'get': 'list'})),
    path('users/<int:user_id>/subscribe/', APIUserFollow.as_view()),
    path('recipes/<int:pk>/favorite/', APIFavorite.as_view()),
    path('recipes/<int:pk>/shopping_cart/', APIShoppingList.as_view()),
    path('recipes/download_shopping_cart/', APIShoppingListDownload.as_view()),
    path('', include('djoser.urls')),
    path('auth/', include('djoser.urls.authtoken')),
    path('', include(router.urls)),
]


from typing import List, Optional

import pymorphy2
from pymorphy2.tagset import OpencorporaTag
from spacy.lang.ru import Russian as SpacyRussian

from documental.token import WordToken, Token
from omniglot.mul.numbers import combine_numbers
from omniglot.mul.punctuation import convert_punctuation_tokens
from omniglot.mul.whitespace import remove_extra_whitespace
from omnilingual import LanguageCode, PartOfSpeech
from omnilingual.features import (
    Animacy,
    Aspect,
    Case,
    Features,
    Gender,
    Mood,
    Number,
    Person,
    Tense,
    Voice,
)

from ..parser import NaturalLanguageProcessor

class RussianParser(NaturalLanguageProcessor):
    def __init__(self):
        super().__init__()

        self.nlp = SpacyRussian()
        self.morphy = pymorphy2.MorphAnalyzer()

    def process(self, text: str) -> List[Token]:
        tokens: List[Token] = self.tokenize(text)

        tokens = convert_punctuation_tokens(tokens)
        tokens = combine_numbers(tokens)
        tokens = remove_extra_whitespace(tokens)

        return tokens

    def supported_languages(self) -> List[LanguageCode]:
        return [LanguageCode.Russian]

    def tokenize(self, text: str) -> List[Token]:
        tokens: List[Token] = []

        for token in self.nlp(text):
            parse = self.morphy.parse(token.text)[0]

            tag: OpencorporaTag = parse.tag

            print(tag)

            word = WordToken(
                language=LanguageCode.Russian,
                text=token.text,
                lemma=parse.normal_form,
                pos=self.convert_pos(tag.POS),
                features=self.convert_features(tag),
            )

            tokens.append(word)

        return tokens

    def convert_pos(self, pos: Optional[str]) -> PartOfSpeech:
        if pos is None:
            return PartOfSpeech.Nil
        elif pos == "NOUN":
            return PartOfSpeech.Noun
        elif pos in ["ADJF", "ADJS"]:
            return PartOfSpeech.Adjective
        elif pos == "COMP":
            return PartOfSpeech.Comparative
        elif pos in ["VERB", "INFN", "PRTF", "PRTS", "GRND"]:
            return PartOfSpeech.Verb
        elif pos == "NUMR":
            return PartOfSpeech.Number
        elif pos == "ADVB":
            return PartOfSpeech.Adverb
        elif pos == "NPRO":
            return PartOfSpeech.Noun
        elif pos == "PRED":
            return PartOfSpeech.Predicate
        elif pos == "PREP":
            return PartOfSpeech.Adposition
        elif pos == "CONJ":
            return PartOfSpeech.Conjunction
        elif pos == "PRCL":
            return PartOfSpeech.Particle
        elif pos == "INTJ":
            return PartOfSpeech.Interjection
        else:
            raise NotImplementedError("Unexpected Russian POS tag %s" % (pos))

    def convert_features(self, tag: OpencorporaTag) -> Features:
        animacy: Optional[Animacy] = None
        aspect: Optional[Aspect] = None
        case: Optional[Case] = None
        gender: Optional[Gender] = None
        mood: Optional[Mood] = None
        number: Optional[Number] = None
        person: Optional[Person] = None
        tense: Optional[Tense] = None
        voice: Optional[Voice] = None

        if tag.animacy == "anim":
            animacy = Animacy.Anim
        elif tag.animacy == "inan":
            animacy = Animacy.Inan

        if tag.aspect == "perf":
            aspect = Aspect.Perf
        elif tag.aspect == "impf":
            aspect = Aspect.Imp

        if tag.case == "nomn":
            case = Case.Nom
        elif tag.case in ["gent", "gen1", "gen2"]:
            case = Case.Gen
        elif tag.case == "datv":
            case = Case.Dat
        elif tag.case in ["accs", "acc1", "acc2"]:
            case = Case.Acc
        elif tag.case == "ablt":
            case = Case.Abl
        elif tag.case in ["loct", "loc1", "loc2"]:
            case = Case.Loc
        elif tag.case == "voct":
            case = Case.Voc

        if tag.gender == "masc":
            gender = Gender.Masc
        elif tag.gender == "femn":
            gender = Gender.Fem
        elif tag.gender == "neut":
            gender = Gender.Neut

        if tag.involvement is not None:
            pass

        if tag.mood is not None:
            if tag.mood == "indc":
                mood = Mood.Ind
            elif tag.mood == "impr":
                mood = Mood.Imp

        if tag.number is not None:
            if tag.number == "sing":
                number = Number.Sing
            elif tag.number == "plur":
                number = Number.Plur

        if tag.person == "1per":
            person = Person.First
        elif tag.person == "2per":
            person = Person.Second
        elif tag.person == "3per":
            person = Person.Third

        if tag.tense == "pres":
            tense = Tense.Pres
        elif tag.tense == "past":
            tense = Tense.Past
        elif tag.tense == "futr":
            tense = Tense.Fut

        if tag.transitivity is not None:
            # TODO
            pass

        if tag.voice == "actv":
            voice = Voice.Act
        elif tag.voice == "pssv":
            voice = Voice.Pass

        return Features(
            Animacy=animacy,
            Aspect=aspect,
            Case=case,
            Gender=gender,
            Mood=mood,
            Number=number,
            Person=person,
            Tense=tense,
            Voice=voice,
        )


import graphlab.connect as _mt
from graphlab.data_structures.sgraph import SGraph as _SGraph
import graphlab.toolkits._main as _main
from graphlab.toolkits.graph_analytics._model_base import GraphAnalyticsModel as _ModelBase

def get_default_options():
    """
    Get the default options for :func:`graphlab.triangle_counting.create`.

    Returns
    -------
    out : dict

    Examples
    --------
    >>> graphlab.triangle_counting.get_default_options()
    """
    _mt._get_metric_tracker().track('toolkit.graph_analytics.triangle_counting.get_default_options')

    return _main.run('triangle_counting_default_options', {})

class TriangleCountingModel(_ModelBase):
    """
    Model object containing the traingle count for each vertex, and the total
    number of triangles. The model ignores the edge directions in that
    it assumes there are no multiple edges between
    the same source ang target pair and ignores bidirectional edges.

    The triangle count of individual vertex characterizes the importance of the
    vertex in its neighborhood. The total number of triangles characterizes the
    density of the graph. It can also be calculated using

    >>> m['triangle_count']['triangle_count'].sum() / 3.

    Below is a list of queryable fields for this model:

    +---------------+------------------------------------------------------------+
    | Field         | Description                                                |
    +===============+============================================================+
    | triangle_count| An SFrame with each vertex's id and triangle count         |
    +---------------+------------------------------------------------------------+
    | num_triangles | Total number of triangles in the graph                     |
    +---------------+------------------------------------------------------------+
    | graph         | A new SGraph with the triangle count as a vertex property  |
    +---------------+------------------------------------------------------------+
    | training_time | Total training time of the model                           |
    +---------------+------------------------------------------------------------+

    This model cannot be constructed directly.  Instead, use
    :func:`graphlab.triangle_counting.create` to create an instance
    of this model. A detailed list of parameter options and code samples
    are available in the documentation for the create function.

    See Also
    --------
    create
    """
    def __init__(self, model):
        '''__init__(self)'''
        self.__proxy__ = model

    def _result_fields(self):
        ret = super(TriangleCountingModel, self)._result_fields()
        ret['num_triangles'] = self['num_triangles']
        ret['triangle_count'] = "SFrame with each vertex's triangle count. See m['triangle_count']"
        return ret

def create(graph, verbose=True):
    """
    Compute the number of triangles each vertex belongs to, ignoring edge
    directions. A triangle is a complete subgraph with only three vertices.
    Return a model object with total number of triangles as well as the triangle
    counts for each vertex in the graph.

    Parameters
    ----------
    graph : SGraph
        The graph on which to compute triangle counts.

    verbose : bool, optional
        If True, print progress updates.

    Returns
    -------
    out : TriangleCountingModel

    References
    ----------
    - T. Schank. (2007) `Algorithmic Aspects of Triangle-Based Network Analysis
      <http://digbib.ubka.uni-karlsruhe.de/volltexte/documents/4541>`_.

    Examples
    --------
    If given an :class:`~graphlab.SGraph` ``g``, we can create
    a :class:`~graphlab.traingle_counting.TriangleCountingModel` as follows:

    >>> g = graphlab.load_graph('http://snap.stanford.edu/data/email-Enron.txt.gz', format='snap')
    >>> tc = graphlab.triangle_counting.create(g)

    We can obtain the number of triangles that each vertex in the graph ``g``
    is present in:

    >>> tc_out = tc['triangle_count']  # SFrame

    We can add the new "triangle_count" field to the original graph g using:

    >>> g.vertices['triangle_count'] = tc['graph'].vertices['triangle_count']

    Note that the task above does not require a join because the vertex
    ordering is preserved through ``create()``.

    See Also
    --------
    TriangleCountingModel
    """
    _mt._get_metric_tracker().track('toolkit.graph_analytics.triangle_counting.create')

    if not isinstance(graph, _SGraph):
        raise TypeError('graph input must be a SGraph object.')

    params = _main.run('triangle_counting', {'graph': graph.__proxy__}, verbose)
    return TriangleCountingModel(params['model'])


"""
L.Higgins 11-vi-18
This is a piece of code to use the equation from 'Gardea-Torresday L Environ Sci Technol 34:4392.'. User inputs the ratio between
the metallic FCC nanoparticles coordination number, N, and the coordination number of the standard foil, N_{foil}. It then takes
the average bond length, d. Both of these values are collected by using EXAFS modelling for the nanoparticles compared to the foil.
The equation is then solved using a numeric method (see: sympy.solve), yielding three solutions (since it is a cubic eqn.). Take
the third solution as the average nanoparticle radius, R.  N.B. This can only work when using a 1st shell EXAFS fitting!

I've designed this to be user friendly, ctrl-c will always break.
"""
try:
    from sympy.solvers import solve
    from sympy import Symbol
except:
    print('//WARNING// Please install the Sympy module: https://docs.sympy.org/latest/install.html ')

try:
    import numpy as np
    import sys
except:
    print('//WARNING// Code requires numpy and sys to be installed')

continuing=True

def inp(txt):
    if sys.version_info[0] == 3:
        return input(txt)
    elif sys.version_info[0] == 2:
        return raw_input(txt)
    else:
        print('FCC requires python > 2')

def get_N():
# Asks the user to give the ratio between the coordination numbers in a user-friendly style.
    N = Symbol('N')
    try:
        N = float(inp('Please input N/N_foil: \n'))
        if N>0 and N<1:
            return N
        else:
            print('N must be a ratio between 0 and 1 exclusive, try again:\n')
            return get_N()
    except ValueError:
        print('N/Nfoil must be a float, try again:\n')
        return get_N()
    except KeyboardInterrupt:
        print('\n  --   Exiting because of keyboard Interuption  --   \n')
        sys.exit()

def get_d():
# Asks the user to give the average 1st shell bond length in a user-friendly style.
    N = Symbol('d')
    try:
        d = float(inp('Please input d e.g. Au ~ 2.85: '))
    except ValueError:
        print('d must be a decimal number, try again:')
        return get_d()
    except KeyboardInterrupt:
        print('\n  --   Exiting because of keyboard Interuption  --   \n')
        sys.exit()
    return d

def cont():
# Asks the user whether they want to continue in a user-friendly style.
    try:
        poss_ans = ['y','Y','n','N']
        var = str(inp('Would you like to continue? y/n \n'))
        if var in poss_ans:
            if var.lower() == 'n':
                return False
            elif var.lower() == 'y':
                return True
        else:
            print('please enter y/Y or n/N, try again:\n')
            return cont()
    except KeyboardInterrupt:
        print('\n  --   Exiting because of keyboard Interuption  --   \n')
        sys.exit()

def find_F0(N,d):
# Numeric solution of the Gardea equation using sympy.solve - no user involvement
    R = Symbol('R')
    F = (R**3 * (N-1)) + (0.75 * d * R**2) - (0.0625 * d**3)
    sol = solve(F,R,cubic=True)
    print('The solutions to F0 are: \n', sol)

while continuing:
# Continues until user breaks with either ctrl-c or 'n'.
    N = get_N()
    print('you entered: ', N)
    d = get_d()
    print('you entered:',  d)
    find_F0(N,d)
    continuing = cont()

print('         Thanks for using the FCC minimum tool!           \n #please cite#\n #"Gardea-Torresday L, Tiemann KJ, Gamez G, et. al.#\n #(2000); Environ Sci Technol 34:4392."#')
sys.exit()


distance = 0
energy = 100
hunger = 0
n_block = 0
s_block = 0
e_block = 0
w_block = 0

while True:
    print("Would you like to walk or run? Enter 'help' for more actions.")
    print("Distance: {} km".format(distance))
    print("Energy: {}".format(energy))
    print("Hunger: {}".format(hunger))
    activity = input().lower()

    if activity == "walk" or activity == "run" and energy > 0 and hunger < 100:
        print("Please choose a direction(north, south, east, west).")
        direction = input().lower()
        if direction == "north" or direction == "n":
            n_block += 1
            print("{} block(s) North".format(n_block))
        elif direction == "south" or direction == "s":
            s_block += 1
            print("{} block(s) South".format(s_block))
        elif direction == "east" or direction == "e":
            e_block += 1
            print("{} block(s) East".format(e_block))
        elif direction == "west" or direction == "w":
            w_block += 1
            print("{} block(s) West".format(w_block))
        else:
            print("Not a direction")
            continue

    if activity == "walk" or activity == "w":
        if energy > 0 and hunger < 100:
            distance += 1
            hunger += 5
        else:
            print("You need to rest or eat first!")
    elif activity == "run" or activity == "r":
        if energy > 10 and hunger < 95:
            distance += 5
            energy -= 20
            hunger += 10
        else:
            print("You need to rest or eat first!")
    elif activity == "rest":
        energy += 30
    elif activity == "eat":
        energy += 50
        hunger = 0
    elif activity == "help":
        print("Eat: Gain 50 energy\nRest: Gain 30 energy\n Go Home: Quit")
    elif activity == "go home"or activity == "home" or activity == "quit" or activity == "q":
        print("Good work out!")
        break
    else:
        print("Improper Input")
        continue

    if energy > 100:
        energy = 100
    elif energy < 0:
        energy = 0


import pytest
from main import *

@pytest.fixture
def client():
    """Configures the app for testing

    Sets app config variable ``TESTING`` to ``True``

    :return: App for testing
    """

    # app.config['TESTING'] = True
    client = app.test_client()

    yield client

def test_landing(client):
    landing = client.get("/")
    html = landing.data.decode()

    assert "Select The State You Live In" in html
    assert landing.status_code == 200

def test_file_display(client):
    aws_files = client.get('/files')
    html = aws_files.data.decode()

    assert "Click on the filename to download it" in html
    assert aws_files.status_code == 200


import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer

class DataConverter(object):

    def __init__(self, target_data_list):
        self.target_data_list = target_data_list

    def convert_text_dictionary(self, text_feature_dict):

        count_vectorizer = CountVectorizer()
        feature_vectors = count_vectorizer.fit_transform(
            [' '.join(text_feature_dict.values())]
        )
        vocabulary = count_vectorizer.get_feature_names()

        keyword_dic = dict()
        for key, value in zip(vocabulary, feature_vectors.toarray()[0]):
            keyword_dic.update({key: value})

        return keyword_dic

    def convert_to_panda(self):

        data_df = None
        cost_df = None

        for target_data in self.target_data_list[:20]:

            # unpack target data
            id = target_data['id']
            cost = target_data['cost']

            # get feature dictionary
            feature_dictionary = dict()
            text_feature_dict = target_data['feature_dict']['text_feature']
            feature_dictionary = self.convert_text_dictionary(text_feature_dict)
            value_feature_dict = target_data['feature_dict']['value_feature']
            keyword_feature_dict = target_data['feature_dict']['keyword_feature']

            if data_df is None:
                data_df = pd.DataFrame(
                    [feature_dictionary.values()],
                    index=[id],
                    columns=feature_dictionary.keys()
                )
                cost_df = pd.DataFrame(
                    [cost],
                    index=[id],
                    columns=['cost']
                )
            else:
                additional_columns = list(set(feature_dictionary.keys()) - set(data_df.columns))
                additional_values = []
                for i in additional_columns:
                    additional_values.append(feature_dictionary[i])
                empty_df = pd.DataFrame(
                    [[0] * len(additional_columns)],
                    index=data_df.index,
                    columns=additional_columns
                )
                data_df = pd.concat([data_df, empty_df], axis=1)
                additional_data_values = []
                for column in data_df.columns:
                    if column in feature_dictionary.keys():
                        additional_data_values.append(feature_dictionary[column])
                    else:
                        additional_data_values.append(0)
                additional_data_df = pd.DataFrame(
                    [additional_data_values],
                    index=[id],
                    columns=data_df.columns
                )
                data_df = pd.concat([data_df, additional_data_df])
                additional_cost_df = pd.DataFrame(
                    [cost],
                    index=[id],
                    columns=['cost']
                )
                cost_df = pd.concat([cost_df, additional_cost_df])

        return {'data_df': data_df, 'cost_df': cost_df}


import datetime
import logging
import pandas as pd

from covid19_scrapers.utils.http import get_json
from covid19_scrapers.scraper import ScraperBase

_logger = logging.getLogger(__name__)

class Indiana(ScraperBase):
    """Indiana lists their COVID-19 case demographics dataset on their
    state health department's open data portal:
    https://hub.mph.in.gov/dataset/62ddcb15-bbe8-477b-bb2e-175ee5af8629

    This includes both the data as an Excel file and a data dictionary.
    """

    DATA_URL = 'https://hub.mph.in.gov/dataset/62ddcb15-bbe8-477b-bb2e-175ee5af8629/resource/2538d7f1-391b-4733-90b3-9e95cd5f3ea6/download/covid_report_demographics.xlsx'
    METADATA_URL = 'https://hub.mph.in.gov/api/3/action/resource_show?id=2538d7f1-391b-4733-90b3-9e95cd5f3ea6'

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    def _scrape(self, **kwargs):
        _logger.debug('Find the update date')
        metadata = get_json(self.METADATA_URL)
        date = datetime.datetime.fromisoformat(
            metadata['result']['last_modified']).date()
        _logger.info(f'Processing data for {date}')

        _logger.debug('Read in the file')
        df_in = pd.read_excel(
            self.DATA_URL,
            sheet_name='Race'
        ).set_index(
            'RACE'
        )

        total_cases = df_in['COVID_COUNT'].sum()
        total_deaths = df_in['COVID_DEATHS'].sum()
        aa_cases = df_in.loc['Black or African American', 'COVID_COUNT']
        aa_deaths = df_in.loc['Black or African American', 'COVID_DEATHS']
        aa_cases_pct = df_in.loc['Black or African American',
                                 'COVID_COUNT_PCT']
        aa_deaths_pct = df_in.loc['Black or African American',
                                  'COVID_DEATHS_PCT']

        return [self._make_series(
            date=date,
            cases=total_cases,
            deaths=total_deaths,
            aa_cases=aa_cases,
            aa_deaths=aa_deaths,
            pct_aa_cases=aa_cases_pct,
            pct_aa_deaths=aa_deaths_pct,
            pct_includes_unknown_race=True,
            pct_includes_hispanic_black=True,
        )]


# DAta collection

# java uses array, in python lists are the same as Arrays
# both serve the same of storing data
#why lists? they help manage data, access data in order, and they give us option to add, remove data

#syntax of lists: [list] hard brackets, (tuple), {dictionary - key:value} pairs
#Tuples are immutable - can't be changed

# List of cities

# cities = ["Tokyo", "Paris", "Prague", "Luxembourg"]
# #whenever you see the word display in exam, it means they want you to print
#
# # print(cities)
# # print(type(cities))
# # print(cities[3]) # this is how you print an index from a list
# cities[3] = "Amsterdam" # this will replace index 3 with dam
# print(cities)
#
# cities.append("Vilnius")
# print(cities)
#
# cities.remove("Paris")
# print(cities)

# Removes last entry in the list
cities = ["Tokyo", "Paris", "Prague", "Luxembourg"]
cities.pop()
print(cities)

# Inserts entry at a specified index
cities.insert(0, "London")
print(cities)

# Lists can hold multiple data types
# mix_type_string = [1, 2, 3], ["one", "two", "three"]
# print(mix_type_string)

# Here is an example of concatenating two lists of different data types
mix_type_string = [1, 2, 3]
string_list = ["one", "two", "three"]
print(string_list + mix_type_string)


from django.http import HttpResponseRedirect
from django.shortcuts import render
from .forms import CodeForm
import clues.models as cModels

def index(request):
    if request.method == 'POST':
        form = CodeForm(request.POST)
        if form.is_valid():
            code = form.cleaned_data['code'].lower()
            if cModels.Clue.objects.filter(codeword_1=code).exists():
                print('Code {} found.'.format(code))
                c = cModels.Clue.objects.get(codeword_1=code)
                redirect = c.view_1
                c.viewed_1 = True
                c.save()
                return HttpResponseRedirect('/clues/{}'.format(redirect))
            elif cModels.Clue.objects.filter(codeword_2=code).exists():
                print('Code {} found.'.format(code))
                c = cModels.Clue.objects.get(codeword_2=code)
                redirect = c.view_2
                c.viewed_2 = True
                c.save()
                return HttpResponseRedirect('/clues/{}'.format(redirect))
            else:
                print('Code {} not found.'.format(code))
                return HttpResponseRedirect('/clues/thanks')
    else:
        form = CodeForm()

    return render(request, 'index.html', {'form': form})

def landing(request, page):
    return render(request, '{}.html'.format(page))


import scipy.io
import math
import tensordiffeq as tdq
from tensordiffeq.models import CollocationSolverND
from tensordiffeq.boundaries import *

Domain = DomainND(["x", "t"], time_var='t')

Domain.add("x", [-1.0, 1.0], 512)
Domain.add("t", [0.0, 1.0], 201)

N_f = 50000
Domain.generate_collocation_points(N_f)

def func_ic(x):
    return x ** 2 * np.cos(math.pi * x)

# Conditions to be considered at the boundaries for the periodic BC
def deriv_model(u_model, x, t):
    u = u_model(tf.concat([x, t], 1))
    u_x = tf.gradients(u, x)[0]
    # u_xx = tf.gradients(u_x, x)[0]
    # u_xxx = tf.gradients(u_xx, x)[0]
    # u_xxxx = tf.gradients(u_xxx, x)[0]
    return u, u_x

init = IC(Domain, [func_ic], var=[['x']])
x_periodic = periodicBC(Domain, ['x'], [deriv_model])

BCs = [init, x_periodic]

def f_model(u_model, x, t):
    u = u_model(tf.concat([x, t], 1))
    u_x = tf.gradients(u, x)
    u_xx = tf.gradients(u_x, x)
    u_t = tf.gradients(u, t)
    c1 = tdq.utils.constant(.0001)
    c2 = tdq.utils.constant(5.0)
    f_u = u_t - c1 * u_xx + c2 * u * u * u - c2 * u
    return f_u

## Which loss functions will have adaptive weights
# "residual" should a tuple for the case of multiple residual equation
# BCs have to follow the same order as the previously defined BCs list
dict_adaptive = {"residual": [True],
                 "BCs": [True, False]}

## Weights initialization
# dictionary with keys "residual" and "BCs". Values must be a tuple with dimension
# equal to the number of residuals and boundary conditions, respectively
init_weights = {"residual": [tf.random.uniform([N_f, 1])],
                "BCs": [100 * tf.random.uniform([512, 1]), None]}

layer_sizes = [2, 128, 128, 128, 128, 1]

model = CollocationSolverND()
model.compile(layer_sizes, f_model, Domain, BCs, isAdaptive=True,
              dict_adaptive=dict_adaptive, init_weights=init_weights)

model.fit(tf_iter=10000, newton_iter=10000)

# Load high-fidelity data for error calculation
data = scipy.io.loadmat('AC.mat')

Exact = data['uu']
Exact_u = np.real(Exact)

x = Domain.domaindict[0]['xlinspace']
t = Domain.domaindict[1]["tlinspace"]

# create mesh for plotting

X, T = np.meshgrid(x, t)

X_star = np.hstack((X.flatten()[:, None], T.flatten()[:, None]))
u_star = Exact_u.T.flatten()[:, None]

# forward pass through model
u_pred, f_u_pred = model.predict(X_star)

error_u = tdq.helpers.find_L2_error(u_pred, u_star)
print('Error u: %e' % (error_u))

U_pred = tdq.plotting.get_griddata(X_star, u_pred.flatten(), (X, T))
FU_pred = tdq.plotting.get_griddata(X_star, f_u_pred.flatten(), (X, T))

lb = np.array([-1.0, 0.0])
ub = np.array([1.0, 1])

tdq.plotting.plot_solution_domain1D(model, [x, t], ub=ub, lb=lb, Exact_u=Exact_u)


import re

from wagtail.embeds.finders.base import EmbedFinder
from wagtail.embeds.exceptions import EmbedNotFoundException

from urllib import request as urllib_request
from urllib.error import URLError
from urllib.parse import urlencode
from urllib.request import Request

class RtvEmbedFinder(EmbedFinder):
    def __init__(self, providers=None, options=None):
        self._endpoints = {}

        for provider in providers:
            patterns = []

            endpoint = provider["endpoint"]

            for url in provider["urls"]:
                patterns.append(re.compile(url))

            self._endpoints[endpoint] = patterns

    def _get_endpoint(self, url):
        for endpoint, patterns in self._endpoints.items():
            for pattern in patterns:
                if re.match(pattern, url):
                    return endpoint

    def accept(self, url):
        """
        Returns True if this finder knows how to fetch an embed for the URL.

        This should not have any side effects (no requests to external servers)
        """

        # check if url matches any of the providers
        return self._get_endpoint(url) is not None

    def find_embed(self, url, max_width=None):
        """
        Takes a URL and max width and returns a dictionary of information about the
        content to be used for embedding it on the site.

        This is the part that may make requests to external APIs.
        """

        return {
            'title': "",
            'author_name': "Multimedijski center RTV Slovenija",
            'provider_name': "RTVSLO.si",
            'type': "video",
            'thumbnail_url': "",
            'width': "",
            'height': "",
            'html': f'<iframe class="custom-video-embed" src="{url}"></iframe>',
        }


import constants
import encryption
import json
import base64
from os import walk, path, remove

def ensure_rsa_keys_exists():
    try:
        priv_key = encryption.LoadRSAPrivateKey(constants.RSA_PRIVATEKEY_FILEPATH)
        pub_key = encryption.LoadRSAPublicKey(constants.RSA_PUBLICKEY_FILEPATH)

        print("RSA keys found")
    except (FileNotFoundError, ValueError):
        print("RSA keys not found. Creating...")

        (priv_key, pub_key) = encryption.GenerateRSAKey()

        encryption.WriteRSAPrivateKey(constants.RSA_PRIVATEKEY_FILEPATH, priv_key)
        encryption.WriteRSAPublicKey(constants.RSA_PUBLICKEY_FILEPATH, pub_key)

    print(priv_key)
    print(pub_key)

def check_public_key_exists():
    try:
        encryption.LoadRSAPublicKey(constants.RSA_PUBLICKEY_FILEPATH)
        return True
    except (FileNotFoundError, ValueError):
        return False

def check_private_key_exists():
    try:
        encryption.LoadRSAPublicKey(constants.RSA_PRIVATEKEY_FILEPATH)
        return True
    except (FileNotFoundError, ValueError):
        return False

def base64ToString(b):
    return base64.encodestring(b).decode('ascii')

def stringToBase64(s):
    return base64.decodebytes(s.encode('ascii'))

def encrypt_file(infile, outfile):
    (RSACipher, ciphertext, iv, tag) = encryption.MyRSAEncrypt(infile,
                                                               constants.RSA_PUBLICKEY_FILEPATH)
    data = {
        "rsa": base64ToString(RSACipher),
        "ciphertext": base64ToString(ciphertext),
        "iv": base64ToString(iv),
        "tag": base64ToString(tag),
    }

    with open(outfile, "w") as file:
        json.dump(data, file)

def decrypt_file(infile, outfile):
    with open(infile, "r") as file:
        data = json.load(file)

    print(data['tag'])

    RSACipher = stringToBase64(data['rsa'])
    ciphertext = stringToBase64(data['ciphertext'])
    iv = stringToBase64(data['iv'])
    tag = stringToBase64(data['tag'])

    print(encryption.MyRSADecrypt(outfile, RSACipher, ciphertext, iv, tag,
                                  constants.RSA_PRIVATEKEY_FILEPATH))

def encrypt_all_files(rootdir):
    for dirName, subdirList, fileList in walk(rootdir):
        for file in fileList:
            try:
                filePath = path.join(dirName, file)
                encrypt_file(filePath, filePath + constants.ENCRYPTED_EXTENSION)

                # Exclude public key
                if path.abspath(filePath) == path.abspath(constants.RSA_PUBLICKEY_FILEPATH):
                    break

                # Exclude private key
                if path.abspath(filePath) == path.abspath(constants.RSA_PRIVATEKEY_FILEPATH):
                    break

                # Executable will be excluded by default because it's running

                remove(filePath)

            except Exception as e:  # TODO Tighten exception
                # Don't kill the whole walk on failure
                print(e)

def decrypt_all_files(rootdir):
    for dirName, subdirList, fileList in walk(rootdir):
        for file in fileList:

            try:
                # Only touch files that we've encrypted
                if path.splitext(file)[1] == constants.ENCRYPTED_EXTENSION:
                    filePath = path.join(dirName, file)
                    decrypt_file(filePath,
                                 path.splitext(filePath)[0])

                    remove(filePath)

            except Exception as e:  # TODO Tighten exception
                # Don't kill the whole walk on failure
                print(e)


import cv2, os
import urllib.request as url
import numpy as np
from matplotlib import pyplot as plt

def main():
    # Load sample image from URL
    req = url.urlopen('https://i.pinimg.com/originals/d3/1b/31/d31b31653d7cc0d672fc72fb1de9bf7a.png')
    arr = np.asarray(bytearray(req.read()), dtype=np.uint8)
    img = cv2.imdecode(arr, -1) # 'Load it as it is'
    img = cv2.imread('/home/ocr_receipt1/src1/result0.png')
    h, w, ch = img.shape

    print('Width: {}'.format(w))
    print('Height: {}'.format(h))
    print('Channel: {}'.format(ch))

    # If reading local image, use below:
    # img = cv2.imread('local_image.jpg')

    #### (1) Effect - Gamma Adjustment ####

    # Convert from RGB to gray scale
    gray = cv2.cvtColor(img,cv2.COLOR_RGB2GRAY)
    gamma = 0.4 # r<1 will increase bright contrast ratio

    # Gamma adjustment
    pic1 = gray.max() * (gray/gray.max()) ** (1/gamma)
    cv2.imwrite(os.path.join('output_image','1_gamma_adjusted.png'), pic1)

    #### (2) Effect - Adaptive Binary Thresholding ####
    pic2 = cv2.adaptiveThreshold(gray,20,cv2.ADAPTIVE_THRESH_MEAN_C,cv2.THRESH_BINARY,67,2)
    cv2.imwrite(os.path.join('output_image','2_binary_thresholding.png'), pic2)

    pic3 = cv2.adaptiveThreshold(gray,20,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY,33,2)
    cv2.imwrite(os.path.join('output_image','3_binary_thresholding_gaussian.png'), pic3)

    #### (3) Effect - Laplacian Filter ####
    pic4 = cv2.Laplacian(gray,cv2.CV_16SC1, ksize=29)
    cv2.imwrite(os.path.join('output_image','4_laplacian_filter.png'), pic4)

    ### Plot results
    titles = ['Original (Blurry Word)', 'Gamma Adjusted (r=0.4)','Adaptive Mean Thresholding (blocksize=67)',
                'Adaptive Gaussian Thresholding', 'Laplacian (kernel size=29)']
    images = [img, pic1, pic2, pic3, pic4]

    for i in range(len(images)):
        plt.subplot(-(-len(images)//2),2,i+1),plt.imshow(images[i],'gray')
        plt.title(titles[i])
        plt.xticks([]),plt.yticks([])
    plt.show()

if __name__ == '__main__':
    main()


import pandas as pd
import numpy as np
import datetime
import matplotlib.pyplot as plt

train=pd.read_csv("trainAdult.csv")
test=pd.read_csv("testAdult.csv")
trainX=train.select_dtypes(include='int64')

# data preprocess
def normalize(data):
    data = data - np.mean(data, axis=0)
    data = data / np.std(data, axis=0)
    return data

s = train['NA']
labels = []
#create dummy for response variable
for x in s:
    if x == s[0]:
        labels.append(-1)
    else:
        labels.append(1)

labels = np.array(labels)
continuous = ['age','fnlwgt','education-num','capital-gain','capital-loss','hours-per-week']
train = train[continuous].values
test = test[continuous].values
train = normalize(train)
test = normalize(test)

# calculate accuracy
def cal_accuracy(predictions, yi):
    correct = 0
    for pred, actual in zip(predictions, yi):
        if pred == actual:
            correct += 1
    return correct / len(yi)

# update and gradient,eta is learning rate,lamb is lambda
# this function calculates the gradient for SVM
def gradient(a, b, eta, lamb, x, yi):
    res = yi * (np.dot(a, x.T) + b)
    if res >= 1:
        temp = (eta * lamb) * a
        a -= temp
    else:
        temp = -yi * x+lamb * a
        a -= eta * temp
        b -= eta * (-yi)
    return a, b

# predict class
#if ax+b positive,predict to one class, else to another class
# think about 2D case
def sign(test_x, a, b):
    result = np.dot(test_x, a.T) + b
    result[result <= 0] = -1
    result[result > 0] = 1
    return result

# train test split
# size determines the ratio of train vs test set
def split(data, size):
    rows = data.shape[0]
    index = np.arange(0, rows)
    np.random.shuffle(index)
    test_id = index[:size]
    train_id = index[size:]
    return test_id, train_id

# choose learning rate by grid search
# trying out learning rate on a small set of data
def small_experiment():
    learning_rate = [0.01, 0.02, 0.05, 0.07, 0.1, 0.2, 0.5]

    # use 1000 obs to train, 100 to test
    test_index, train_index = split(train, 100)
    train_index = train_index[:1000]
    small_train = train[train_index]
    small_y = labels[train_index]
    small_test = train[test_index]
    small_target = labels[test_index]

    accuracy = []
    time = []
    # train and record accuracy and time
    for lr in learning_rate:
        start = datetime.datetime.now()
        # since there're 6 continuous variables included, a is a vector with 6 numbers
        a = np.random.rand(6)
        b = np.random.rand(1)[0]
        for i in range(5000):
            #pick one number from 1-1000 randomly, this is the single sample to run SGD
            rand_ind = np.random.choice(range(1000), 1)[0]
            xi = small_train[rand_ind]
            yi = small_y[rand_ind]
            a, b = gradient(a, b, lr, 0.01, xi, yi)
        end = datetime.datetime.now()
        # add the average error calculated from this particular a,b pair
        accuracy.append(cal_accuracy(sign(small_test, a, b), small_target))
        time.append((end-start).total_seconds())

    # plot accuracy and time
    plt.scatter(accuracy, time)
    for i, txt in enumerate(learning_rate):
        plt.annotate(txt, (accuracy[i], time[i]))
    plt.show()

# grid search lambda
lambda_vals = [1e-3, 1e-2, 1e-1, 5e-1, 1]
def search(eta):
    best_lambda = 0
    best_accuracy = 0
    for l in lambda_vals:
        avg_acc = 0
        for round in range(50):
            # train validation split,0.1 is the train test split ratio
            val_index, season_index = split(train, int(0.1 * len(train)))
            search_train = train[season_index]
            search_yi = labels[season_index]
            validation = train[val_index]
            val_label = labels[val_index]

            # train model
            a = np.random.rand(6)
            b = np.random.rand(1)[0]
            for i in range(5000):
                rand_ind = np.random.choice(range(len(search_train)), 1)[0]
                xi = search_train[rand_ind]
                yi = search_yi[rand_ind]
                a, b = gradient(a, b, eta, l, xi, yi)
            avg_acc += cal_accuracy(sign(validation, a, b), val_label)

        # find the max accuracy
        if avg_acc > best_accuracy:
            best_accuracy = avg_acc
            best_lambda = l
    return best_lambda

# parameters
epochs = 50
held_out = 50
season_steps = 300
acc_steps = 30
m = 1
n = 10

# SVM training
acc_30_steps = []
magnitude = []
def svm(lamb):
    a = np.random.rand(6)
    b = np.random.rand(1)[0]
    for i in range(epochs):
        print("Epochs ", i)

        # update eta, learning rate for each season
        eta = m / (i + n)

        # held out 50 evaluation examples
        eval_index, train_index = split(train, held_out)
        new_train = train[train_index]
        new_yi = labels[train_index]
        new_test = train[eval_index]
        new_label = labels[eval_index]

        # season and step are to ensure each data has been seen in the random picking method
        for j in range(season_steps):
            # gradient update parameter a and b
            rand_ind = np.random.choice(range(len(new_train)), 1)[0]
            xi = new_train[rand_ind]
            yi = new_yi[rand_ind]
            a, b = gradient(a, b, eta, lamb, xi, yi)

            # calculate accuracy every 30 steps
            if (j % acc_steps == 0) & (j > 0):
                predictions = sign(new_test, a, b)
                season_acc = cal_accuracy(predictions, new_label)
                acc_30_steps.append(season_acc)
                magnitude.append(np.linalg.norm(a))
                print("in step ", j, " season accuracy is ", season_acc)

        predictions = sign(new_test, a, b)
        epoch_acc = cal_accuracy(predictions, new_label)
        acc_30_steps.append(epoch_acc)
        magnitude.append(np.linalg.norm(a))
        print("Epochs accuracy is ", epoch_acc)
    return a, b

# train svm
plot_acc = {}
plot_mag = {}
for l in lambda_vals:
    acc_30_steps = []
    magnitude = []
    a, b = svm(l)
    predictions = sign(test, a, b)
    plot_acc["reg="+str(l)] = acc_30_steps
    plot_mag["reg="+str(l)] = magnitude
plot_mag['x'] = range(len(acc_30_steps))
plot_acc['x'] = range(len(magnitude))

#compare response in test set with predictions
test_50k=[]
for label in test.NA:
    if label == ' <=50K.':
        test_50k.append(-1)
    else:
        test_50k.append(1)

cal_accuracy(predictions, test_50k)


import os

from gaia_bot.core.console_manager import ConsoleManager
from gaia_bot.modules.local.models.task_detect.prompt_to_response import inference

class AssistantSkill:
    first_activation = False
    console_manager = ConsoleManager()
    skill_dir = os.path.dirname(__file__)

    @classmethod
    def console(cls, text="", info_log=None, error_log=None, warning_log=None, debug_log=None, refresh_console=False):
        cls.console_manager.console_output(text=text,
                                           info_log=info_log,
                                           error_log=error_log,
                                           warning_log=warning_log,
                                           debug_log=debug_log,
                                           refresh_console=refresh_console)

    @classmethod
    def response(cls, transcript, refresh_console=False):
        cls.console_manager.console_output(text=transcript, info_log="Skill Handling", refresh_console=refresh_console)

    @classmethod
    def detect_skill_tag(cls, transcript):
        try:
            infer = inference.infer(transcript)
            cls.console_manager.console_output(info_log="Skill tag:" + infer)
            return infer
        except:
            infer = None
            cls.console_manager.console_output(error_log="Failed to detect skill tag.")
            return infer

    @classmethod
    def validate_assistant_response(cls, detected_skill, SKILLS):
        for skill in SKILLS:
            for tag in str(skill['tags']).split(', '):
                if detected_skill.__contains__(tag):
                    cls.execute_skill(skill['func'], detected_skill)
                    break

    @classmethod
    def execute_skill(cls, skill, text, *kwargs):
        if skill:
            cls.console_manager.console_output(info_log='Executing skill...')
            try:
                skill(text)
            except Exception as e:
                cls.console_manager.console_output(error_log="Failed to execute skill...")

    @classmethod
    def sentence_detect(cls, text, SKILLS):
        for skill in SKILLS:
            for tag in str(skill['tags']).split(', '):
                if tag == 'default skill' or tag == 'first skill':
                    cls.execute_skill(skill['func'], text)
                    break


from math import prod

def count_trees(map, slope=(3, 1)):
    dx, dy = slope
    height, width = len(map), len(map[0])
    encounters = [map[y][y // dy * dx % width] for y in range(0, height, dy)]
    return sum(1 if e == '#' else 0 for e in encounters)

if __name__ == '__main__':
    with open('input.txt') as input_file:
        map = [line[:-1] for line in input_file]
        slopes = [(1, 1), (3, 1), (5, 1), (7, 1), (1, 2)]
        print(prod(count_trees(map, slope) for slope in slopes))