# Use this token to access the HTTP API:
# 5620370083:AAGu5OcD-59_sNmPevlZqq8AplOxJkGKwR0

# t.me/Small_test_project_bot

import src.settings as settings # Импортируем "защищенный" файл с "важной" информацией

BOT_TOKEN        = settings.API_KEY # Токен из "защищенного" файла settings.py
DB_PATH          = './TG_bot_for_timesheets/src/bot.db'
CREDENTIALS_FILE = '../src/pythonextension-202bab519501.json'  # Файла "pythonextension-202bab519501.json", содержащий закрытый ключ

from pprint import pprint
import logging
import inspect
import sys, os

# sys.path.insert(0, os.path.abspath('./')) # Добавляем папку выше уровнем для получения их содержимого

# pprint(sys.path)

import dependence.Google_sheet_extension_v3_in_class as GSE
import dependence.sqlite_tg_bot as STB

logging.basicConfig(filename='./TG_bot_for_timesheets/bot.log', level=logging.INFO,
                    format="%(asctime)s %(levelname)s %(message)s" ) # Логгируем ВСЕ сообщения в файл "bot.log" по формату <Врмемя Тип сообщения Сообщение>

def showVarType(var) -> None: # Узнаём тип данных произвольной переменной
    caller_locals = inspect.currentframe().f_back.f_locals
    varName = [name for name, value in caller_locals.items() if var is value] # Получаем имя переменной
    print(var, f'\n{varName} type is {type(var)}') # Узнаём тип данных переменной

# extention = GSE.extension(CREDENTIALS_FILE=CREDENTIALS_FILE,)

DB_handler = STB.tgbot_db(path=DB_PATH)

from telegram import InlineKeyboardButton, InlineKeyboardMarkup, Update, ReplyKeyboardMarkup, KeyboardButton

from telegram.ext import Updater, CommandHandler, MessageHandler, Filters, CallbackQueryHandler
import telegram
import telegram.ext

def start(update : telegram.update.Update, context : telegram.ext.callbackcontext.CallbackContext):

    print("Funcion 'start' was colled !")
    logging.info("Funcion 'start' was colled !")

    keyboard = [ [ InlineKeyboardButton('Let`s start !', callback_data='startCallback')] ] # Создаём кнопку с введённым текстом
    markup = InlineKeyboardMarkup(keyboard) # Создаём разметку с полученной кнопкой
    update.message.reply_text( text='Bot is online !', reply_markup = markup) # Выводим кнопку в чат
    context.user_data['status'] = 'Free' # Записываем сосояние без конкретной привязки к действиям пользователя
    print(f'Context -', context.user_data['status'])

def login(update : telegram.update.Update, context : telegram.ext.callbackcontext.CallbackContext):

    print("Funcion 'login' was colled !")
    logging.info("Funcion 'login' was colled !")

    print(context.user_data['status'])

    # showVarType(update)
    # userInfo = update.message.chat

    if DB_handler.check_userExistence(update) == False: # Проверяем есть ли такой пользователь в базе данных
        print(f'User with id = {update.callback_query.message.chat.id} not existe')
        DB_handler.set_newUser(update)
        print(f'User with id = {update.callback_query.message.chat.id} now in db')

        set_GoogleSheetID(update, context)
    else: # Такой пользователь уже есть
        print(f'User with id = {update.callback_query.message.chat.id} was existe in db')

        keyboard = [ [ InlineKeyboardButton('Yes', callback_data='set_GoogleSheetID'),  InlineKeyboardButton('No', callback_data='use_extention')] ] # Создаём кнопку с введённым текстом
        markup = InlineKeyboardMarkup(keyboard) # Создаём разметку с полученной кнопкой
        update.callback_query.message.reply_text( text='Do you want to change your Google Sheet ID ?', reply_markup = markup) # Выводим кнопку в чат

        # update.callback_query.message.reply_text(text='Looks like you already was here. \n Please enter your Google sheet ID !', reply_markup = markup )

def set_GoogleSheetID(update : telegram.update.Update, context : telegram.ext.callbackcontext.CallbackContext): # Записываем новую ссылку на Google Sheet
    print("Funcion 'set_GoogleSheetID' was colled !")
    logging.info("Funcion 'set_GoogleSheetID' was colled !")

    context.user_data['status'] = 'set_GoogleSheetID' # Записываем сосояние ожидания ввода Google Sheet ID
    old_ID = str(DB_handler.get_userSheetId(update))
    keyboard = [ [ KeyboardButton(old_ID) ] ] # Создаём кнопку с введённым текстом | Нет параметра "Callback_data"
    markup = ReplyKeyboardMarkup(keyboard, resize_keyboard = True, one_time_keyboard = True, input_field_placeholder = "Enter your Google sheet ID here:") # Создаём разметку с полученной кнопкой
    update.callback_query.message.reply_text( text='Please enter your Google sheet ID !', reply_markup = markup) # Выводим кнопку в чат;  Остаток текста - Looks like you already was here. \n
    # showVarType(old_ID)

def insert_Google_Sheet_ID(update : telegram.update.Update, context : telegram.ext.callbackcontext.CallbackContext):
    print(f"Funcion 'insert_Google_Sheet_ID' was colled with context status {context.user_data['status']} !")
    logging.info(f"Funcion 'insert_Google_Sheet_ID' was colled with context status {context.user_data['status']}")

    DB_handler.set_userSheetId(update) # Записываем новый Google sheet ID в БД для данного пользователя
    check_Google_Sheet_ID(update, context)

def check_Google_Sheet_ID(update : telegram.update.Update, context : telegram.ext.callbackcontext.CallbackContext):
    print(f"Funcion 'check_Google_Sheet_ID' was colled with context status {context.user_data['status']} !")
    logging.info(f"Funcion 'check_Google_Sheet_ID' was colled with context status {context.user_data['status']}")

    # status = GSE.extension(CREDENTIALS_FILE = CREDENTIALS_FILE, sheet_id= DB_handler.get_userSheetId(update)).status

    # print(DB_handler.get_userSheetId(update))
    # print(status)

    if GSE.extension(CREDENTIALS_FILE = CREDENTIALS_FILE, sheet_id= DB_handler.get_userSheetId(update)).is_connected() == True:
        keyboard = [ [  ] ] # Создаём кнопку с введённым текстом | Нет параметра "Callback_data"
        markup = ReplyKeyboardMarkup(keyboard, resize_keyboard = True) # Создаём разметку с полученной кнопкой
        update.message.reply_text(text='The bot has successfully connected to your Google sheet !', reply_markup=markup)

        context.user_data['status'] = 'Google' # Записываем сосояние без конкретной привязки к действиям пользователя
        googleHandler(update, context)
    else:
        update.message.reply_text(text='The bot has not successfully connected to your Google sheet !')
        # DB_handler.set_userSheetId(update)

def googleHandler(update : telegram.update.Update, context : telegram.ext.callbackcontext.CallbackContext): # Функция обработки запросов к google sheet
    print(f"Funcion 'googleHandler' was colled with context status {context.user_data['status']} !")
    logging.info(f"Funcion 'googleHandler' was colled with context status {context.user_data['status']}")

    pass

def callbackHandler(update : telegram.update.Update, context : telegram.ext.callbackcontext.CallbackContext): # Функция обработки обаратных запросов (callback_data)
    global path
    print("Funcion 'callbackHandler' was colled !")
    logging.info("Funcion 'callbackHandler' was colled !")
    query = update.callback_query.data # Вычленяем обаратный запрос (callback_data)
    logging.info(f"Query - '{query}' !")

    # showVarType(context)

    print(f"Query - '{query}' !")

    if query == 'startCallback':
        login(update, context)
    elif query == 'set_GoogleSheetID':
        set_GoogleSheetID(update, context)
    elif query == 'use_extention':
        googleHandler(update, context)

def textHandler(update : telegram.update.Update, context : telegram.ext.callbackcontext.CallbackContext): # Функция обработки текстовых сообщений

    print(f"Funcion 'textHandler' was colled with context status {context.user_data} !")
    logging.info(f"Funcion 'textHandler' was colled with context status {context.user_data} !")

    context_status = context.user_data['status']
    # print(context_status == 'set_GoogleSheetID')
    showVarType(context_status)

    if context_status == 'set_GoogleSheetID':
        print("Go to insert_Google_Sheet_ID !")
        insert_Google_Sheet_ID(update, context)

def photoHandler(update : telegram.update.Update, context): # Функция обработки сообщений с фотографиями
    global path
    print("Funcion 'photoHandler' was colled !")
    logging.info("Funcion 'photoHandler' was colled !")

def set_commandHandlers(mybot : Updater): # Функция обявляет ручки для диспетчра
    dp = mybot.dispatcher # Создаём объект <Диспетчер>

    dp.add_handler(CommandHandler('start', start)) # Ручка для команды "/start"
    dp.add_handler(MessageHandler(Filters.text, textHandler)) # Ручка для всего получаемого текста
    dp.add_handler(MessageHandler(Filters.photo, photoHandler)) # Ручка для всех получаемых фотографий
    dp.add_handler(CallbackQueryHandler(callbackHandler)) # Ручка для обработки различных callback_data
    return dp # Возвращаем диспетчер со всеми "ручками"

def main(strBotToken): # Функция основоного стека вызова
    mybot = Updater(strBotToken, use_context=True) # Создаём объект "mybot" класса "Updater" отвечающий за взаимодействие с сервером telegram

    dp = set_commandHandlers(mybot) # Создаём объект <Диспетчер>

    mybot.start_polling() # Бот начинает обращатся к серверу telegram

    print("-"*10,'\nBot is alive !') # Выводим в консоль что бот запущен
    logging.info('"Bot" is alive !')

    mybot.idle() # бот работает пока работает хост

if __name__ == '__main__': # Если файл tg_bot.py вызаван, то будет запущен main(strBotToken); Если он будет импортироват то ничего не произайдёт
    main(BOT_TOKEN)


from typing import List
class Solution:
    def maxNumberOfApples(self, arr: List[int]) -> int:
        arr.sort()
        res = 0
        tmp = 0
        for i in range(len(arr)):
            tmp += arr[i]
            if tmp > 5000: return res
            else: res += 1
        return res


import json
import requests
from urllib.request import urlopen
import pandas as pd
import dash
import dash_core_components as dcc
import dash_html_components as html
import plotly.express as px
import dash_bootstrap_components as dbc
import dash
from dash.dependencies import Input, Output
import dash_auth

USERNAME_PASSWORD_PAIRS = [['username','password'],['kopisusu','1998']]

def pprint(string):
  print(json.dumps(string, indent=2))

result = requests.get("http://192.168.18.110:14240/restpp/query/MyGraph/hobby_param")

hasil = json.loads(result.text)
pprint(hasil)
interest =[]
avg_age = []

for item in hasil["results"][0]["tmp"]:
    interest.append(item['attributes']['interest'])
    avg_age.append(item['attributes']['@avg'])

print (interest,avg_age)

dff = pd.DataFrame({
    "rata2umur": avg_age,
    "hobby": interest,

})

#params= {'id':"\"\"error\"\""}

# req =  requests.get("http://192.168.1.90:14240/restpp/query/mygraph/ip_tracking",params=params)

########### INI UNTUK MENGAMBIL DATA DARI RESPP API TIGERGRAPH ###############

with urlopen("http://192.168.18.110:14240/restpp/query/MyGraph/most_followers") as response:
    data = response.read()

########### LALU DATA TERSEBUT KITA MASUKAN KEDALAM VARIABLE PYTHON ##################
source = json.loads(data)
f_name=[]
num_follower =[]
print (source)

############# DIMANA KITA HARUS MENGGUNAKAN LOGIC LOOPING UNTUK MEMBUATNYA ############

for a in source['results'][0]['followers']:
    f_name.append(a['attributes']['first_name'])
    num_follower.append (a['attributes']['@num_followers'])
    # print(f_name,num_follower)

############## SETELAH ITU DATA TERSEBUT DI MASUKAN KEDALAM DATAFRAME PANDAS #############
df1 = pd.DataFrame({
    "first_name": f_name,
    "count_follow": num_follower,

})

######### INI UNTUK MENGECEK APAKAH DATA TERSEBUT UDAH SIAP UNTUK KITA INPUT KEDALAM PARAMETER DASH #############
# print (df1)

print ('===========================================')

def pprint(string):
  print(json.dumps(string, indent=2))

############# INI QUERY KE DUA YANG MEMAKAI PARAMETER DIDALAMNYA #############

params = {
  "v_type": "users",
  "e_type": "reffered_by",
  "max_change": 0.001,
  "max_iter": 25,
  "damping": 0.85,
  "top_k": 100,
  "print_accum": True,
  "result_attr": "",
  "file_path": "",
  "display_edges": False
}

req = requests.get("http://192.168.18.110:14240/restpp/query/MyGraph/pagerank",params=params)

data2 = json.loads(req.text)

q1= data2['results'][0]['@@topScores']
vertex_id= []
score = []

# print (q1)
for i in q1:
    vertex_id.append(i['Vertex_ID'])
    score.append(i['score'])
    # print (vertex_id,score)

df = pd.DataFrame({
    "v_id": vertex_id,
    "val_score": score,
})

print(df)
print(df1)
# df=pd.DataFrame(q1)

# df = pd.DataFrame(req)
# print (df)

############ INI KITA MEMASUKAN DATA YANG SUDAH KITA BUAT KEDALAM DATAFRAME UNTUK SIAP DI VISUALISASIKAN ######

fig = px.bar(df1, x="first_name", y="count_follow", color="first_name")
fig1 = px.bar( df, x = "v_id", y= "val_score", )
fig3= px.pie (dff,values= "rata2umur",color_discrete_map= "hobby",names="hobby",)

###### MULAI DARI SINI KITA MEMBUAT LAYOUT UNTUK MEMBUAT SEBUAH VISUALISASI ##########

app = dash.Dash(__name__, external_stylesheets=[dbc.themes.SUPERHERO],suppress_callback_exceptions=True)
# app = dash.Dash(__name__)
auth = dash_auth.BasicAuth(app,USERNAME_PASSWORD_PAIRS)
#

colors = {
    'background': '#111111',
    'text': '#7FDBFF'
}

fig3.update_layout (
    plot_bgcolor=colors['background'],
    paper_bgcolor=colors['background'],
    font_color=colors['text']
)

fig.update_layout (
    plot_bgcolor=colors['background'],
    paper_bgcolor=colors['background'],
    font_color=colors['text']
)

fig1.update_layout(
    plot_bgcolor=colors['background'],
    paper_bgcolor=colors['background'],
    font_color=colors['text']
)

PLOTLY_LOGO = "https://i.ibb.co/9Vp8QKx/Untitled-design-9-1.png"
titlecard = dbc.Card([
  dbc.CardBody([
    dbc.Row([dbc.Col(html.Center(html.H1("VISUALISASI VIRTUAL-BOX")),style={'fontColor':'text'}),
    dbc.Col(html.Img(src=PLOTLY_LOGO, width="225px" ,style={"right":0,"margin-bottom": "14px",}))]),
  ])
], color="#ff9100",
   style={
     "width":"70rem",
     "margin-top":"1rem",
     "margin-bottom":"1rem",
     "margin-right":"0rem"

   })

piechartcard = dbc.Card([
  dbc.CardBody([
    html.H1("Rata-Rata Umur hobby", className='card-title'),
    html.P("", className='card-body'),
    html.Label(['Choose column:'],style={'font-weight': 'bold'}),
    dbc.DropdownMenuItem(dcc.Dropdown(id="dropdown",multi= True,
       options=[{'label':x,'value': x}for x in sorted(dff.hobby)],
       value= ["running","karaoke"],),style={"width":"65rem","color":"#014c63"}),
    html.Br(),
    dcc.Graph(id='graph', figure= {},),
  ])
],
  outline=True,
  color='secondary',
  style={
    "width":"71rem",
    "margin-right":"0rem",
    "margin-left":"1rem",
    "margin-bottom":"1rem"
  }
)

markdown = ''' Most Follower
            Grafik Diatas merupakan hasil Dari algoritma Query
            untuk menghitung berapa user account yang mempunyai follower terbanyak
            Grafik ini merupakan hasil dari Graph Schema social media
            yang dimana merepresentasikan keterhubungan antar user account
  '''
# piechartcard = dbc.Card([
#   dbc.CardBody([
#     html.H1("Rata-Rata Umur hobby", className='card-title'),
#     html.P("", className='card-body'),
#     dcc.Graph(id='graph', figure= fig3)
#   ])
# ],
#   outline=True,
#   color='secondary',
#   style={
#     "width":"70rem",
#     "margin-right":"0rem",
#     "margin-left":"2rem",
#     "margin-bottom":"1rem"
#   }
# )

bar = html.Div([html.Header("Most Followers"),
                dcc.Graph(id='bar', figure= fig)
              ])

bar1 = html.Div([html.Header("PageRank"),
                 dcc.Graph(id='barchart' ,figure=fig1)
])

barchartcard = dbc.Card([
  dbc.CardBody([
    html.H1("Most Followers", className='card-title'),
    html.P("", className='card-body'),
    dcc.Graph(id='bar', figure= fig)
  ])
],
  outline=True,
  color='secondary',
  style={
    "width":"35rem",
    "margin-right":"0rem",
    "margin-left":"1rem",
    "margin-bottom":"1rem"
  }
)

barchartcard1 = dbc.Card([
  dbc.CardBody([
    html.H1("Algoritma Pagerank", className='card-title'),
    html.P("", className='card-body'),
    dcc.Graph(id='barchart', figure= fig1)
  ])
],
  outline=True,
  color='secondary',
  style={
    "width":"35rem",
    "margin-right":"0rem",
    "margin-left":"1rem",
    "margin-bottom":"1rem"
  }
)
SIDEBAR_STYLE = {
    "position": "fixed",
    "top": 0,
    "left": 0,
    "bottom": 0,
    "width": "16rem",
    "padding": "2rem 1rem",
    "background-color": "#424242",
}

WHITE = "#fffff8"
TG_LOGO = "https://i.pinimg.com/originals/98/c7/e9/98c7e95ea1207eef6f70f0ae9e694e67.png"
sidebar =dbc.Card([html.Div(
    [
        html.H2("Main Menu", className="display-5"),
        html.Hr(),
        html.Label(
            "TigerGraph as a GraphDatabase", className="display-6"
        ),
        html.Hr(),
        dbc.Nav(
            [
                dbc.NavLink("Home", href="/", active="exact"),
                dbc.NavLink("Page 1", href="/page-1", active="exact"),
                dbc.NavLink("Page 2", href="/page2", active="exact"),
                dbc.NavLink("Page 3", href="/page3", active="exact"),
            ],
            vertical=True,
            pills=True,

        ),
        html.Hr(),
        html.Li(html.A("Inroduction to Graphs", href='https://www.tigergraph.com/blog/what-is-a-graph-database-and-why-should-you-care/', target="_blank", style={'color':WHITE}), style={'color':WHITE}),
        html.Br(),
        html.Li(html.A("Introduction to Dash", href='https://www.youtube.com/watch?v=e4ti2fCpXMI', target="_blank", style={'color':WHITE}), style={'color':WHITE}),
        html.Br(),
        html.Br(),
        html.Br(),
        html.Br(),
        html.Br(),
        html.Label(["Achmad Irfandi Darmawan"],style= {"fontcolor":"black"})
    ],
    style=SIDEBAR_STYLE,
    className="card"
)])

page = html.Center(html.Div([
  titlecard,
  dbc.Row([barchartcard,
    barchartcard1,piechartcard
  ],justify='center'),

]
, style={"width":"90rem"}))

# page1 = html.Center(html.Div([
#   titlecard,
#   dbc.Row(dbc.Col([
#     bar
#   ],width={'size': 12, "offset": 0}),)

# ]
# , ))

page1 =dbc.Card([
        dbc.CardBody([
        html.H1("Most Follower", className='card-title'),
        html.P("", className='card-body'),
        bar,
        html.Div([html.Label(dcc.Markdown(children=markdown),className="display-5")])
          ])
        ],outline=True,
  color='info',
  style={
    "width":"89rem",
    "margin-right":"1rem",
    "margin-left":"18rem",
    "margin-bottom":"1rem",
    "margin-top":"5rem"
  }),

page2 =dbc.Card([
        dbc.CardBody([
        html.H1("PageRank", className='card-title'),
        html.P("", className='card-body'),
        bar1,
        html.Div()
          ])
        ],outline=True,
  color='info',
  style={
    "width":"89rem",
    "margin-right":"1rem",
    "margin-left":"18rem",
    "margin-bottom":"1rem",
    "margin-top":"5rem"
  }),
dcc.Markdown('''# Most Follower

               ## Grafik Diatas merupakan hasil Dari algoritma Queri
               untuk menghitung berapa user account yang mempunyai
               follower terbanyak

               ## Grafik ini merupakan hasil dari Graph Schema social media
               yang dimana merepresentasikan keterhubungan antar user account
  ''')

# page2= html.Center(html.Div([
#   titlecard,
#   dbc.Row([
#     barchartcard1,
#   ],justify='center'),

# ]
# , style={"width":"90rem"}))

page3= html.Center(html.Div([

  dbc.Row([
    piechartcard,
  ],justify='center'),

]
, style={"width":"90rem"}))

CONTENT_STYLE = {
    "margin-left": "2rem",
    "margin-right": "0rem",
    "padding": "1rem 1rem",
}

# app.layout= html.Div([
#   sidebar,
#   html.Div(page),

# ])

app.layout = html.Div([
    dcc.Location(id='url', refresh=False),
    sidebar,
    html.Div(id='page-content', style=CONTENT_STYLE)
])

@app.callback(dash.dependencies.Output('page-content', 'children'),
              [dash.dependencies.Input('url', 'pathname')],suppress_callback_exceptions=True)

def display_page(pathname):
  if pathname == "/":
    return page
  elif pathname == "/page-1":
    return page1
  elif pathname == "/page2":
    return page2
  elif pathname == "/page3":
    return page3

@app.callback(Output(component_id="graph", component_property='figure'),
[Input(component_id='dropdown', component_property='value')],suppress_callback_exceptions=True)

def update_my_graph(val_chosen):
  if len(val_chosen)>0:
    print(f"value user choose:{val_chosen}")
    print (type(val_chosen))
    dff1 = dff[dff["hobby"].isin(val_chosen)]
    pig = px.pie(dff1,values="rata2umur", names="hobby")
    pig.update_layout(
    plot_bgcolor=colors['background'],
    paper_bgcolor=colors['background'],
    font_color=colors['text']
)
    return pig

# style= CONTENT_STYLE

if __name__ == '__main__':
    app.run_server(debug=True)


import glob
import json
import os
import shutil
from datetime import datetime

import imagesize
from detectron2.config import get_cfg
from detectron2.data.datasets import register_coco_instances
from detectron2.engine import DefaultPredictor
import numpy as np
import torch
import matplotlib.gridspec as gridspec

def simple_filename(filename_ext):
    filename_base = os.path.basename(filename_ext)
    filename_noext = os.path.splitext(filename_base)[0]
    return filename_noext

def model_loader(settings):
    cfg = get_cfg()
    cfg.merge_from_file(settings["yaml_path"])
    cfg.MODEL.WEIGHTS = settings["pkl_path"]
    # cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5
    model = DefaultPredictor(cfg).model
    model.eval()
    for param in model.parameters():
        param.requires_grad = False
    return model, cfg

def pick_coco_exp(name, targetlist):
    if os.path.isdir(name):
        shutil.rmtree(name)
    os.makedirs(name, exist_ok=True)

    coco_path = os.environ["DETECTRON2_DATASETS"]
    anno_path = "./dataset/annotations/instances_OpenImage_v6.json"

    file_list = glob.glob(os.path.join(coco_path, "*.jpg"))
    file_list = [x for x in file_list if simple_filename(x) in targetlist]

    file_name_list = [os.path.basename(x) for x in file_list]
    with open(anno_path, "r") as anno_file:
        coco_json = json.load(anno_file)
    my_json = {}
    my_json["info"] = coco_json["info"]
    my_json["licenses"] = coco_json["licenses"]
    my_json["images"] = []
    my_json["annotations"] = []
    my_json["categories"] = coco_json["categories"]

    my_json["images"].extend(
        [x for x in coco_json["images"] if x["file_name"] in file_name_list]
    )
    image_id_list = [x["id"] for x in my_json["images"]]
    my_json["annotations"].extend(
        [x for x in coco_json["annotations"] if x["image_id"] in image_id_list]
    )

    for filepath in file_list:
        shutil.copy(filepath, name)
    with open(f"{name}/my_anno.json", "w") as my_file:
        my_file.write(json.dumps(my_json))
    register_coco_instances(name, {}, f"{name}/my_anno.json", name)

def print_settings(settings, index):
    model_name = settings["model_name"]
    VTM_param = settings["VTM"]
    print()
    print("Evaluation of proposed methods for:", model_name.upper())
    print(f"Settings ID: {index}")
    print(f"VTM paramerters      : {VTM_param}")

import cv2
import numpy as np

def save_feature_map(filename, features):
    features_draw = features.copy()
    del features_draw["p6"]
    _save_feature_map(filename, features_draw)

def _save_feature_map(filename, features, debug=False):
    feat = [features["p2"].squeeze(), features["p3"].squeeze(), features["p4"].squeeze(), features["p5"].squeeze()]
    width_list = [16, 32, 64, 128]
    height_list = [16, 8, 4, 2]
    tile_big = np.empty((0, feat[0].shape[2] * width_list[0]))
    for blk, width, height in zip(feat, width_list, height_list):
        big_blk = np.empty((0, blk.shape[2] * width))
        for row in range(height):
            big_blk_col = np.empty((blk.shape[1], 0))
            for col in range(width):
                tile = blk[col + row * width].cpu().numpy()
                if debug:
                    cv2.putText(
                        tile,
                        f"{col + row * width}",
                        (32, 32),
                        cv2.FONT_HERSHEY_SIMPLEX,
                        0.5,
                        (255, 255, 255),
                        1,
                    )
                big_blk_col = np.hstack((big_blk_col, tile))
            big_blk = np.vstack((big_blk, big_blk_col))
        tile_big = np.vstack((tile_big, big_blk))
    tile_big = tile_big.astype(np.uint16)
    cv2.imwrite(filename, tile_big)

def result_in_list(settings, number, result, set_index):
    res = list(result.values())[0]
    ap = res["AP"]
    ap50 = res["AP50"]
    aps = res["APs"]
    apm = res["APm"]
    apl = res["APl"]

    return [
        datetime.now(),
        set_index,
        number,
        f"{ap:.3f}",
        f"{ap50:.3f}",
        f"{aps:.3f}",
        f"{apm:.3f}",
        f"{apl:.3f}",
    ]

def get_size(start_path = '.'):
    total_size = 0
    for dirpath, dirnames, filenames in os.walk(start_path):
        for f in filenames:
            fp = os.path.join(dirpath, f)
            # skip if it is symbolic link
            if not os.path.islink(fp):
                total_size += os.path.getsize(fp)

    return total_size


import my_bag.file_util
import my_bag.str_util

#反转字符串
str=input("输入字符串")
print(my_bag.str_util.str_reverse(str))

#切片
str=input("输入字符串")
x=int(input("输入首下标"))
y=int(input("输入尾下标"))
print(my_bag.str_util.substr(str,x,y))

#打印文件内容
name=input("输入文件名")
my_bag.file_util.print_file_info(name)

#追加写入数据到文件
str=input("输入字符串")
name=input("输入文件名")
my_bag.file_util.append_to_file(name,str)


#!/usr/bin/env python
# coding: utf-8

import warnings
warnings.filterwarnings(action='ignore', category=FutureWarning)
warnings.filterwarnings('ignore')
import pandas as pd
import argparse
import subprocess
import shutil
import sys
import os.path
import time
import lightgbm as lgb
from catboost import CatBoostClassifier
from sklearn.metrics import balanced_accuracy_score
# from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import make_scorer
from sklearn.model_selection import cross_val_score
from sklearn.metrics import f1_score
from hyperopt import hp, fmin, tpe, STATUS_OK, Trials
pd.options.mode.chained_assignment = None  # default='warn'

from sklearn.model_selection import train_test_split

#from source.repDNA.repDNA.nac import *
#from source.repDNA.repDNA.psenac import *
#from source.repDNA.repDNA.ac import *

path = os.path.dirname(os.path.abspath(__file__))
sys.path.append(path + '/source/repDNA/repDNA/')
from nac import *
from psenac import *
from ac import *

def make_path(path_input):
    """
    Group some strings in a path

    Arg:
    path_input (list): a list of all parts of the path
    Result:
    path_output (str): a output path
    """

    path_output = []
    for i in range(len(path_input)):
        path_output.append(os.path.join(path_input[i]))

    return path_output

def check_path(paths, type_path='This path'):
    """
    Check if this path exist

    Args:
    paths (str): a list of all paths to check its existence
    type_path (str): path name for identification
    """
    for subpath in paths:
        if os.path.exists(subpath):
            print(f'{type_path} - {subpath}: Found File')
        else:
            print(f'{type_path} - {subpath}: File not exists')
            sys.exit()

class extraction(): #extraction class of the datasets

    """Extracts the features from the sequences in the fasta files."""

    def __init__(self, ftrain, flabel, ftype, foutput, ftest=''):
        self.ftrain = ftrain
        self.flabel = flabel
        self.ftype = ftype
        self.foutput = foutput
        self.ftest = ftest

        self.path = os.path.join(foutput, 'feat_extraction')
        self.path_results = foutput

    def delete_directory(self):
        """Excludes the output directory"""
        try:
            shutil.rmtree(self.path)
            shutil.rmtree(self.path_results)
        except OSError as e:
            print(f"Error: {e.filename} - {e.strerror}.")
            print('Creating Directory...')

    def check(self):
        """Checks and creates all output directories"""

        if not os.path.exists(self.path_results):
          os.mkdir(self.path_results)

        if not os.path.exists(self.path):
            os.mkdir(self.path)
            os.mkdir(os.path.join(self.path, 'train'))
            os.mkdir(os.path.join(self.path, 'test'))

        if not os.path.exists(os.path.join(self.path, 'train', self.ftype)):
            os.mkdir(os.path.join(self.path, 'train', self.ftype))
            os.mkdir(os.path.join(self.path, 'test', self.ftype))

    def extrac_features(self):
        """Uses the input information to extract the features"""

        global features_nucleot, features_amino

        labels = [self.flabel]
        fasta = [self.ftrain]
        train_size = 0

        if self.ftest:
            labels.append(self.flabel)
            fasta.append(self.ftest)

        datasets = []
        fasta_list = []

        nucl_type = {'dna': '1', 'rna': '2'}

        print(f'{labels[0]} - Extracting features with MathFeature...')

        assert self.ftype in ['dna', 'rna', 'protein'], f"Error: Sequence type {self.ftype} not expected "

        if self.ftype == "dna" or self.ftype == "rna":
            features_nucleot = [1,2,3]#,4,5,6,7,8,9,10,13,14,15,16,17,18]#,13,14,15,16,17,18]#,2]#,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18]
            """Feature extraction for nucleotide-based sequences """
            for i in range(len(fasta)):
                for j in range(len(fasta[i])):
                    file = fasta[i][j].split('/')[-1] # i: train/test; j: label 1 or 2
                    if i == 0:
                        preprocessed_fasta = os.path.join(self.path + '/train' + '/'+self.ftype +'/pre_' + file)
                        #print(preprocessed_fasta)
                        subprocess.run(['python', './source/other-methods/preprocessing.py',
                                        '-i', fasta[i][j], '-o', preprocessed_fasta],
                                        stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)

                        train_size += len([1 for line in open(preprocessed_fasta) if line.startswith(">")])
                    else:  # Test
                        preprocessed_fasta = os.path.join(self.path + '/test'+ '/'+self.ftype +'/pre_' + file)
                        subprocess.run(['python', './source/other-methods/preprocessing.py',
                                        '-i', fasta[i][j], '-o', preprocessed_fasta],
                                        stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
                    fasta_list.append(preprocessed_fasta)

                    if 1 in features_nucleot:
                        dataset = os.path.join(self.path, 'NAC_' + self.ftype + '.csv')
                        subprocess.run(['python', './source/MathFeature/methods/ExtractionTechniques.py',
                                        '-i', preprocessed_fasta, '-o', dataset, '-l', labels[i],
                                        '-t', 'NAC', '-seq', nucl_type[self.ftype]], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
                        datasets.append(dataset)

                    if 2 in features_nucleot:
                        dataset = os.path.join(self.path, 'DNC_' + self.ftype + '.csv')
                        subprocess.run(['python', './source/MathFeature/methods/ExtractionTechniques.py', '-i',
                                        preprocessed_fasta, '-o', dataset, '-l', labels[i],
                                        '-t', 'DNC', '-seq', nucl_type[self.ftype]], stdout=subprocess.DEVNULL,
                                        stderr=subprocess.STDOUT)
                        datasets.append(dataset)

                    if 3 in features_nucleot:
                        dataset = os.path.join(self.path, 'TNC_' + self.ftype + '.csv')
                        subprocess.run(['python', './source/MathFeature/methods/ExtractionTechniques.py', '-i',
                                        preprocessed_fasta, '-o', dataset, '-l', labels[i],
                                        '-t', 'TNC', '-seq', nucl_type[self.ftype]], stdout=subprocess.DEVNULL,
                                        stderr=subprocess.STDOUT)
                        datasets.append(dataset)

                    if 4 in features_nucleot:
                        dataset_di = os.path.join(self.path, 'kGap_di_' + self.ftype + '.csv')
                        dataset_tri = os.path.join(self.path, 'kGap_tri_' + self.ftype + '.csv')

                        subprocess.run(['python', './source/MathFeature/methods/Kgap.py', '-i',
                                        preprocessed_fasta, '-o', dataset_di, '-l',
                                        labels[i], '-k', '1', '-bef', '1',
                                        '-aft', '2', '-seq', nucl_type[self.ftype]],
                                        stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)

                        subprocess.run(['python', './source/MathFeature/methods/Kgap.py', '-i',
                                        preprocessed_fasta, '-o', dataset_tri, '-l',
                                        labels[i], '-k', '1', '-bef', '1',
                                        '-aft', '3', '-seq', nucl_type[self.ftype]],
                                        stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
                        datasets.append(dataset_di)
                        datasets.append(dataset_tri)

                    if 5 in features_nucleot:
                        dataset = os.path.join(self.path, 'ORF_' + self.ftype + '.csv')
                        subprocess.run(['python', './source/MathFeature/methods/CodingClass.py', '-i',
                                        preprocessed_fasta, '-o', dataset, '-l', labels[i]],
                                        stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
                        datasets.append(dataset)

                    if 6 in features_nucleot:
                        dataset = os.path.join(self.path, 'Fickett_' + self.ftype + '.csv')
                        subprocess.run(['python', './source/MathFeature/methods/FickettScore.py', '-i',
                                        preprocessed_fasta, '-o', dataset, '-l', labels[i],
                                        '-seq', nucl_type[self.ftype]], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
                        datasets.append(dataset)

                    if 7 in features_nucleot:
                        dataset = os.path.join(self.path, 'Shannon_' + self.ftype + '.csv')
                        subprocess.run(['python', './source/MathFeature/methods/EntropyClass.py', '-i',
                                        preprocessed_fasta, '-o', dataset, '-l', labels[i],
                                        '-k', '5', '-e', 'Shannon'],
                                        stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
                        datasets.append(dataset)

                    if 8 in features_nucleot:
                        dataset =os.path.join(self.path, 'FourierBinary_' + self.ftype + '.csv')
                        subprocess.run(['python', './source/MathFeature/methods/FourierClass.py', '-i',
                                        preprocessed_fasta, '-o', dataset, '-l', labels[i],
                                        '-r', '1'], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
                        datasets.append(dataset)

                    if 9 in features_nucleot:
                        dataset = os.path.join(self.path, 'FourierComplex_' + self.ftype + '.csv')
                        subprocess.run(['python', './source/other-methods/FourierClass.py', '-i',
                                        preprocessed_fasta, '-o', dataset, '-l', labels[i],
                                        '-r', '6'], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
                        datasets.append(dataset)

                    if 10 in features_nucleot:
                        dataset = os.path.join(self.path, 'Tsallis_' + self.ftype + '.csv')
                        subprocess.run(['python', './source/other-methods/TsallisEntropy.py', '-i',
                                        preprocessed_fasta, '-o', dataset, '-l', labels[i],
                                        '-k', '5', '-q', '2.3'], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
                        datasets.append(dataset)

                    if '12' in features_nucleot:
                        #dataset = path + '/BinaryMapping.csv'
                        dataset = os.path.join(self.path, 'BinaryMapping_' + self.ftype + '.csv')
                        #labels_list = ftrain_labels + ftest_labels
                        labels_list = [self.flabel] + [self.flabel]
                        text_input = ''
                        for i in range(len(fasta_list)):
                            text_input += fasta_list[i] + '\n' + labels_list[i] + '\n'

                        subprocess.run(['python', './source/MathFeature/methods/MappingClass.py',
                                        '-n', str(len(fasta_list)), '-o',
                                        dataset, '-r', '1'], text=True, input=text_input,
                                       stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)

                        with open(dataset, 'r') as temp_f:
                            col_count = [len(l.split(",")) for l in temp_f.readlines()]

                        colnames = ['BinaryMapping_' + str(i) for i in range(0, max(col_count))]

                        df = pd.read_csv(dataset, names=colnames, header=None)
                        df.rename(columns={df.columns[0]: 'nameseq', df.columns[-1]: 'label'}, inplace=True)
                        df.to_csv(dataset, index=False)

                        datasets.append(dataset)

                if 13 in features_nucleot:
                    rev_kmer = RevcKmer(k=3, normalize=True, upto=True)
                    data_kmer = rev_kmer.make_revckmer_vec(open(fasta_list[0]))
                    data_kmer = pd.DataFrame(data_kmer)
                    data_kmer = data_kmer.add_prefix('Rev_kmer_')
                    dataset = os.path.join(self.path, 'Rev_kmer_'  + self.ftype + '.csv')
                    data_kmer.to_csv(dataset, index=False)
                    datasets.append(dataset)

                if 14 in features_nucleot:
                    psednc = PseDNC()
                    data_psednc = psednc.make_psednc_vec(open(fasta_list[0]))
                    data_psednc = pd.DataFrame(data_psednc)
                    data_psednc = data_psednc.add_prefix('Pse_dnc_')
                    dataset = os.path.join(self.path, 'Pse_dnc_'  + self.ftype + '.csv')
                    data_psednc.to_csv(dataset, index=False)
                    datasets.append(dataset)

                if 15 in features_nucleot:
                    pseknc = PseKNC()
                    data_pseknc = pseknc.make_pseknc_vec(open(fasta_list[0]))
                    data_pseknc = pd.DataFrame(data_pseknc)
                    data_pseknc = data_pseknc.add_prefix('Pse_knc_')
                    dataset = os.path.join(self.path, 'Pse_knc_'  + self.ftype + '.csv')
                    data_pseknc.to_csv(dataset, index=False)
                    datasets.append(dataset)

                if 16 in features_nucleot:
                    sc_psednc = SCPseDNC()
                    data_sc_psednc = sc_psednc.make_scpsednc_vec(open(fasta_list[0]), all_property=True)
                    data_sc_psednc = pd.DataFrame(data_sc_psednc)
                    data_sc_psednc = data_sc_psednc.add_prefix('SCPseDNC_')
                    dataset = os.path.join(self.path, 'SCPseDNC_'  + self.ftype + '.csv')
                    data_sc_psednc.to_csv(dataset, index=False)
                    datasets.append(dataset)

                if 17 in features_nucleot:
                    sc_psetnc = SCPseTNC(lamada=2, w=0.05)
                    data_sc_psetnc = sc_psetnc.make_scpsetnc_vec(open(fasta_list[0]), all_property=True)
                    data_sc_psetnc = pd.DataFrame(data_sc_psetnc)
                    data_sc_psetnc = data_sc_psetnc.add_prefix('SCPseTNC_')
                    dataset = os.path.join(self.path, 'SCPseTNC_'  + self.ftype + '.csv')
                    data_sc_psetnc.to_csv(dataset, index=False)
                    datasets.append(dataset)

                if 18 in features_nucleot:
                    dac = DAC(2)
                    data_dac = dac.make_dac_vec(open(fasta_list[0]), all_property=True)
                    data_dac = pd.DataFrame(data_dac)
                    data_dac = data_dac.add_prefix('DAC_')
                    dataset = os.path.join(self.path, 'DAC_'  + self.ftype + '.csv')
                    data_dac.to_csv(dataset, index=False)
                    datasets.append(dataset)

                if '19' in features_nucleot:
                    tac = TAC(2)
                    data_tac = tac.make_tac_vec(open(fasta_list[0]), all_property=True)
                    data_tac = pd.DataFrame(data_tac)
                    data_tac = data_tac.add_prefix('TAC_')
                    dataset = os.path.join(self.path, 'TAC_'  + self.ftype + '.csv')
                    data_tac.to_csv(dataset, index=False)
                    datasets.append(dataset)

                if '20' in features_nucleot:
                    tcc = TCC(2)
                    data_tcc = tcc.make_tcc_vec(open(fasta_list[0]), all_property=True)
                    data_tcc = pd.DataFrame(data_tcc)
                    data_tcc = data_tcc.add_prefix('TCC_')
                    dataset = os.path.join(self.path, 'TCC_'  + self.ftype + '.csv')
                    data_tcc.to_csv(dataset, index=False)
                    datasets.append(dataset)

                if '21' in features_nucleot:
                    tacc = TACC(2)
                    data_tacc = tacc.make_tacc_vec(open(fasta_list[0]), all_property=True)
                    data_tacc = pd.DataFrame(data_tacc)
                    data_tacc = data_tacc.add_prefix('TACC_')
                    dataset = os.path.join(self.path, 'TACC_'  + self.ftype + '.csv')
                    data_tacc.to_csv(dataset, index=False)
                    datasets.append(dataset)

        if self.ftype == "protein":

            features_amino = [1,2,3]#,4,6,7,8,9,10,11]
            """Feature extraction for aminoacids-based sequences"""
            for i in range(len(fasta)):
                for j in range(len(fasta[i])):
                    file = fasta[i][j].split('/')[-1]
                    if i == 0:  # Train
                        preprocessed_fasta = os.path.join(self.path + '/train' + '/'+self.ftype +'/pre_' + file)
                        subprocess.run(['python', './source/other-methods/preprocessing.py',
                                        '-i', fasta[i][j], '-o', preprocessed_fasta],
                                        stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
                        train_size += len([1 for line in open(preprocessed_fasta) if line.startswith(">")])
                    else:  # Test
                        preprocessed_fasta = os.path.join(self.path + '/test'+ '/'+self.ftype +'/pre_' + file)
                        subprocess.run(['python', './source/other-methods/preprocessing.py',
                                        '-i', fasta[i][j], '-o', preprocessed_fasta],
                                        stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
                    fasta_list.append(preprocessed_fasta)

                    if 1 in features_amino:
                        dataset = os.path.join(self.path, 'Shannon_' + self.flabel + '.csv')
                        subprocess.run(['python', './source/MathFeature/methods/EntropyClass.py',
                                        '-i', preprocessed_fasta, '-o', dataset, '-l', labels[i],
                                        '-k', '5', '-e', 'Shannon'], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
                        datasets.append(dataset)

                    if 2 in features_amino:
                        dataset = os.path.join(self.path, 'Tsallis_23_'  + self.flabel + '.csv')
                        subprocess.run(['python', './source/other-methods/TsallisEntropy.py',
                                        '-i', preprocessed_fasta, '-o', dataset, '-l', labels[i],
                                        '-k', '5', '-q', '2.3'], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
                        datasets.append(dataset)

                    if 3 in features_amino:
                        dataset = os.path.join(self.path, 'Tsallis_30_'  + self.flabel + '.csv')
                        subprocess.run(['python', './source/other-methods/TsallisEntropy.py',
                                        '-i', preprocessed_fasta, '-o', dataset, '-l', labels[i],
                                        '-k', '5', '-q', '3.0'], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
                        datasets.append(dataset)

                    if 4 in features_amino:
                        dataset = os.path.join(self.path, 'Tsallis_40_'  + self.flabel + '.csv')
                        subprocess.run(['python', './source/other-methods/TsallisEntropy.py',
                                        '-i', preprocessed_fasta, '-o', dataset, '-l', labels[i],
                                        '-k', '5', '-q', '4.0'], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
                        datasets.append(dataset)

                    if 5 in features_amino:
                        dataset = os.path.join(self.path, 'ComplexNetworks_' + self.flabel + '.csv')
                        subprocess.run(['python', './source/MathFeature/methods/ComplexNetworksClass-v2.py', '-i',
                                        preprocessed_fasta, '-o', dataset, '-l', labels[i],
                                        '-k', '3'], stdout=subprocess.DEVNULL,
                                        stderr=subprocess.STDOUT)
                        datasets.append(dataset)

                    if 6 in features_amino:
                        dataset_di = os.path.join(self.path, 'kGap_di_' + self.flabel + '.csv')
                        subprocess.run(['python', './source/MathFeature/methods/Kgap.py', '-i',
                                        preprocessed_fasta, '-o', dataset_di, '-l',
                                        labels[i], '-k', '1', '-bef', '1',
                                        '-aft', '1', '-seq', '3'],
                                        stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
                        datasets.append(dataset_di)

                    if 7 in features_amino:
                        dataset = os.path.join(self.path, 'AAC_' + self.flabel + '.csv')
                        subprocess.run(['python', './source/MathFeature/methods/ExtractionTechniques-Protein.py', '-i',
                                        preprocessed_fasta, '-o', dataset, '-l', labels[i],
                                        '-t', 'AAC'], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
                        datasets.append(dataset)

                    if 8 in features_amino:
                        dataset = os.path.join(self.path, 'DPC_' + self.flabel + '.csv')
                        subprocess.run(['python', './source/MathFeature/methods/ExtractionTechniques-Protein.py', '-i',
                                        preprocessed_fasta, '-o', dataset, '-l', labels[i],
                                        '-t', 'DPC'], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
                        datasets.append(dataset)

                    if '9' in features_amino:
                        dataset = os.path.join(self.path, 'TPC_'  + self.flabel + '.csv')
                        subprocess.run(['python', './source/MathFeature/methods/ExtractionTechniques-Protein.py', '-i',
                                        preprocessed_fasta, '-o', dataset, '-l', labels[i],
                                        '-t', 'TPC'], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
                        datasets.append(dataset)

                    if '10' in features_amino:
                        #dataset = path + '/Fourier_Integer.csv'
                        dataset = os.path.join(self.path, 'Fourier_Integer_'  + self.flabel + '.csv')
                        labels_list = [self.flabel] + [self.flabel]
                        text_input = ''
                        for i in range(len(fasta_list)):
                            text_input += fasta_list[i] + '\n' + labels_list[i] + '\n'

                        subprocess.run(['python', './source/MathFeature/methods/Mappings-Protein.py',
                                        '-n', str(len(fasta_list)), '-o',
                                        dataset, '-r', '6'], text=True, input=text_input,
                                       stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)

                        with open(dataset, 'r') as temp_f:
                            col_count = [len(l.split(",")) for l in temp_f.readlines()]

                        colnames = ['Integer_Fourier_' + str(i) for i in range(0, max(col_count))]

                        df = pd.read_csv(dataset, names=colnames, header=0)
                        df.rename(columns={df.columns[0]: 'nameseq', df.columns[-1]: 'label'}, inplace=True)
                        df = df.iloc[:, :-1]
                        df.to_csv(dataset, index=False)
                        datasets.append(dataset)

                    if 11 in features_amino:
                        dataset = os.path.join(self.path, 'Fourier_EIIP_'  + self.flabel + '.csv')
                        labels_list = [self.flabel] + [self.flabel]
                        text_input = ''
                        for i in range(len(fasta_list)):
                            text_input += fasta_list[i] + '\n' + labels_list[i] + '\n'

                        subprocess.run(['python', './source/MathFeature/methods/Mappings-Protein.py',
                                        '-n', str(len(fasta_list)), '-o',
                                        dataset, '-r', '8'], text=True, input=text_input,
                                       stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)

                        with open(dataset, 'r') as temp_f:
                            col_count = [len(l.split(",")) for l in temp_f.readlines()]

                        colnames = ['EIIP_Fourier_' + str(i) for i in range(0, max(col_count))]

                        df = pd.read_csv(dataset, names=colnames, header=0)
                        df.rename(columns={df.columns[0]: 'nameseq', df.columns[-1]: 'label'}, inplace=True)
                        df = df.iloc[:, :-1]
                        df.to_csv(dataset, index=False)
                        datasets.append(dataset)

                    if '12' in features_amino:
                        dataset = os.path.join(self.path, 'EIIP_'  + self.flabel + '.csv')
                        labels_list = [self.flabel] + [self.flabel]
                        text_input = ''
                        for i in range(len(fasta_list)):
                            text_input += fasta_list[i] + '\n' + labels_list[i] + '\n'

                        subprocess.run(['python', './source/MathFeature/methods/Mappings-Protein.py',
                                        '-n', str(len(fasta_list)), '-o',
                                        dataset, '-r', '7'], text=True, input=text_input,
                                       stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)

                        with open(dataset, 'r') as temp_f:
                            col_count = [len(l.split(",")) for l in temp_f.readlines()]

                        colnames = ['EIIP_' + str(i) for i in range(0, max(col_count))]

                        df = pd.read_csv(dataset, names=colnames, header=None)
                        f.rename(columns={df.columns[0]: 'nameseq', df.columns[-1]: 'label'}, inplace=True)
                        df = df.iloc[:, :-1]
                        df.to_csv(dataset, index=False)
                        datasets.append(dataset)

                    if '13' in features_amino:
                        dataset = os.path.join(self.path, 'AAAF_'  + self.flabel + '.csv')
                        labels_list = [self.flabel] + [self.flabel]
                        text_input = ''
                        for i in range(len(fasta_list)):
                            text_input += fasta_list[i] + '\n' + labels_list[i] + '\n'

                        subprocess.run(['python', './source/MathFeature/methods/Mappings-Protein.py',
                                        '-n', str(len(fasta_list)), '-o',
                                        dataset, '-r', '1'], text=True, input=text_input,
                                       stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)

                        with open(dataset, 'r') as temp_f:
                            col_count = [len(l.split(",")) for l in temp_f.readlines()]

                        colnames = ['AccumulatedFrequency_' + str(i) for i in range(0, max(col_count))]

                        df = pd.read_csv(dataset, names=colnames, header=None)
                        df.rename(columns={df.columns[0]: 'nameseq', df.columns[-1]: 'label'}, inplace=True)
                        df = df.iloc[:, :-1]
                        df.to_csv(dataset, index=False)
                        datasets.append(dataset)

        """Concatenating all the extracted features"""

        assert datasets != [], 'any features was extratc, there is any error'

        datasets = list(dict.fromkeys(datasets))
        dataframes = pd.concat([pd.read_csv(f) for f in datasets], axis=1)
        dataframes = dataframes.loc[:, ~dataframes.columns.duplicated()]
        dataframes['nameseq'] = dataframes['nameseq'].astype(str)
        dataframes = dataframes[~dataframes.nameseq.str.contains("nameseq")]
        #print(dataframes)
        columns = list(dataframes.columns)
        for i in range(1,len(columns)):
            columns[i] = columns[i] + '_' + self.flabel
        dataframes.columns=columns

        X_train = dataframes.iloc[:train_size, :]
        X_train.pop('label'+ '_' + self.flabel)
        ftrain = self.path + '/train' + '/'+self.ftype+ '/ftrain_'+self.flabel+'.csv'
        X_train.to_csv(ftrain, index=False)

        ftest = ''
        X_test = []

        if self.ftest:
            X_test = dataframes.iloc[train_size:, :]
            X_test.pop('label'+ '_' + self.flabel)
            ftest = self.path + '/test'+ '/'+self.ftype + '/ftest_'+self.flabel+'.csv'
            X_test.to_csv(ftest, index=False)

        return  X_train, X_test

def make_dataset(data1, data2, label_1, label_2, table, output, arq_name):
    """
    Concatenates both datasets

    Args:
    data1 (pandas.DataFrame): First dataset.
    data2 (pandas.DataFrame): Second dataset.
    label_1 (str): Label for the first dataset.
    label_2 (str): Label for the second dataset.
    table (pandas.DataFrame): Interaction table.
    output (str): Output directory.
    arq_name (str): File name for output.

    Returns:
    Tuple of output file paths (str) for data and labels.

    """

    data = data1
    seqnames = data1['nameseq']
    #print(seqnames)

    if len(table.columns)-1 != 0:
        for i in range(len(table.columns)-1):
            seqnames = data1['nameseq']
            seqnames = seqnames +'_'+ data2['nameseq'].iloc[i]

            new_rows= data2.loc[i:i]
            labels = list(new_rows)

            new_rows = new_rows.T
            new_rows = new_rows[i].tolist()
            new_rows = [new_rows[1:len(new_rows)]] * len(table[table.columns])
            data[labels[1:]] = new_rows
            data['Label'] = table[table.columns[i+1]]

            if i == 0:
                dataf = data.copy()
                fnameseq = seqnames
            else:
                dataf = pd.concat([dataf, data])
                fnameseq = pd.concat([fnameseq, seqnames])
        data_concat = dataf
        data_concat.reset_index()
        data_concat.index = range(len(data_concat.index))

        fnameseq.reset_index()
        fnameseq.index = range(len(fnameseq.index))

        data_concat = data_concat.drop('nameseq', axis=1)
        y = data_concat.pop('Label')

    else:
        data_names = data1['nameseq'] +'-'+ data2['nameseq']
        fnameseq = data_names.to_frame()

        data1 = data1.drop('nameseq', axis=1)
        data2 = data2.drop('nameseq', axis=1)
        data_concat = data1.join(data2)
        y = table

    assert len(data_concat) > 0, 'Interaction table is blank or no sequence pairs were found'

    output_dir = os.path.join(output, 'model_data')
    os.makedirs(output_dir, exist_ok=True)

    foutput_label = os.path.join(output_dir, f'{arq_name}_label.csv')
    foutput_data = os.path.join(output_dir, f'{arq_name}.csv')
    foutput_nameseq = os.path.join(output_dir, f'{arq_name}_nameseq.csv')

    y.to_csv(foutput_label, index=False)
    data_concat.to_csv(foutput_data, index=False)
    fnameseq.to_csv(foutput_nameseq, index=False)

    return foutput_data, foutput_label, foutput_nameseq

def create_test(foutput_data1, foutput_label1, foutput_data2, foutput_label2, fnameseqtrain, output):
    """
    Creates test dataset by splitting the original dataset.

    Args:
        foutput_data1 (str): File path for the original dataset.
        foutput_label1 (str): File path for the labels of the original dataset.
        foutput_data2 (str): File path for the test dataset.
        foutput_label2 (str): File path for the labels of the test dataset.
        output (str): Output directory.

    Returns:
        Tuple of output file paths (str) for the original and test datasets and their labels.
    """
    foutput_data2 = output + '/model_data' + '/' + 'data_test' + '.csv'
    foutput_label2 = output + '/model_data' + '/' + 'data_test' + '_label.csv'
    foutput_seqname = output+'/model_data/data_test_nameseq.csv'

    X = pd.read_csv(foutput_data1, sep=',')
    y = pd.read_csv(foutput_label1, sep=',')
    foutput_data1, foutput_data2, foutput_label1, foutput_label2
    X_train, X_test, y_train, y_test = train_test_split(X,y, train_size=0.8, random_state=42, shuffle=True)

    nameseq = pd.read_csv(fnameseqtrain)
    nameseqtrain, nameseqtest = train_test_split(nameseq, train_size=0.8, random_state=42, shuffle=True)

    X_train = X_train.iloc[:len(X_train), :]
    y_train = y_train.iloc[:len(y_train), :]
    X_test =  X_test.iloc[:len(X_test), :]
    y_test =  y_test.iloc[:len(y_test), :]

    X_train.to_csv(foutput_data1, index=False)
    X_test.to_csv(foutput_data2, index=False)
    y_train.to_csv(foutput_label1, index=False)
    y_test.to_csv(foutput_label2, index=False)
    nameseqtest.to_csv(foutput_seqname, index=False)
    nameseqtrain.to_csv(fnameseqtrain, index=False)

    return foutput_data1, foutput_label1, foutput_data2, foutput_label2,  foutput_seqname

def objective_rf(space):

    """Automated Feature Engineering - Objective Function - Bayesian Optimization"""

    index = list()

    descriptors_nucleot = {}
    nucleot_space =[0,4,16,64,320,10,2,5,19,19,5,0,0,44,19,65,54,88,76,24]
    add = 0
    for i in features_nucleot:
        descriptors_nucleot['n'+str(i)] = list(range(add, nucleot_space[i]+add))
        add += nucleot_space[i]

    descriptors_amino = {}
    amino_space =[0,5,5,5,5,78,400,20,400,0,0,19]
    for i in features_amino:
        descriptors_amino['a'+str(i)] = list(range(add, amino_space[i]+add))
        add += amino_space[i]

    dic_type = {'dna': descriptors_nucleot, 'rna': descriptors_nucleot, 'protein': descriptors_amino}
    descriptors = {**dic_type[type1], **dic_type[type2]}

    for descriptor, ind in descriptors.items():
        if int(space[descriptor]) == 1:
            index = index + ind

    x = df_x.iloc[:, index]

    if int(space['Classifier']) == 0:
        if len(fasta_label_train) > 2:
            model = AdaBoostClassifier(random_state=63) #MUDAR
        else:
            model = CatBoostClassifier(n_estimators=500,
                                       thread_count=n_cpu, nan_mode='Max',
                                       logging_level='Silent', random_state=63)
    elif int(space['Classifier']) == 1:
        model = RandomForestClassifier(n_estimators=500, n_jobs=n_cpu, random_state=63)
    else:
        model = lgb.LGBMClassifier(n_estimators=500, n_jobs=n_cpu, random_state=63)

    if len(fasta_label_train) > 2:
        score = make_scorer(f1_score, average='weighted')
    else:
        score = make_scorer(balanced_accuracy_score)

    kfold = StratifiedKFold(n_splits=10, shuffle=True)
    metric = cross_val_score(model,
                             x,
                             labels_y,
                             cv=kfold,
                             scoring=score,
                             n_jobs=n_cpu).mean()

    return {'loss': -metric, 'status': STATUS_OK}

def feature_engineering(estimations, train, train_labels, test, foutput):

    """Automated Feature Engineering - Bayesian Optimization"""

    global df_x, labels_y

    print('Automated Feature Engineering - Bayesian Optimization')

    df_x = pd.read_csv(train)
    labels_y = pd.read_csv(train_labels)

    if test != '':
        df_test = pd.read_csv(test)

    path_bio = foutput + '/best_descriptors'
    if not os.path.exists(path_bio):
        os.mkdir(path_bio)

    param_nucleot = {}
    for i in features_nucleot:
        param_nucleot['n'+str(i)] = [0, 1]

    param_amino = {}
    for i in features_amino:
        param_amino['a'+str(i)] = [0, 1]

    param_classif = {'Classifier': [0, 1, 2]}

    param_type = {'dna': param_nucleot, 'rna': param_nucleot, 'protein': param_amino}
    param = {**param_type[type1], **param_type[type2], **param_classif}

    space_nucleot = {}
    for i in features_nucleot:
        space_nucleot['n'+str(i)] = hp.choice('n'+str(i), [0, 1])

    space_amino = {}
    for i in features_amino:
        space_amino['a'+str(i)] = hp.choice('a'+str(i), [0, 1])

    space_classif = {'Classifier': hp.choice('Classifier', [0, 1, 2])}
    space_type = {'dna': space_nucleot, 'rna': space_nucleot, 'protein': space_amino}
    space = {**space_type[type1], **space_type[type2], **space_classif}

    trials = Trials()
    best_tuning = fmin(fn=objective_rf,
                space=space,
                algo=tpe.suggest,
                max_evals=estimations,
                trials=trials)

    index = list()
    descriptors_nucleot = {}
    nucleot_space =[0,4,16,64,320,10,2,5,19,19,5,0,0,44,19,65,54,88,76,24]
    add = 0
    for i in features_nucleot:
        descriptors_nucleot['n'+str(i)] = list(range(add, nucleot_space[i]+add))
        add += nucleot_space[i]

    descriptors_amino = {}
    amino_space =[0,5,5,5,5,78,400,20,400,0,0,19]
    for i in features_amino:
        descriptors_amino['a'+str(i)] = list(range(add, amino_space[i]+add))
        add += amino_space[i]

    dic_type = {'dna': descriptors_nucleot, 'rna': descriptors_nucleot, 'protein': descriptors_amino}
    descriptors = {**dic_type[type1], **dic_type[type2]}

    for descriptor, ind in descriptors.items():
        result = param[descriptor][best_tuning[descriptor]]
        if result == 1:
            index = index + ind

    classifier = param['Classifier'][best_tuning['Classifier']]

    btrain = df_x.iloc[:, index]
    path_btrain = path_bio + '/best_train.csv'
    btrain.to_csv(path_btrain, index=False, header=True)

    if test != '':
        btest = df_test.iloc[:, index]
        path_btest = path_bio + '/best_test.csv'
        btest.to_csv(path_btest, index=False, header=True)
    else:
        btest, path_btest = '', ''

    return classifier, path_btrain, path_btest, btrain, btest

#################################################################################################################################
#################################################################################################################################
parser = argparse.ArgumentParser()
parser.add_argument('-input1_fasta_train', '--input1_fasta_train',  nargs='+', help='fasta format file, e.g., fasta/ncRNA.fasta')
parser.add_argument('-input1_fasta_test', '--input1_fasta_test',  nargs='+', help='fasta format file, e.g., fasta/ncRNA.fasta')
parser.add_argument('-label_1', '--label_1', help='problem labels, e.g., LncRNA')
parser.add_argument('-sequence_type1', '--sequence_type1', help='sequence type problem, e.g., RNA')

parser.add_argument('-input2_fasta_train', '--input2_fasta_train', nargs='+', help='fasta format file, e.g., fasta/protein.fasta')
parser.add_argument('-input2_fasta_test', '--input2_fasta_test', nargs='+', help='fasta format file, e.g., fasta/protein.fasta')
parser.add_argument('-label_2', '--label_2', help='problem labels, e.g., Regulatory_proteins')
parser.add_argument('-sequence_type2', '--sequence_type2', help='sequence type problem, e.g., Protein')

parser.add_argument('-interaction_table', '--interaction_table', help='txt format file, e.g., interactions/table.txt')
parser.add_argument('-output', '--output', help='resutls directory, e.g., result/')

parser.add_argument('-n_cpu', '--n_cpu', default=1, help='number of cpus - default = 1')
parser.add_argument('-estimations', '--estimations', default=2, help='number of estimations - BioAutoML - default = 10')

################################################## inputs
args = parser.parse_args()
input1_fasta_train = args.input1_fasta_train
input1_fasta_test = args.input1_fasta_test
label_1 = args.label_1
type1 = args.sequence_type1.lower()

input2_fasta_train = args.input2_fasta_train
input2_fasta_test = args.input2_fasta_test
label_2 = args.label_2
type2 = args.sequence_type2.lower()

interaction_table = args.interaction_table
foutput = args.output

estimations = int(args.estimations)
n_cpu = int(args.n_cpu)

fasta_label_train = [['0'],['1']]

################################################# check and make paths
check_path(input1_fasta_train,'input1_fasta_train')
check_path(input2_fasta_train,'input2_fasta_train')
input1_fasta_train = make_path(input1_fasta_train)
input2_fasta_train = make_path(input2_fasta_train)

if None != input1_fasta_test and None != input2_fasta_test:
    check_path(input1_fasta_test,'input1_fasta_test')
    check_path(input2_fasta_test,'input2_fasta_test')
    input1_fasta_test = make_path(input1_fasta_test)
    input2_fasta_test = make_path(input2_fasta_test)
else:
    print('Any test input')

check_path([interaction_table],'interaction_table')
table = pd.read_csv(interaction_table, sep=',')

################################################# general extraction
input1 = extraction(input1_fasta_train, label_1, type1, foutput, ftest=input1_fasta_test)
input1.delete_directory()
input1.check()
train1, test1 = input1.extrac_features()

input2 = extraction(input2_fasta_train, label_2, type2, foutput, ftest=input2_fasta_test)
input2.check()
train2, test2 = input2.extrac_features()

################################################# concatenation
foutput_data1, foutput_label1, fnameseqtrain = make_dataset(train1,train2,label_1,label_2, table, foutput, 'data_train')
if input1_fasta_test:
    foutput_data2, foutput_label2, fnameseqtest = make_dataset(test1,test2,label_1,label_2, table, foutput, 'data_test')
else:
    foutput_data2, foutput_label2, fnameseqtest = '','',''
    foutput_data1, foutput_label1, foutput_data2, foutput_label2, fnameseqtest = create_test(foutput_data1, foutput_label1, foutput_data2, foutput_label2, fnameseqtrain, foutput)

################################################# feature engineering and binary bioautoml
classifier, path_train, path_test, train_best, test_best = \
        feature_engineering(estimations, foutput_data1, foutput_label1, foutput_data2, foutput)

subprocess.run(['python', './source/BioAutoML-binary.py', '-train', path_train,
                     '-train_label', foutput_label1, '-test', path_test, '-test_label',
                     foutput_label2, '-test_nameseq', fnameseqtest,'-nf', 'True', '-imbalance',
                     'True','-classifier', str(classifier), '-n_cpu', str(n_cpu),
                     '-output', foutput])


import unittest
from restserver import app as flask_app

class MyTestCase(unittest.TestCase):

    def setUp(self):
        flask_app.testing = True
        self.app = flask_app.test_client()

    def test_home(self):
        result = self.app.get('/recentcves')
        self.assertEqual(result.status, '200 OK')

    #def test_get_leaks_base(self):
    #    result = self.app.get('/recentcves')
    #    resp = json.loads(result.data)
    #    checks = []
    #    for leak in resp:
    #        checks.append(all(True for key in config.resp_fields if key in leak.keys()))
    #    self.assertTrue(all(checks))

if __name__ == '__main__':
    unittest.main()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from mock import create_autospec, MagicMock, patch
from hypothesis.extra.django import TestCase
from hypothesis.extra.django.models import models as djangoModels
from django.contrib.auth.models import User
from django.db import transaction
from django.utils import timezone
from urllib import quote_plus
from hypothesis import given, assume, note, settings as hypothesisSettings
import math
from hypothesis.strategies import characters, just, sampled_from, emails, floats, composite, lists, dictionaries, one_of, text, none, fixed_dictionaries
import string
from geopy.location import Location
from geopy.point import Point
from . import models, exporting
from rest_framework.test import APITestCase
import pytest
import functools
from django.urls.base import reverse

geocodeDict = {
    'raw': text(),
    'street_number': text(),
    'route': text(),
    'locality': text(),
    'postal_code': text(alphabet=('-',) + tuple(string.digits)),
    'state': text(),
    'country': text(),
    'neighborhood': text(),
    'lat': floats(),
    'lng': floats()
}

geocodeTest = \
    one_of(
        dictionaries(text(), one_of(text(), none())),
        fixed_dictionaries(geocodeDict)
    )

@pytest.fixture
def person():
    return models.Person.objects.get_or_create(name='',
            email='test@example.com')[0]

@composite
def peopleArgs(draw):
    return draw(one_of(
        fixed_dictionaries(dict(
            name=one_of(just(None), text()),
            email=emails().filter(lambda x: len(x) < 100 and '/' not in x),
            address=one_of(just(None), text()),
        )),
        fixed_dictionaries(dict(
            name=one_of(just(None), text()),
            email=emails().filter(lambda x: len(x) < 100 and '/' not in x),
        )),
        fixed_dictionaries(dict(
            name=one_of(just(None), text()),
            email=emails().filter(lambda x: len(x) < 100 and '/' not in x),
        )),
        fixed_dictionaries(dict(
            email=emails().filter(lambda x: len(x) < 100 and '/' not in x),
        ))))

@composite
def people(draw):
    return models.Person.objects.create(
        name=draw(nonblanks()),
        email=draw(emails().filter(lambda x: len(x) < 100 and '/' not in x)),
        address=None,
    )

@composite
def nonblanks(draw):
    return draw(text(min_size=1, alphabet=characters(whitelist_categories=('Lu', 'Ll'))))

@composite
def latlngs(draw):
    return {
        'lat': draw(floats(min_value=-90, max_value=90)),
        'lng': draw(floats(min_value=-180, max_value=180))
    }

@composite
def googleAddressTypes(draw):
    return draw(lists(sampled_from((
        'locality', 'street_number', 'route', 'postal_code', 'state',
        'administrative_area_level_1',
        'administrative_area_level_2',
        'administrative_area_level_3',
        'administrative_area_level_4',
        'administrative_area_level_5',
        'neighborhood', 'country', 'intersection', 'political',
        'colloquial_area', 'ward', 'sublocality', 'premise', 'subpremise',
        'natural_feature', 'airport', 'park', 'point_of_interest', 'floor',
        'establishment', 'parking', 'post_box', 'postal_town', 'room',
        'bus_station', 'train_station', 'transit_station'
    )), min_size=1, unique=True))

@composite
def googleAddressComponents(draw):
    return {
        'types': draw(googleAddressTypes()),
        'long_name': draw(text()),
        'short_name': draw(text())
    }

@composite
def googleResponses(draw):
    return {
        'formatted_address': draw(text()),
        'place_id': draw(text()),
        'geometry': {
            'location': draw(latlngs()),
            'viewport': {
                'northeast': draw(latlngs()),
                'southwest': draw(latlngs())
            },
            'bounds': {
                'northeast': draw(latlngs()),
                'southwest': draw(latlngs())
            }
        },
        'types': draw(googleAddressTypes()),
        'address_components': draw(lists(googleAddressComponents(), min_size=1))
    }

@composite
def locations(draw):
    resp = draw(googleResponses())
    asPoint = Point(resp['geometry']['location']['lat'], resp['geometry']['location']['lng'])
    return Location(resp['formatted_address'], asPoint, resp)

def assertValidResponse(resp, status=200):
    __tracebackhide__ = True
    assert resp.status_code == status
    return resp

@pytest.mark.django_db
def testPermissions(api_client, person):
    """Ensure that we do not have access to PII by default"""
    assertValidResponse(api_client.get('/api/people/'), 403)
    assertValidResponse(api_client.get('/api/people/'+quote_plus(person.email)+'/'), 403)

@pytest.mark.skip_auth
@pytest.mark.django_db
@given(emails().filter(lambda e: '/' not in e))
@hypothesisSettings(deadline=None)
def testValidEmailURLs(api_client, email):
    """Test that all permutations of an email address generate a proper url"""
    url = reverse('person-detail', kwargs={'email': email})
    assertValidResponse(api_client.get(url), 404)

@pytest.mark.skip(reason="Buggy under pytest")
@pytest.mark.skip_auth
@pytest.mark.django_db
@given(person=peopleArgs(), newTag=nonblanks())
def testCreatePersonWithTag(api_client, settings, person, newTag):
    note('Email %s'%person['email'])
    person['tags'] = [newTag]
    note("Submit %r"%(person,))
    resp = assertValidResponse(api_client.post('/api/people/',
            person, format='json'), 201).data
    note('Result %r'%(resp,))
    personName = person.get('name', None)
    if personName is not None:
        personName = personName.strip()
    else:
        personName = ''
    assert resp['tags'] == person['tags']
    assert resp['email'] == person['email']
    assert resp['name'] == personName

    person['tags'].append(newTag+'-added')
    resp = assertValidResponse(api_client.put('/api/people/'+person['email']+'/',
            person, format='json'), 200).data
    assert resp['tags'] == person['tags']

@pytest.mark.django_db
def testExportPersonToMailchimp(person, settings):
    settings.MAILCHIMP_SECRET_KEY = '0' * 32
    settings.MAILCHIMP_LIST_ID = 'list-id'
    with patch('mailchimp3.entities.listmembers.ListMembers.create_or_update') as patched:
        exporter = exporting.MailchimpExporter()
        assert len(exporter) == 1
        page = iter(exporter).next()
        assert page.dict[0]['email'] == person.email
        exporter.export_page(page)
        patched.assert_called_with('list-id', '55502f40dc8b7c769880b10874abc9d0',
                dict(email_address=person.email, status_if_new='subscribed'))


from os import fsdecode
from xml.dom.minidom import Document
from searchindex.buildindex import index
from searchindex.preprocessing import preprocess_text
import re
from collections import defaultdict
from math import log10
from searchengine.models import Document
def run_ranked_search(query):
    results = defaultdict(float)

    query_preproc = preprocess_text(query)

    for term in query_preproc:
        if term in index:
            for doc_id in index[term]:

                tf = len(index[term][doc_id])
                df = index[term].doc_freq
                if (tf != 0) and (df != 0):
                    w = (1 + log10(tf)) * log10(index.doc_count / df)

                results[doc_id] += w
    return sorted(results, key=results.get, reverse=True)[:50]

def listToString(s):

    # initialize an empty string
    str1 = ""

    # traverse in the string
    for ele in s:
        str1 += ele

    # return string
    return str1

def run_query(query):

    match = "#(\d+)\((\w+), (\w+)\)"
    frstprt = []
    scndprt = []
    spl = query.split()

    if ' AND NOT ' in query:
        frstprt, scndprt = query.split('AND NOT')
        frstprt = preprocess_text(frstprt)[0]
        scndprt = preprocess_text(scndprt)[0]
        frstprt = index[frstprt]
        scndprt = index[scndprt]
        result = [i for i in frstprt if i not in scndprt]
        return result

    elif ' AND ' in query:
        frstprt, scndprt = query.split('AND')
        frstprt = preprocess_text(frstprt)[0]
        scndprt = preprocess_text(scndprt)[0]
        frstprt = index[frstprt]
        scndprt = index[scndprt]
        print("LOG, AND", frstprt, scndprt)
        result = [i for i in frstprt if i in scndprt]
        return result

    elif " OR NOT " in query:
        word1, word2 = query.split("OR NOT")
        word1 = preprocess_text(word1)[0]
        word2 = preprocess_text(word2)[0]

        results1 = index[word1]
        results2 = index[word2]

        results1_ids = []
        for i in results1:
            results1_ids.append(i)

        notresult2 = []
        # for i in range(1, index.doc_count + 1):
        for i in Document.objects.values_list('doc_id')[:50]:
            # The doc ids in the index are strings
            # but range gives us integers
            i = str(i[0])
            if i not in results2:
                notresult2.append(i)

        results1_ids = [i for i in notresult2 if i not in results1_ids]
        return results1_ids

    elif 'OR' in query:
        i = 0
        while i < len(spl):
            if spl[i] == "OR":
                i += 1
                break
            frstprt.append(spl[i])
            i += 1
        while i < len(spl):

            if spl[i] == "OR":
                i += 1
                break
            scndprt.append(spl[i])
            i += 1

        frstprt = listToString(frstprt)
        frstprt = preprocess_text(frstprt)[0]
        scndprt = listToString(scndprt)
        scndprt = preprocess_text(scndprt)[0]
        res1 = index[frstprt]
        res2 = index[scndprt]
        res1.update(res2)

        return res1

    elif query.startswith('"') and query.endswith('"'):
        word1,word2 = query.split(" ")

        word1 = preprocess_text(word1)[0]
        word2 = preprocess_text(word2)[0]

        results = set()
        res1 = index[word1]

        for id in res1:
            pos1 = index[word1][id]
            pos2 = index[word2][id]
            locResults = [i for i in pos1 if i + 1 in pos2]
            if locResults:
                results.add(id)
        return results

    elif re.match(match, query):

        m = re.match(match, query)
        num = int(m.group(1))
        word1 = m.group(2)
        word2 = m.group(3)
        word1 = preprocess_text(word1)[0]
        word2 = preprocess_text(word2)[0]

        result = set()
        res1 = index[word1]

        for id in res1:
            cur = 0
            while cur <= num:
                pos1 = index[word1][id]
                pos2 = index[word2][id]
                locResults = [i for i in pos1 if (i + cur) in pos2]
                if locResults:
                    result.add(id)
                cur += 1
            cur = 0
            while cur <= num:
                pos1 = index[word1][id]
                pos2 = index[word2][id]
                locResults = [i for i in pos1 if (i - cur) in pos2]
                if locResults:
                    result.add(id)
                cur += 1
        return result
    else:
        return run_ranked_search(query)


from rest_framework import serializers

from .models import Ayah, Sura

class SuraSerializer(serializers.ModelSerializer):

    class Meta:
        model = Sura
        fields = ['index', 'name']

class AyahSerializer(serializers.ModelSerializer):
    sura_index = serializers.IntegerField(source='sura.pk')
    sura_name = serializers.CharField(source='sura.name')
    ayah_number = serializers.IntegerField(source='number')

    class Meta:
        model = Ayah
        fields = ['sura_index', 'sura_name', 'ayah_number', 'text']


#!/usr/bin/env python
# -*- coding: utf-8 -*- #

DEFAULT_LANG = 'en'
# Blogroll
# LINKS =  (('Pelican', 'http://docs.notmyidea.org/alexis/pelican/'),
#           ('Python.org', 'http://python.org'),
#           ('Jinja2', 'http://jinja.pocoo.org'),
#           ('You can modify those links in your config file', '#'),)

DEFAULT_PAGINATION = 10

AUTHOR = 'Ken Cochrane'
DISQUS_SITENAME = 'kencochrane'
GITHUB_URL = 'https://github.com/kencochrane'
GITHUB_ACTIVITY_FEED = 'https://github.com/kencochrane.atom'
GOOGLE_ANALYTICS='UA-67696-11'
CLICKY_SITE_ID='66415850'
SITEURL = 'http://kencochrane.net'
SITENAME = 'KenCochrane.net'
SOCIAL = (('twitter', 'http://twitter.com/kencochrane'),
          ('github', 'https://github.com/kencochrane'),)
#TAG_FEED_ATOM = 'feeds/%s.atom.xml'
THEME='notmyidea' #THEME='bootstrap'
TWITTER_USERNAME = 'KenCochrane'
PLUGINS = ['pelican.plugins.gravatar',
           'pelican.plugins.github_activity',
           'pelican.plugins.related_posts',
           'pelican.plugins.html_rst_directive']
ARTICLE_URL = 'blog/{date:%Y}/{date:%m}/{slug}/'
ARTICLE_SAVE_AS = 'blog/{date:%Y}/{date:%m}/{slug}/index.html'
PAGE_URL = '{slug}/index.html'
PAGE_SAVE_AS = '{slug}/index.html'
DISPLAY_PAGES_ON_MENU = True
MARKUP = (('rst', 'html'))

TIMEZONE = 'America/New_York'
#DIRECT_TEMPLATES = (('index', 'tags', 'categories', 'archives'))
#CSS_FILE = "wide.css"


# lstm
from keras.models import Sequential
from keras.layers.convolutional import Conv3D
from keras.layers.convolutional_recurrent import ConvLSTM2D
from keras.layers.normalization import BatchNormalization
from numpy import concatenate
import numpy as np
import pandas
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from pandas import DataFrame, Series
from sklearn.preprocessing import MinMaxScaler
from keras import backend as K
data = np.loadtxt('D:/干旱/data/result8/qm_predict.txt') #month *[pts*lead*model]
row = 65
col = 37
pts = row*col
month = data.shape[0]
lead = 9
model_num = 8

index_nan = np.where(np.isnan(data))
data[index_nan] = 0

# normalize features
scaler = MinMaxScaler(feature_range=(0, 1))
data_scaled = scaler.fit_transform(data)

data = np.reshape(data_scaled, (month, pts, lead, model_num), order='F')

start_year = 1982
end_year = 2016
train_end_year = 2010
train_end = (train_end_year - start_year + 1)*12
# all = (end_year - start_year + 1)*12
year = (end_year - start_year + 1)

gpcc_china = np.loadtxt('D:/干旱/data/result8/gpcc_china.txt') #month*pts
index_nan = np.where(np.isnan(gpcc_china))
gpcc_china[index_nan] = 0
gpcc_scaled = scaler.fit_transform(gpcc_china)

lstm_predict = np.zeros((month, pts, lead), dtype=np.float)
for i in range(pts):
    for j in range(lead):
        obs = gpcc_scaled[:, i]
        model = data[:, i, j, :]
        model = np.reshape(model, (month, model_num), order='F')
        train_x = model[0:train_end, :]
        train_y = obs[0:train_end]
        test_x = model

        if sum(train_y) == 0:
            continue

        # reshape input to be 3D [samples, timesteps, features]
        train_x = train_x.reshape(( train_end, 1, model_num), order='F')
        train_y = train_y.reshape(( train_end, 1), order='F')
        test_x = test_x.reshape(( month, 1, model_num), order='F')

        model = Sequential()
        model.add(LSTM(10, input_shape=(train_x.shape[1], train_x.shape[2])))  # 50
        model.add(Dense(1))
        model.compile(loss='mse', optimizer='adam')
        # fit network
        history = model.fit(train_x, train_y, epochs=1000, \
                            batch_size = train_x.shape[0], verbose=2, shuffle=False, validation_split=0.1) #train_x.shape[0]
        y_pre = model.predict(test_x)
        lstm_predict[:, i, j] = y_pre[:, 0]
        print ("%d %d"%(i,j))
        # destroy the current TF graph and create a new one
        K.clear_session()
        cfg = K.tf.ConfigProto()
        cfg.gpu_options.allow_growth = True
        K.set_session(K.tf.Session(config=cfg))

# inverse transform
for i in range(lead):
    lstm_predict[:, :, i] = scaler.inverse_transform(lstm_predict[:, :, i])

lstm_predict = lstm_predict.reshape((month,pts*lead), order='F')
np.savetxt('D:/干旱/data/result8/lstm_predict.txt', lstm_predict)


from django.forms import ModelForm, DateInput
from walkie_talkie.models import WalkieTalkie

class WalkieTalkieForm(ModelForm):

    class Meta:
        model = WalkieTalkie
        fields = '__all__'
        labels = {'id_organization': 'Название организации'}

        widgets = {
            'date_of_purchase': DateInput(
                format='%d-%m-%Y',
                attrs={'type': 'datetime-local'
                       }),
            'date_of_received': DateInput(
                format='%d-%m-%Y',
                attrs={'type': 'datetime-local'
                       }),
        }


# Gambiarras
import sys
this_dir = sys.path[0]
import matplotlib
matplotlib.use('tkagg')

#Importação das bibliotecas de interesse:
import numpy as np
import math
import matplotlib.pyplot as plt
from scipy import signal
import os

def funcao_gera_figura(f,s,t,nome):
    #ATENÇÃO! NÃO HÁ NECESSIDADE DE ALTERAR ESTA FUNÇÃO.

    #Entradas:
    #f [numpy array] : função periódica que estamos interessados
    #em analisar
    #s [numpy array]: expansão em Série de Fourier da função f, pre-
    #viamente calculada
    #t [numpy array]: vetor de tempo
    #nome [string]: nome do arquivo a ser salvo.
    #Importação das bibliotecas de interesse:

    #Resolução da figura:
    dpi = 300
    #Criação da figura, com tamanho e resolução especificados
    fig, ax = plt.subplots()
    fig.set_size_inches(5, 3)
    fig.set_dpi(dpi)

    #Gera os gráficos da função de interesse e da sua aproximação
    #por Série de Fourier:
    plt.plot(t,f,'-b')
    plt.plot(t,s,'-r')
    #Definição dos eixos
    ax.axis([min(t), max(t), -0.5, 1.5])
    #Gera a grade para facilitar a leitura do gráfico
    ax.grid()
    #Nomes de cada eixo:
    plt.xlabel(r'$t[s]$')
    #Salva a imagem em formato pdf.
    plt.savefig(f'{nome}.jpg', format='jpg', dpi=dpi)

def coeficientes_fourier(numero_de_harmonicas):
    #Entrada:
    #numero_de_harmonicas [int]: numero de harmônicas a serem consi-
    #deradas
    #Saída:
    #a_n [numpy array]: vetor de coeficientes a_n (n entre 1 e
    #numero_de_harmonicas) da expansão em Série de Fourier

    an = np.zeros(numero_de_harmonicas+1)
    an[0] = 0.5

    for n in range(1, numero_de_harmonicas+1):
        an[n] = 0.0 if n % 2 == 0 else (2/(n*np.pi)) * (-1)**((n-1)/2)
    return an

# Script principal:
t = np.linspace(-2, 2, 10000)

T0 = 2

f0 = 1/T0

f = 0.5*(signal.square(2*np.pi*f0*(t+T0/4))+np.ones_like(t))

a0 = 0.5

vetor_harmonicas = [1,3,5,7,9,11,13,25,50]

for N in vetor_harmonicas:
    an = coeficientes_fourier(N)
    s = a0 * np.ones_like(t)
    for n in range(1,N+1):
        s += an[n] * np.cos(n*2*np.pi*f0*t)

    nome_figura = os.path.join(this_dir, f"quadrada_{N}_harmonicas")
    funcao_gera_figura(f,s,t,nome_figura)


import unittest
from app import test_table

class TestTasks(unittest.TestCase):

    def test_tasks(self):
        expected_results = [{'login': 'admin','mdp':'admin'}, {'login': 'user', 'mdp': 'user'}]

if __name__ == 'main':
    unittest.main()


import importlib
from Constants import Constants
from Logger import *
from CustomException import CustomException
#=============================================================================#
# File: KeywordFactory.py
#
#  Copyright (c) 2015, Vinoth Kumar Ravichandran, Joe DiMauro
#  All rights reserved.
#
# Description: Classes and functions for KeywordFactory
#    These classes and functions are application dependent, performs activities based
#    on the user input from the instruction sheet
#
#    The keyword passed on from instruction sheet is analyzed and appropriate
#    class and its methods would be dynamically called based on instruction
#
#--
# Depends On:
#    Instruction Sheet (calls in the instruction sheet)
#    KeywordDriver
#    Constants
#
# Classes Added or Extended:
#    Constants (Constants.py)
#
#++
#=============================================================================#

#=============================================================================#
# Class: KeywordFactory
#=============================================================================#
#
# Description: Functions for KeywordFactory
#    These functions and methods are application and platform independent
#
# Table of Contents
#
#  Please manually update this TOC with the alphabetically sorted names of the items in this module,
#  and continue to add new methods alphabetically into their proper location within the module.
#
#  Key:   () = No parameters,  (...) = parameters required
#
# Methods:
#   __init__(oInstruction)
#   Execute()
#
# Pre-requisites:
# ++
#=============================================================================#

class KeywordFactory:
  #=============================================================================#
  # Method: initialize()
  # Description: initializes the keywordfactory with one row of instruction
  # Returns: N/A
  # Usage Examples: oKeywordFactory = KeywordFactory(oInstruction)
  # where oInstruction should of type Instruction
  #=============================================================================#
  def __init__(self,oInstruction):
    # member variables
    self.instruction = oInstruction

  #=============================================================================#
  # Method: Execute()
  # Description: executes the specific instruction after analysis
  # Returns: N/A
  # Usage Examples: oKeywordFactory.Execute()
  #=============================================================================
  def Execute(self):
    # capturing the length of the keywords
    arKeyword = self.instruction.action.split(Constants.DELIMITER_STOP)
    oObject = ""
    iTotalCount = len(arKeyword)

    # currently only keywords of 2 levels are supported. analyzing the same
    if iTotalCount > 3:
        sTemp = "Keyword level cannot be greater than 3 <{%s}>" %(self.instruction.action)
        raise IndexError(sTemp)

    # Raises an error for unknown class or known class with unknown methods
    sModuleName = arKeyword[0]
    sClassName = arKeyword[1]
    sMethodName = arKeyword[2]
    oModule = __import__('scripts.'+ sModuleName, fromlist=['*'])
    oClass = getattr(oModule,sClassName)
    oObject = oClass(self.instruction)
    sOutput = getattr(oObject,sMethodName)()
    Logger.note.debug("Output for execution of keyword <%s.%s> is %s" %(sClassName,sMethodName,sOutput))
    #return sOutput

#=============================================================================#
# End Of Class: KeywordFactory
#=============================================================================#


# Definition for singly-linked list.
class ListNode(object):
    def __init__(self, val=0, next=None):
        self.val = val
        self.next = next

class Solution(object):
    def mergeKLists(self, lists):
        """
        :type lists: List[ListNode]
        :rtype: ListNode
        """
        ret = ListNode()
        temp = ret
        while lists != []:
            val = 10000
            i = 0
            idx = 0
            while i < len(lists):
                print("debugging))i:{}, len:{}".format(i, len(lists), lists))
                if not lists[i]:
                    print("del list"+str(i))
                    del lists[i]
                    continue
                # print("debugging:{}".format(type(lists[i])))
                if val > lists[i].val:
                    val = lists[i].val
                    idx = i
                i += 1
            if val < 10000:
                temp.next = ListNode(val)
                temp = temp.next
                lists[idx] = lists[idx].next

        return ret.next

c = Solution()
# l1 = ListNode(1)
# l1.next = ListNode(4)
# l1.next.next = ListNode(5)
# l2 = ListNode(1)
# l2.next = ListNode(3)
# l2.next.next = ListNode(4)
# l3 = ListNode(2)
# l3.next = ListNode(6)

ret = c.mergeKLists([ListNode(2), [], ListNode(-1)])
while ret:
    print(ret.val)
    ret = ret.next


import math

with open('../inputs/day_12.txt', 'r') as f:
    insns = [(line.strip()[0], int(line.strip()[1:])) for line in f]
pos = [0, 0, 0]  # degree starts from +x axis

"""
What is the Manhattan distance between that location and the ship's starting position?
"""

def move(instruction):
    direc, val = instruction
    if direc == 'N':
        pos[1] += val
    elif direc == 'S':
        pos[1] -= val
    elif direc == 'E':
        pos[0] += val
    elif direc == 'W':
        pos[0] -= val
    elif direc == 'L':
        pos[2] = (pos[2] + val) % 360
    elif direc == 'R':
        pos[2] = (pos[2] - val) % 360
    elif direc == 'F':
        ang = math.radians(pos[2])
        pos[0] += val * math.cos(ang)
        pos[1] += val * math.sin(ang)

for insn in insns:
    move(insn)
print(abs(pos[0]) + abs(pos[1]))

"""
The instructions actually indicate how to move a waypoint which is relative to the ship's position.
What is the Manhattan distance between that location and the ship's starting position?
"""
ship = [0, 0]
wp = [10, 1]

def move_wp(instruction):
    direc, val = instruction
    if direc == 'N':
        wp[1] += val
    elif direc == 'S':
        wp[1] -= val
    elif direc == 'E':
        wp[0] += val
    elif direc == 'W':
        wp[0] -= val
    elif direc == 'L' or direc == 'R':
        if direc == 'R':
            val = 360 - val
        ang = math.radians(val)
        np = [math.cos(ang) * wp[0] - math.sin(ang) * wp[1], math.sin(ang) * wp[0] + math.cos(ang) * wp[1]]
        wp[0], wp[1] = np[0], np[1]
    elif direc == 'F':
        for i in range(val):
            ship[0] += wp[0]
            ship[1] += wp[1]

for insn in insns:
    move_wp(insn)
print(abs(ship[0]) + abs(ship[1]))


## 资料串接2
## 以大乐透为例

import pandas as pd

url = "https://www.taiwanlottery.com.tw/lotto/lotto649/history.aspx"

df=pd.read_html(url)
# 因读取的资料是表格 所以是用 read_html

# for d in df[2:]:
#     # 看全部抓取的资料
#     # 资料从第2列开始
#     print(d.iloc[1,0])
#     print("-------")

colName = ['期别', '开奖日', '兑奖截止', '销售金额', '奖金总额', '奖号1', '奖号2', '奖号3', '奖号4', '奖号5', '奖号6', '特别号' ]
lotto = pd.DataFrame(columns=colName)
# 只单纯建立一个只有字段名称，没有资料的DataFrame

# 点df看整个表格资料，抓取大乐透的特定数值
for d in df[2:12]:
    data=[]
    data.append(d.iloc[1,0]) #期别
    data.append(d.iloc[1,1]) #开奖日
    data.append(d.iloc[1,3]) #兑奖截止
    data.append(d.iloc[1,5]) #销售金额
    data.append(d.iloc[1,7]) #奖金总额
    for i in range(2,9): #六个中奖号与一个特别号
        data.append(d.iloc[4,i])
# 中奖号在表格中的第四列，所以列固定在4，而从小到大的号码，分别在第二栏至第八栏的位置，所以用for循环抓取
    print(data)
    dftemp = pd.DataFrame([data], columns=colName)
    # data是一维的串列，所以多加[]使它变成DataFrame的二维型态
    # dftemp是让资料与字段名称写入的物件。可以点左边的dftemp查看
    lotto = pd.concat([lotto, dftemp])
    # 将空的字段名称lotto与资料dftemp串接
    print("-----")


from typing import List

class Solution:
    def minSubArrayLen(self, s: int, nums: List[int]) -> int:
        nums_sum = 0
        min_sub_len = 0
        left_index = 0
        sum_dict = {}

        for index, element in enumerate(nums):
            nums_sum += element
            sum_dict[index] = nums_sum
            if not min_sub_len and nums_sum >= s:
                min_sub_len = index + 1

        if not min_sub_len:
            return 0

        for index, element in enumerate(nums):
            if sum_dict[index] - sum_dict[left_index] >= s:
                min_sub_len = min(min_sub_len, index - left_index)
                while sum_dict[index] - sum_dict[left_index + 1] >= s:
                    left_index += 1
                    min_sub_len = min(min_sub_len, index - left_index)

        return min_sub_len

class Solution2:
    def minSubArrayLen(self, s: int, nums: List[int]) -> int:
        sum_nums = 0
        left_index = 0
        sub_len = len(nums) + 1

        for index, element in enumerate(nums):
            sum_nums += element
            while sum_nums >= s:
                sub_len = min(sub_len, index + 1 - left_index)
                sum_nums -= nums[left_index]
                left_index += 1
        return sub_len if sub_len <= len(nums) else 0


"""
35 - Sejam a e b os catetos de um triângulo, onde a hipotenusa é obtida pela equação:
hipotenusa = (raiz quadrada de a² + b²). Faça um programa que receba os valores de a e b
e calcule o valor da hipotenusa através da equação. Imprima o resultado dessa
operação.

"""

valor_a = float(input("Digite o valor do cateto a: "))
valor_b = float(input("Digite o valor do cateto b: "))

rq_a = valor_a ** 2
rq_b = valor_b ** 2

hipotenusa = (rq_a + rq_b) ** (1/2)

print(f"A raiz quadrada de {valor_a}² + {valor_b}² tem a seguinte hipotenusa: {hipotenusa:.2f}.")

Exercicios Variaveis e tipos de dados/exercicio 35.py


"""
Renderer object
"""

import pygame
import scripts.utilities as utilities

class Renderer:
    def __init__(self, app) -> None:

        """Initialize Renderer object.

        Keyword arguments:
        app - Main application class
        """

        self.app = app
        self.config = app.config

    def render(self) -> None:

        """Renders everything"""

        self.surface = self.app.window.app_window
        size = self.app.vrt_space.size
        for i in range(30):
            for j in range(30):
                if (i + j) % 2 == 0:
                    pygame.draw.rect(self.surface, (30, 30, 30), (i * size, j * size, size, size))

        self.app.vrt_space.food.draw(self.surface)
        for worm in self.app.vrt_space.worms[:self.app.game_instance.active_players]:
            worm.draw(self.surface)

        # Show debug values
        utilities.debug(self.surface, self.app.window.font, (750, 10), self.app.window.fps)


# When we apply ** to a dictionary, then it expands the contents in dictionary as a collection of key value pairs.

dict1 = {'Ritika': 5, 'Sam': 7, 'John' : 10 }
dict2 = {'Aadi': 8,'Sam': 20,'Mark' : 11 }
dict3 = {'Mark': 18,'Rose': 22,'Wong' : 22 }

# Merge contents of dict3, dict2 and dict1 to dict4
dict4 = {**dict1, **dict2, **dict3}

print('Dictionary 4 :')
print(dict4)

Python3/55_merge two or more Dictionaries using kwargs.py