# python 20200730_PPT_LinkArgumentCrawler_NBATemplate.py -L https://www.ptt.cc/bbs/sex/index.html -P 5

import bs4
import urllib.request as req
import argparse

ap = argparse.ArgumentParser()
ap.add_argument("-L", "--Link", required=True, type=str,
	help="Input PPT's Internet Link")
ap.add_argument("-P", "--Page", required=True, type=int,
	help="Input PPT's last new's web-page's number")

args = vars(ap.parse_args())

def getData(url):

    request = req.Request(url, headers={
        "cookie":"over18=1",
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36"
    })
    with req.urlopen(request) as response: # (url) 轉-> (request)
        data = response.read().decode("utf-8")

    root = bs4.BeautifulSoup(data, "html.parser")
    print("標題名稱 : ",root.title.string)
    titles_all = root.find_all("div",class_="title")

    for i in titles_all :
        if i.a != None:
            print(i.a.string)

    # 抓取下一頁的連結 :
    nextlink = root.find("a", string ="‹ 上頁") # 找到內文是 < 上頁的 a 標籤
    #print(nextlink)
    #print(nextlink["href"]) # 抓出屬性href(href 是我們要找的資料: )
    return nextlink["href"]

# (!!!!!!!!!!!!!)主程序 : 抓取多個頁面的標題 :
pageURL = args["Link"]
print(pageURL)
    # 多抓幾頁資料 : 寫迴圈抓資料 ~
count = 0
while count < args["Page"] :
    pageURL = "https://www.ptt.cc" + getData(pageURL) # getData come from "bbs/...", so we add...連結中前面的連結資料(http....)
    count += 1
    #print(pageURL)


# coding: utf-8

from CRNN import CRNN
import keras
import os
import keras.backend as K
from keras.callbacks import TensorBoard
from keras.utils.vis_utils import plot_model

cp_folder = 'checkpoints'
cp_file = 'weights.{epoch:02d}-{val_loss:.2f}.hdf5'

if not os.path.isdir(cp_folder):
    os.mkdir(cp_folder)

crnn = CRNN(120)
batch_size = 10

tensorboard = TensorBoard(log_dir='./logs', histogram_freq=0, batch_size=batch_size, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None)

cb = [keras.callbacks.ModelCheckpoint(
    os.path.join(cp_folder, cp_file),
    save_best_only=True,
    save_weights_only=True), tensorboard]

plot_model(crnn.model,to_file='./new.png',show_shapes=True)
crnn.train(epochs=80,batch_size=batch_size,pic_path='./samples/',vali_path='./samples/',cb=cb)

K.clear_session()


import imageio
imageio.plugins.ffmpeg.download()

from moviepy.video.io.ffmpeg_tools import ffmpeg_extract_subclip
import easygui, sys, os, subprocess

if not easygui.ccbox("Select the full video."):
    sys.exit(0)
videoPath = easygui.fileopenbox(filetypes=['*.mp4'])

if not easygui.ccbox('Determine the time in the full video when the "Start Experiment" button was pressed.'):
    sys.exit(0)
if sys.platform == "win32":
    os.startfile(videoPath)
else:
    opener ="open" if sys.platform == "darwin" else "xdg-open"
    subprocess.call([opener, videoPath])

offset = easygui.integerbox('Type in the time in the full video when the "Start Experiment" button was pressed.')
if not offset:
    sys.exit(0)

if not easygui.ccbox("Select the clip directory."):
    sys.exit(0)
clipPath = easygui.diropenbox()

if not easygui.ccbox("Select the log file."):
    sys.exit(0)
logPath = easygui.fileopenbox(filetypes=['*.txt'])

logFile = open(logPath, 'r')
times = [int(line.split(' ')[0]) for line in logFile]
for pos in range(len(times) - 1):
    ffmpeg_extract_subclip(videoPath, offset + times[pos], offset + times[pos + 1],
        targetname = clipPath + "/Clip" + ("0" if pos + 1 < 10 else "") + str(pos + 1) + ".mp4")


import time
from helpers import clear_console, validate_int_input
from player import Player
from round import Round

class Match:
    def __init__(self):
        self.players: list[Player] = []
        self.score_to_win = 0
        self.round_num = 1
        self.num_players = 2
        self.match_winner = None
        self.pause_time = 3

    def run(self):
        self.setup_match()
        while not self.match_winner:
            self.display_round()
            Round(self.players.copy()).play()
            self.display_player_scores()
            self.has_winner()
            self.round_num += 1
        input('\nPress return to continue...')

    def has_winner(self):
        for player in self.players:
            if player.rounds_won == self.score_to_win:
                self.match_winner = player
                self.declare_winner()

    def setup_match(self):
        clear_console()
        print("Match setup: ")
        self.choose_score_to_win()
        self.set_player_names()

    def set_player_names(self):
        i = 1
        for player in self.players:
            player.validate_and_set_name(self.players, i)
            i += 1

    def declare_winner(self):
        print(f"\n******{self.match_winner.name.upper()} WINS!!******")

    def init_players(self, player_type):
        for i in range(self.num_players):
            self.players.append(player_type())

    def choose_score_to_win(self):
        prompt = "How many victories necessary to win the game? "
        while self.score_to_win <1:
            self.score_to_win = validate_int_input(prompt)
            prompt = "Score to win must be greater than 0."

    def display_round(self):
        clear_console()
        print(f"\nRound {self.round_num}")
        time.sleep(1)

    def display_player_scores(self):
        print("\nScoreboard:\n")
        for player in self.players:
            print(f"{player.name}: {player.rounds_won}")
        time.sleep(self.pause_time)


import curses
import asyncio

from rocket_pictures import rocket_frame1, rocket_frame2

SPACE_KEY_CODE = 32
# Here we're init wasd buttons
LEFT_KEY_CODE = 97
RIGHT_KEY_CODE = 100
UP_KEY_CODE = 119
DOWN_KEY_CODE = 115

def read_controls(canvas):
    """Read keys pressed and returns tuple with controls state."""
    rows_direction = columns_direction = 0
    space_pressed = False

    pressed_key_code = canvas.getch()

    if pressed_key_code == UP_KEY_CODE:
        rows_direction = -1
    elif pressed_key_code == DOWN_KEY_CODE:
        rows_direction = 1
    elif pressed_key_code == RIGHT_KEY_CODE:
        columns_direction = 1
    elif pressed_key_code == LEFT_KEY_CODE:
        columns_direction = -1
    elif pressed_key_code == SPACE_KEY_CODE:
        space_pressed = True
    else:
        return -200, 0, False

    return rows_direction, columns_direction, space_pressed

def draw_frame(canvas, start_row, start_column, text, negative=False):
    rows_number, columns_number = canvas.getmaxyx()

    for row, line in enumerate(text.splitlines(), round(start_row)):
        if row < 0:
            continue

        if row >= rows_number:
            break

        for column, symbol in enumerate(line, round(start_column)):
            if column < 0:
                continue

            if column >= columns_number:
                break

            if symbol == ' ':
                continue

            # Check that current position it is not
            # in a lower right corner of the window
            # Curses will raise exception in that case. Don`t ask why…
            # https://docs.python.org/3/library/curses.html#curses.window.addch
            if row == rows_number - 1 and column == columns_number - 1:
                continue

            symbol = symbol if not negative else ' '
            canvas.addch(row, column, symbol)

def get_frame_size(text):
    lines = text.splitlines()
    rows = len(lines)
    columns = max([len(line) for line in lines])
    return rows, columns

async def fire(
        canvas, start_row,
        start_column, rows_speed=-1,
        columns_speed=0):
    row, column = start_row, start_column

    canvas.addstr(round(row), round(column), '*')
    await asyncio.sleep(1)

    canvas.addstr(round(row), round(column), 'O')
    await asyncio.sleep(1)
    canvas.addstr(round(row), round(column), ' ')

    row += rows_speed
    column += columns_speed

    symbol = '-' if columns_speed else '|'

    rows, columns = canvas.getmaxyx()
    max_row, max_column = rows - 1, columns - 1

    curses.beep()

    while 0 < row < max_row and 0 < column < max_column:
        canvas.addstr(round(row), round(column), symbol)
        await asyncio.sleep(0)
        canvas.addstr(round(row), round(column), ' ')
        row += rows_speed
        column += columns_speed

async def display_rocket(canvas, rocket_frame_x, rocket_frame_y, window_width, window_height):
    rows_direction, columns_direction, space_pressed = 0, 0, False
    # We're init vars that denote the position of the rocket at a certain time
    rows, columns = 0, 0

    while True:
        rows_direction, columns_direction, space_pressed = read_controls(canvas)

        # Here we indefinitely check that any button is pressed
        if rows_direction != -200:
            rows += rows_direction
            columns += columns_direction
            await asyncio.sleep(0)

        # Here we need to check that rocket is in play field and
        # if it is not in field we return it back
        # To do it we determine the coordinates of corners of rocket
        # To determine the coordinates we
        y_edge_indent = 5
        x_edge_indent = 10
        line_should_be_considered = 1
        top = y_edge_indent + rows + line_should_be_considered
        bottom = y_edge_indent + rows + rocket_frame_y + line_should_be_considered
        indent_in_the_picture_of_rocket = 4
        left = x_edge_indent + columns + indent_in_the_picture_of_rocket
        right = x_edge_indent + columns + rocket_frame_x + indent_in_the_picture_of_rocket
        if top <= 0:
            rows += 1
        elif bottom >= window_width:
            rows -= 1
        elif left <= 0:
            columns += 1
        elif right >= window_height:
            columns -= 1

        draw_frame(canvas, y_edge_indent + rows, x_edge_indent + columns, rocket_frame1)
        canvas.refresh()
        await asyncio.sleep(0.08)

        draw_frame(canvas, y_edge_indent + rows, x_edge_indent + columns,
                   rocket_frame1, negative=True)
        canvas.refresh()
        await asyncio.sleep(0)

        draw_frame(canvas, y_edge_indent + rows, x_edge_indent + columns, rocket_frame2)
        canvas.refresh()
        await asyncio.sleep(0.08)

        draw_frame(canvas, y_edge_indent + rows, x_edge_indent + columns,
                   rocket_frame2, negative=True)
        canvas.refresh()
        await asyncio.sleep(0)

        some_text = " "
        canvas.addstr(9 + rows, 34 + columns, some_text)
        await asyncio.sleep(0)


from sqlite3 import adapters
from numpy.core.shape_base import stack
import torch
import torch.nn.functional as F
from torch import nn
torch.backends.cudnn.benchmark = True
from einops.einops import rearrange
from einops.layers.torch import Rearrange
import numpy as np

from util import box_ops
from util.misc import (NestedTensor, nested_tensor_from_tensor_list,
                       accuracy, get_world_size, interpolate,
                       is_dist_avail_and_initialized)

from .convnext import build_convbackbone, LayerNorm
from .matcher import build_matcher

from .rpet_pose import (RPETpose, PostProcessPoseDR, PostProcessPoseHM, NMTNORMCritierion, filter_target_simdr)
from .transformer import build_transformer

class RPET(nn.Module):
    """ This is the Person Detection Network module that performs perrson detection """
    def __init__(self, backbone, transformer, num_classes, num_queries,
                    num_txrx=8, aux_loss=False, box_feature='x', signal_ratio=0.75):

        super().__init__()
        self.num_queries = num_queries
        self.transformer = transformer
        hidden_dim = transformer.d_model
        print("transformer.d_model = ", hidden_dim)

        self.class_embed = nn.Linear(hidden_dim, num_classes + 1)
        self.bbox_embed = MLP(hidden_dim, hidden_dim, 4, 3)
        self.query_embed = nn.Embedding(num_queries, hidden_dim)

        self.signal_ratio=signal_ratio
        self.hidden_dim = hidden_dim
        self.input_proj_dim = hidden_dim if box_feature =='x' else int(hidden_dim * self.signal_ratio)#75)
        self.input_proj = nn.Sequential(
                LayerNorm(backbone.num_channels, eps=1e-6, data_format="channels_first"),
                nn.Conv1d(backbone.num_channels, self.input_proj_dim, kernel_size=1),
        ) if self.input_proj_dim > 0. else None

        self.backbone = backbone
        self.aux_loss = aux_loss
        self.stack_num = backbone.stack_num
        self.frame_skip = backbone.frame_skip

        self.num_txrx = num_txrx
        self.box_feature = box_feature

        if box_feature != 'x':
            print("use visual clue")
            vc_input_chan = hidden_dim - self.input_proj_dim #64
            self.vc_input_proj = nn.Sequential(
                nn.Conv2d(256, vc_input_chan, kernel_size=1),
                nn.BatchNorm2d(vc_input_chan, momentum=0.1),
            )

        dummy_input = torch.zeros((128, self.stack_num//self.frame_skip, num_txrx**2, 768))

        self.check_dim(dummy_input)

    def forward(self, samples, targets=None):
        """ The forward expects a NestedTensor, which consists of:
               - samples.tensor: batched images, of shape [batch_size x 3 x H x W]
               - samples.mask: a binary mask of shape [batch_size x H x W], containing 1 on padded pixels

            It returns a dict with the following elements:
               - "pred_logits": the classification logits (including no-object) for all queries.
                                Shape= [batch_size x num_queries x (num_classes + 1)]
               - "pred_boxes": The normalized boxes coordinates for all queries, represented as
                               (center_x, center_y, height, width). These values are normalized in [0, 1],
                               relative to the size of each individual image (disregarding possible padding).
                               See PostProcess for information on how to retrieve the unnormalized bounding box.
               - "aux_outputs": Optional, only returned when auxilary losses are activated. It is a list of
                                dictionnaries containing the two above keys for each decoder layer.
        """
        pos, vc_query = None, None
        backbone_out = self.backbone(samples)
        src, mask, pos, vc = backbone_out['src'], backbone_out['mask'], \
                        backbone_out['pos'], backbone_out['pred_feature']
        assert mask is not None

        src = src.flatten(2)
        if self.input_proj != None:
            input_proj = self.input_proj(src)
            input_proj = rearrange(input_proj, 'b n (t1 t2) -> b n t1 t2', t1=16)

        if vc is not None:
            vc_query = self.vc_input_proj(vc)
            input_proj = torch.cat((input_proj, vc_query), dim=1) if self.input_proj != None else vc_query

        hs = self.transformer(input_proj, mask, self.query_embed.weight, pos)[0]
        hs = hs[-1]

        outputs_class = self.class_embed(hs)
        outputs_coord = self.bbox_embed(hs).sigmoid()
        out = {'pred_logits': outputs_class, 'pred_boxes': outputs_coord}

        if self.box_feature != 'x':
            out['pred_feature'] = vc

        if self.aux_loss:
            out['aux_outputs'] = self._set_aux_loss(outputs_class, outputs_coord)

        return out

    def check_dim(self, samples): #samples: NestedTensor):
        print("____check dimension in RPET_____")
        print(f"input shape : {samples.shape}")

        #features= self.backbone(samples)
        pos, vc_query= None, None
        backbone_out = self.backbone(samples)
        src, mask, pos, vc = backbone_out['src'], backbone_out['mask'], \
                        backbone_out['pos'], backbone_out['pred_feature']
        print(f"backbone = src : {src.shape}, mask : {mask.shape}")
        if pos is not None:
            print("position embedding = ", pos.shape)
        if vc is not None:
            print("visual clue = ", vc.shape)
            vc_query = self.vc_input_proj(vc)
            print("visual clue to patch = ", vc_query.shape)
        assert mask is not None

        src = src.flatten(2)
        if self.input_proj != None:
            input_proj = self.input_proj(src)
            input_proj = rearrange(input_proj, 'b n (t1 t2) -> b n t1 t2', t1=16)
            if vc is not None:
                print(f"input_proj[{input_proj.shape}] + vc_query[{vc_query.shape}]" )
        else:
            print(f"input_proj[X] + vc_query[{vc_query.shape}]" )

        if vc is not None:
            input_proj = torch.cat((input_proj, vc_query), dim=1) if self.input_proj != None else vc_query

        print(f"transofmer input self.input_proj(src) : {input_proj.shape}, mask : {mask.shape}", \
                f"\n self.query_embed.weight {self.query_embed.weight.shape}")
        hs = self.transformer(input_proj, mask, self.query_embed.weight, pos)[0]
        print(f"hs(decoder) = {hs.shape}")

        hs = hs[-1]
        outputs_class = self.class_embed(hs)
        outputs_coord = self.bbox_embed(hs).sigmoid()
        print("after ffn ", outputs_class.shape, outputs_coord.shape)

    @torch.jit.unused
    def _set_aux_loss(self, outputs_class, outputs_coord):
        # this is a workaround to make torchscript happy, as torchscript
        # doesn't support dictionary with non-homogeneous values, such
        # as a dict having both a Tensor and a list.
        return [{'pred_logits': a, 'pred_boxes': b}
                for a, b in zip(outputs_class[:-1], outputs_coord[:-1])]

def _expand(tensor, length: int):
    return tensor.unsqueeze(1).repeat(1, int(length), 1, 1, 1).flatten(0, 1)

class SetCriterion(nn.Module):
    """ This class computes the loss for DETR.
    The process happens in two steps:
        1) we compute hungarian assignment between ground truth boxes and the outputs of the model
        2) we supervise each pair of matched ground-truth / prediction (supervise class and box)
    """
    def __init__(self, num_classes, matcher, weight_dict, eos_coef, losses, device=None, pose_method=None):
        """ Create the criterion.
        Parameters:
            num_classes: number of object categories, omitting the special no-object category
            matcher: module able to compute a matching between targets and proposals
            weight_dict: dict containing as key the names of the losses and as values their relative weight.
            eos_coef: relative classification weight applied to the no-object category
            losses: list of all the losses to be applied. See get_loss for list of available losses.
        """
        super().__init__()
        self.num_classes = num_classes
        self.matcher = matcher
        self.weight_dict = weight_dict
        self.eos_coef = eos_coef
        self.losses = losses
        empty_weight = torch.ones(self.num_classes + 1)
        empty_weight[-1] = self.eos_coef
        self.register_buffer('empty_weight', empty_weight)
        if pose_method == 'hm' or pose_method =='hmdr':
            self.hm_criterion = nn.MSELoss(reduction='mean').to(device)
        if pose_method == 'simdr' or pose_method =='hmdr':
            self.cd_criterion = NMTNORMCritierion(label_smoothing=0.1)

        self.device = device
        if 'feature' in losses:
            self.feature_criterion = nn.MSELoss(reduction='mean').to(device)

        self.pose_method=pose_method

    def loss_labels(self, outputs, targets, indices, num_boxes, log=True):
        """Classification loss (NLL)
        targets dicts must contain the key "labels" containing a tensor of dim [nb_target_boxes]
        """
        assert 'pred_logits' in outputs
        src_logits = outputs['pred_logits']
        idx = self._get_src_permutation_idx(indices)
        target_classes_o = torch.cat([t["labels"][J] for t, (_, J) in zip(targets, indices)])
        target_classes = torch.full(src_logits.shape[:2], self.num_classes,
                                    dtype=torch.int64, device=src_logits.device)
        target_classes[idx] = target_classes_o
        loss_ce = F.cross_entropy(src_logits.transpose(1, 2), target_classes, self.empty_weight)
        losses = {'loss_ce': loss_ce}

        if log:
            losses['class_error'] = 100 - accuracy(src_logits[idx][..., :-1], target_classes_o)[0]
        return losses

    @torch.no_grad()
    def loss_cardinality(self, outputs, targets, indices, num_boxes):
        """ Compute the cardinality error, ie the absolute error in the number of predicted non-empty boxes
        This is not really a loss, it is intended for logging purposes only. It doesn't propagate gradients
        """
        pred_logits = outputs['pred_logits']
        device = pred_logits.device
        tgt_lengths = torch.as_tensor([len(v["labels"]) for v in targets], device=device)
        card_pred = (pred_logits.argmax(-1) != pred_logits.shape[-1] - 1).sum(1)
        card_err = F.l1_loss(card_pred.float(), tgt_lengths.float())
        losses = {'cardinality_error': card_err}
        return losses

    def loss_boxes(self, outputs, targets, indices, num_boxes):
        """Compute the losses related to the bounding boxes, the L1 regression loss and the GIoU loss
           targets dicts must contain the key "boxes" containing a tensor of dim [nb_target_boxes, 4]
           The target boxes are expected in format (center_x, center_y, w, h), normalized by the image size.
        """
        assert 'pred_boxes' in outputs
        idx = self._get_src_permutation_idx(indices)
        src_boxes = outputs['pred_boxes'][idx]
        target_boxes = torch.cat([t['boxes'][i] for t, (_, i) in zip(targets, indices)], dim=0)

        loss_bbox = F.l1_loss(src_boxes, target_boxes, reduction='none')

        losses = {}
        losses['loss_bbox'] = loss_bbox.sum() / num_boxes

        loss_giou = 1 - torch.diag(box_ops.generalized_box_iou(
            box_ops.box_cxcywh_to_xyxy(src_boxes),
            box_ops.box_cxcywh_to_xyxy(target_boxes)))

        losses['loss_giou'] = loss_giou.sum() / num_boxes
        return losses

    def loss_pose_dr(self, outputs, targets, indices, num_boxes):
        assert 'x_coord' in outputs

        idx = self._get_src_permutation_idx(indices)
        src_x = outputs['x_coord'][idx]
        src_y = outputs['y_coord'][idx]
        output_size = src_x.shape[2]
        target_pose = torch.cat([t["cd"][J] for t, (_, J) in zip(targets, indices)])
        target_coord = target_pose[:,:,:2]
        target_vis = target_pose[:,:,2]

        target_pose = torch.round(target_coord * output_size)
        target_weight, filtered_joints = filter_target_simdr(target_pose.clone(), target_vis.clone(), output_size)
        target_weight = target_weight.to(src_x)
        filtered_joints = filtered_joints.to(src_x).type(torch.int64)
        loss = self.cd_criterion(src_x, src_y, filtered_joints, target_weight)
        losses = {}
        losses['loss_simdr'] = loss
        return losses

    def loss_pose_hm(self, outputs, targets, indices, num_boxes):
        assert 'pred_hm' in outputs

        src_idx = self._get_src_permutation_idx(indices)
        tgt_idx = self._get_tgt_permutation_idx(indices)
        src_pose = outputs['pred_hm'][src_idx]  #(idx_num, 17, 256)

        tgt_pose = torch.cat([t["hm"][J] for t, (_, J) in zip(targets, indices)])
        tgt_pose = tgt_pose.to(src_pose)
        num_joints = src_pose.size(1)
        heatmaps_pred = src_pose.split(1,1)
        heatmaps_gt = tgt_pose.split(1,1)
        loss = 0

        for j in range(num_joints):
            heatmap_pred = heatmaps_pred[j].squeeze()
            heatmap_gt = heatmaps_gt[j].squeeze()
            tmp_loss = 0.5 * self.hm_criterion(heatmap_pred, heatmap_gt)
            loss += tmp_loss

        loss = loss

        losses = {'loss_hm': loss}
        return losses

    def loss_feature(self, outputs, targets, indices, num_boxes):
        assert 'pred_feature' in outputs

        src_feature = outputs['pred_feature']
        batch_size = src_feature.size(0)

        tgt_feature = torch.cat([t["features"] for t in targets])
        tgt_feature = tgt_feature.to(src_feature)
        losses = {}

        features_pred = src_feature.reshape((batch_size, -1))
        features_gt = tgt_feature.reshape((batch_size, -1))
        feature_loss = self.feature_criterion(features_pred, features_gt)

        losses['loss_feature'] = feature_loss #/ (x*y)
        if indices == None:
            losses['class_error'] = 0.

        return losses

    def _get_src_permutation_idx(self, indices):
        batch_idx = torch.cat([torch.full_like(src, i) for i, (src, _) in enumerate(indices)])
        src_idx = torch.cat([src for (src, _) in indices])
        return batch_idx, src_idx

    def _get_tgt_permutation_idx(self, indices):
        batch_idx = torch.cat([torch.full_like(tgt, i) for i, (_, tgt) in enumerate(indices)])
        tgt_idx = torch.cat([tgt for (_, tgt) in indices])
        return batch_idx, tgt_idx

    def get_loss(self, loss, outputs, targets, indices, num_boxes, **kwargs):
        loss_map = {
            'labels': self.loss_labels,
            'cardinality': self.loss_cardinality,
            'boxes': self.loss_boxes,
            'posedr' : self.loss_pose_dr,
            'posehm' : self.loss_pose_hm,
            'feature': self.loss_feature,
        }
        assert loss in loss_map, f'do you really want to compute {loss} loss?'
        return loss_map[loss](outputs, targets, indices, num_boxes, **kwargs)

    def forward(self, outputs, targets):
        """ This performs the loss computation.
        Parameters:
             outputs: dict of tensors, see the output specification of the model for the format
             targets: list of dicts, such that len(targets) == batch_size.
                      The expected keys in each dict depends on the losses applied, see each loss' doc
        """
        outputs_without_aux = {k: v for k, v in outputs.items() if k != 'aux_outputs'}

        if 'pred_logits' in outputs:
            indices = self.matcher(outputs_without_aux, targets)

            # Compute the average number of target boxes accross all nodes, for normalization purposes
            num_boxes = sum(len(t["labels"]) for t in targets)
            num_boxes = torch.as_tensor([num_boxes], dtype=torch.float, device=next(iter(outputs.values())).device)
            if is_dist_avail_and_initialized():
                torch.distributed.all_reduce(num_boxes)
            num_boxes = torch.clamp(num_boxes / get_world_size(), min=1).item()
        else:
            indices = None
            num_boxes = 0
        # Compute all the requested losses
        losses = {}
        for loss in self.losses:
            losses.update(self.get_loss(loss, outputs, targets, indices, num_boxes))

        # In case of auxiliary losses, we repeat this process with the output of each intermediate layer.
        if 'aux_outputs' in outputs:
            for i, aux_outputs in enumerate(outputs['aux_outputs']):
                indices = self.matcher(aux_outputs, targets)
                for loss in self.losses:
                    if loss == 'posehm' or loss =='posedr':
                        # Intermediate masks losses are too costly to compute, we ignore them.
                        continue
                    kwargs = {}
                    if loss == 'labels':
                        # Logging is enabled only for the last layer
                        kwargs = {'log': False}
                    l_dict = self.get_loss(loss, aux_outputs, targets, indices, num_boxes, **kwargs)
                    l_dict = {k + f'_{i}': v for k, v in l_dict.items()}
                    losses.update(l_dict)

        return losses

def soft_nms_pytorch(dets, box_scores, sigma=0.5, thresh=0., cuda=1):
    # Indexes concatenate boxes with the last column
    N = dets.shape[0]
    if cuda:
        #indexes = torch.arange(0, N, dtype=torch.float).cuda().view(N, 1)
        indexes = torch.arange(0, N, dtype=torch.float).view(N, 1).to(dets)
    else:
        indexes = torch.arange(0, N, dtype=torch.float).view(N, 1)
    dets = torch.cat((dets, indexes), dim=1)
    #print(dets)
    # The order of boxes coordinate is [y1,x1,y2,x2]
    x1 = dets[:, 0]
    y1 = dets[:, 1]
    x2 = dets[:, 2]
    y2 = dets[:, 3]
    scores = box_scores
    areas = (x2 - x1 + 1) * (y2 - y1 + 1)
    for i in range(N):
        # intermediate parameters for later parameters exchange
        tscore = scores[i].clone()
        pos = i + 1

        if i != N - 1:
            maxscore, maxpos = torch.max(scores[pos:], dim=0)
            if tscore < maxscore:
                dets[i], dets[maxpos.item() + i + 1] = dets[maxpos.item() + i + 1].clone(), dets[i].clone()
                scores[i], scores[maxpos.item() + i + 1] = scores[maxpos.item() + i + 1].clone(), scores[i].clone()
                areas[i], areas[maxpos + i + 1] = areas[maxpos + i + 1].clone(), areas[i].clone()

        # IoU calculate
        yy1 = np.maximum(dets[i, 0].to("cpu").numpy(), dets[pos:, 0].to("cpu").numpy())
        xx1 = np.maximum(dets[i, 1].to("cpu").numpy(), dets[pos:, 1].to("cpu").numpy())
        yy2 = np.minimum(dets[i, 2].to("cpu").numpy(), dets[pos:, 2].to("cpu").numpy())
        xx2 = np.minimum(dets[i, 3].to("cpu").numpy(), dets[pos:, 3].to("cpu").numpy())

        w = np.maximum(0.0, xx2 - xx1 + 1)
        h = np.maximum(0.0, yy2 - yy1 + 1)
        inter = torch.tensor(w * h).to(dets) if cuda else torch.tensor(w * h)
        ovr = torch.div(inter, (areas[i] + areas[pos:] - inter))

        # Gaussian decay
        weight = torch.exp(-(ovr * ovr) / sigma)
        scores[pos:] = weight * scores[pos:]

    # select the boxes and keep the corresponding indexes
    #keep = dets[:, 4][scores > thresh].int()
    keep = dets[:, 4].int()

    return dets[:, :4], scores, keep

class PostProcess(nn.Module):
    """ This module converts the model's output into the format expected by the coco api"""
    @torch.no_grad()
    def forward(self, outputs, target_sizes, nms=False):
        """ Perform the computation
        Parameters:
            outputs: raw outputs of the model
            target_sizes: tensor of dimension [batch_size x 2] containing the size of each images of the batch
                          For evaluation, this must be the original image size (before any data augmentation)
                          For visualization, this should be the image size after data augment, but before padding
        """
        out_logits, out_bbox = outputs['pred_logits'], outputs['pred_boxes']

        assert len(out_logits) == len(target_sizes)
        assert target_sizes.shape[1] == 2

        #batch_size = out_bbox.shape[0]
        prob = F.softmax(out_logits, -1)
        #print(prob.shape)
        #print(prob[..., :-1].shape)
        scores, labels = prob[..., :-1].max(-1) # No Obejct는 제외함

        # convert to [x0, y0, x1, y1] format
        boxes = box_ops.box_cxcywh_to_xyxy(out_bbox)
        # and from relative [0, 1] to absolute [0, height] coordinates
        img_h, img_w = target_sizes.unbind(1)
        scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1)
        boxes = boxes * scale_fct[:, None, :]

        batch_size, N = boxes.shape[0], boxes.shape[1]
        indices = torch.zeros((batch_size, N))
        if nms:
            for b in range(batch_size):
                dets = boxes[b]
                score = scores[b]
                #print(boxes[b], scores[b])
                boxes[b], scores[b], keep = soft_nms_pytorch(dets, score)
                labels[b] = torch.index_select(labels[b], 0, keep)
                indices[b] = keep
        else:
            for b in range(batch_size):
                indices[b] = torch.arange(0, N, dtype=torch.float)
        #print(indices)
        results = [{'scores': s, 'labels': l, 'boxes': b, 'indices': idx} for s, l, b, idx in zip(scores, labels, boxes, indices)]

        return results

class MLP(nn.Module):
    """ Very simple multi-layer perceptron (also called FFN)"""

    def __init__(self, input_dim, hidden_dim, output_dim, num_layers):
        super().__init__()
        self.num_layers = num_layers
        h = [hidden_dim] * (num_layers - 1)
        self.layers = nn.ModuleList(nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim]))

    def forward(self, x):
        for i, layer in enumerate(self.layers):
            x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x)
        return x

def build_rpet(args):
    num_classes = 91
    device = torch.device(args.device)
    backbone = build_convbackbone(args)

    transformer = build_transformer(args)

    model = RPET(
        backbone,
        transformer,
        num_classes=num_classes,
        num_queries=args.num_queries,
        aux_loss=args.aux_loss,
        box_feature=args.box_feature,
        signal_ratio=args.signal_ratio
    )

    if args.pose is not None:
        model = RPETpose(model, freeze_rpet=(args.frozen_weights is not None),
                            method=args.pose, dr_size=args.dr_size)

    matcher = build_matcher(args)
    weight_dict = {'loss_ce': 1, 'loss_bbox': args.bbox_loss_coef}
    weight_dict['loss_giou'] = args.giou_loss_coef

    if args.pose == 'simdr':
        weight_dict["loss_simdr"] = args.pose_loss_coef
    elif args.pose =='hm':
        weight_dict["loss_hm"] = args.pose_loss_coef * 5
    elif args.pose =='hmdr':
        weight_dict["loss_simdr"] = args.pose_loss_coef
        weight_dict["loss_hm"] = args.pose_loss_coef

    if args.box_feature == '16' and args.pose == None:
        weight_dict['loss_feature'] = args.feature_loss_coef

    if args.aux_loss:
        aux_weight_dict = {}
        for i in range(args.dec_layers - 1):
            aux_weight_dict.update({k + f'_{i}': v for k, v in weight_dict.items()})
        weight_dict.update(aux_weight_dict)

    losses = ['labels', 'boxes', 'cardinality']

    if args.pose == 'simdr':
        losses += ['posedr']
    elif args.pose == 'hm':
        losses += ['posehm']
    elif args.pose =='hmdr':
        losses += ['posedr']
        losses += ['posehm']

    if args.box_feature == '16' and args.pose == None:
        losses += ['feature']

    criterion = SetCriterion(num_classes, matcher=matcher, weight_dict=weight_dict,
                            eos_coef=args.eos_coef, losses=losses, device=device, pose_method=args.pose)
    criterion.to(device)
    postprocessors = {'bbox': PostProcess()}
    if args.pose == 'simdr':
        postprocessors['posedr'] = PostProcessPoseDR()
    elif args.pose == 'hm':
        postprocessors['posehm'] = PostProcessPoseHM()
    elif args.pose == 'hmdr':
        postprocessors['posedr'] = PostProcessPoseDR()

    print(f"losses = {losses}")
    print(f"weight_dict = {weight_dict}")
    print(f"postprocessors = {postprocessors.keys()}")

    return model, criterion, postprocessors


from re import search
from setuptools import setup, find_packages

with open("src/graphql/version.py") as version_file:
    version = search('version = "(.*)"', version_file.read()).group(1)

with open("README.md") as readme_file:
    readme = readme_file.read()

setup(
    name="graphql-core",
    version=version,
    description="GraphQL-core-next is a Python port of GraphQL.js,"
    " the JavaScript reference implementation for GraphQL.",
    long_description=readme,
    long_description_content_type="text/markdown",
    keywords=["graphql"],
    url="https://github.com/nudjur/graphql-core",
    author="Christoph Zwerschke",
    author_email="cito@online.de",
    license="MIT license",
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Intended Audience :: Developers",
        "Topic :: Software Development :: Libraries",
        "License :: OSI Approved :: MIT License",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
    ],
    install_requires=[],
    python_requires=">=3.6,<4",
    packages=find_packages("src"),
    package_dir={"": "src"},
    # PEP-561: https://www.python.org/dev/peps/pep-0561/
    package_data={"graphql": ["py.typed"]},
    include_package_data=True,
    zip_safe=False,
)


from django.urls import path
from .views import task_list, task_detail, task_create, task_delete

urlpatterns = [
    path('tasks/', task_list ),
    path('tasks/int:pk', task_detali),
    path('task_create/', task_create),
    path('task_delete/int:pk', task_delete)

]


import os
import sys
import json
import torch
import random
import inspect
import requests
import argparse
import importlib
import numpy as np
from pathlib import Path
from itertools import product
from tqdm import tqdm
from PyInquirer import prompt, Separator, Validator, ValidationError

DATA_FOLDER = 'data/'

class FastaValidator(Validator):
    def validate(self, document):
        if(type(document) == str):
            text = document
        else:
            text = document.text
        data_folder = os.path.abspath(DATA_FOLDER)
        my_file = Path(text)
        if not my_file.is_file():
            raise ValidationError(
                message='Please enter a valid file',
                cursor_position=len(text))
        elif my_file.suffix not in ['.fa', '.fna', '.fsa', '.fasta']:
            raise ValidationError(
                message='Please enter a FASTA file. Extensions: .fa .fna .fsa .fasta',
                cursor_position=len(text))
        elif not str(os.path.dirname(my_file.absolute())) == data_folder:
            raise ValidationError(
                message='Make sure your data is in the correct folder',
                cursor_position=len(text)
                )

class AnnotationValidator(Validator):
    def validate(self, document):
        if(type(document) == str):
            text = document
        else:
            text = document.text
        data_folder = os.path.abspath(DATA_FOLDER)
        my_file = Path(text)
        if not my_file.is_file():
            raise ValidationError(
                message='Please enter a valid file',
                cursor_position=len(text))
        elif my_file.suffix not in ['.sga', '.bed']:
            raise ValidationError(
                message='Please enter an annotation file. Extensions: .sga and .bed',
                cursor_position=len(text))
        elif not str(os.path.dirname(my_file.absolute())) == data_folder:
            raise ValidationError(
                message='Make sure your data is in the correct folder',
                cursor_position=len(text)
                )

class FolderValidator(Validator):
    def validate(self, document):
        if(type(document) == str):
            text = document
        else:
            text = document.text
        my_folder = Path(text)
        if my_folder.exists():
            if not my_folder.is_dir():
                raise ValidationError(
                    message='Please enter a valid folder',
                    cursor_position=len(text))
        else:
            os.makedirs(my_folder)

class IsFileValidator(Validator):
    def validate(self, document):
        if(type(document) == str):
            text = document
        else:
            text = document.text
        my_file = Path(text)
        if not my_file.exists():
            raise ValidationError(
                    message='Please enter a file that exists',
                    cursor_position=len(text))

class FileExistsValidator(Validator):
    def validate(self, document):
        if(type(document) == str):
            text = document
        else:
            text = document.text
        my_file = Path(text)
        if my_file.exists():
            raise ValidationError(
                    message='Please enter a file that does not exist',
                    cursor_position=len(text))

class ConfigValidator(Validator):
    def validate(self, document):
        if(type(document) == str):
            text = document
        else:
            text = document.text
        config = Path(text)
        if config.is_file():
            if not config.suffix == '.json':
                raise ValidationError(
                    message='Please enter a configuration file. Extension: .json',
                    cursor_position=len(text))
        else:
            raise ValidationError(
                message='Please enter a valid file',
                cursor_position=len(text))

class PositiveValidator(Validator):
    def validate(self, document):
        if(type(document) == int):
            text = str(document)
        elif(type(document) == float):
            text = str(document)
        elif(type(document) == str):
            text = document
        else:
            text = document.text
        try:
            if not float(text) > 0:
                raise ValidationError(
                    message='Please enter a positive number',
                    cursor_position=len(text))
        except:
            raise ValidationError(
                message='Input %s was not a valid number' % text,
                cursor_position=len(text))

class SeedValidator(Validator):
    def validate(self, document):
        if(type(document) == int):
            text = str(document)
        elif(type(document) == str):
            text = document
        else:
            text = document.text
        try:
            if not int(text) >= 0:
                raise ValidationError(
                    message='Please enter a positive number or use 0 to generate random seed',
                    cursor_position=len(text))
        except:
            raise ValidationError(
                message='Seed was not a valid number',
                cursor_position=len(text))

class FunctionValidator(Validator):
    def validate(self, document):
        if(type(document) == str):
            text = document
        else:
            text = document.text
        if not text in Helper._NN_OUTPUT_FUNCTIONS:
            raise ValidationError(
                message=f'Please enter a valid output function. Functions: {Helper._NN_OUTPUT_FUNCTIONS}',
                cursor_position=len(text))

class ModuleValidator(Validator):
    def validate(self, document):
        if(type(document) == str):
            text = document
        else:
            text = document.text
        if not text in Helper._NN_MODULES:
            raise ValidationError(
                message=f'Please enter a valid module. Functions: {Helper._NN_MODULES}',
                cursor_position=len(text))

class OptimizerValidator(Validator):
    def validate(self, document):
        if(type(document) == str):
            text = document
        else:
            text = document.text
        if not text in Helper._NN_OPTIMIZERS:
            raise ValidationError(
                message=f'Please enter a valid optimizer. Functions: {Helper._NN_OPTIMIZERS}',
                cursor_position=len(text))

class Helper:
    DNA_DICT = {}
    _k = 1
    CONF_DICT: dict
    _FILE_TYPES = ['sld', 'fasta', 'kmer']
    _EXPERIMENT_CONFIGURATION_FILENAME = 'ExperimentConfiguration.json'
    _EXPERIMENT_MODULE_ARGS_FILENAME = 'module.args'
    _EXPERIMENT_OPTIMIZER_ARGS_FILENAME = 'optimizer.args'
    _NN_OUTPUT_FUNCTIONS = ['sigmoid', 'softmax']
    _NN_MODULES: dict
    _NN_MODULE_ARGS = {}
    _NN_OPTIMIZER_ARGS = {}
    _NN_OPTIMIZERS = ['Adam', 'Adadelta', 'Adagrad', 'AdamW', 'SparseAdam', 'Adamax', 'ASGD', 'LBFGS', 'NAdam', 'RAdam', 'RMSprop', 'Rprop', 'SGD'] # Algorithms: https://pytorch.org/docs/stable/optim.html
    _F5_SPECIES = ['human', 'mouse', 'rat', 'dog', 'chicken', 'rhesus']
    _EPD_DATABASES = ['human', 'human_nc', 'M_mulatta', 'mouse', 'mouse_nc', 'R_norvegicus', 'C_familiaris', 'G_gallus', 'drosophila', 'A_mellifera', 'zebrafish', 'worm', 'arabidopsis', 'Z_mays', 'S_cerevisiae', 'S_pombe', 'P_falciparum']
    _EPD_TATA_FILTERS = ['all', 'with', 'without']
    _DATA_FOLDER = 'data/'
    _DICTKEY_NN_OUTPUT_FUNCTION = 'function'
    _DICTKEY_NN_MODULE = 'module'
    _DICTKEY_NN_TRAIN_ARGS = 'training_arguments'
    _DICTKEY_NN_MAX_EPOCHS = 'max_epochs'
    _DICTKEY_NN_PATIENCE = 'patience'
    _DICTKEY_NN_LEARNING_RATE = 'learning_rate'
    _DICTKEY_NN_BATCH_SIZE = 'batch_size'
    _DICTKEY_NN_OPTIMIZER = 'optimizer'
    _DICTKEY_INPUT_FILE = 'input'
    _DICTKEY_OUTPUT_FILE = 'output'
    _DICTKEY_ANNOTATIONS = 'annotations'
    _DICTKEY_CONFIGURATION = 'dataset_configuration'
    _DICTKEY_DATASET_TYPE = 'dataset'
    _DICTKEY_EXPERIMENT_FOLDER = 'experiment'
    _DICTKEY_SEQ_UPSTREAM_LEN = 'sequence_upstream_length'
    _DICTKEY_SEQ_DOWNSTREAM_LEN = 'sequence_downstream_length'
    _DICTKEY_STRIDE = 'stride'
    _DICTKEY_ERROR_TYPE = 'error_type'
    _DICTKEY_ERROR_MARGIN = 'error_margin'
    _DICTKEY_SEED = 'seed'
    _DICTKEY_F5_SPECIES = 'species'
    _DICTKEY_EPD_DATABASE = 'database'
    _DICTKEY_EPD_TATA_FILTER = 'tata_filter'
    _DICTKEY_INPUT_TYPE = 'input_type'
    _DICTKEY_OUTPUT_TYPE = 'output_type'
    _overwrite = False
    _maxepochs_default = 100
    _batchsize_default = 16
    _learningrate_default = 0.001
    _patience_default = 10
    _upstream_default = 1000 # promoter upstream length for the annotations
    _downstream_default = 400 # promoter downstream length for the annotations
    _stride_default = 50 # step size for the number of nucleotides to skip while moving the window
    _error_type_default = 'tss-proximity' # error types: proximity to TSS, sequence overlap
    _IO_ERROR_TYPES = ['sequence-overlap', 'tss-proximity', 'seq-proximity']
    _error_margin_default = [100] # list, if 1 value then its TSS proximity, if 2 values then its sequence overlap (from upstream, from downstream)
    _DATASET_CHOICES = ['IO', 'BIO', 'BME', 'CBG', 'CBPS', 'LITERATURE'] # "CBG" cluster by gene, "CBPS" cluster by promoter similarity

    def __init__(self):
        self.create_kmers()
        self._NN_MODULES = self.get_modules()
        dataset_configuration = {self._DICTKEY_SEQ_UPSTREAM_LEN: self._upstream_default,
                                 self._DICTKEY_SEQ_DOWNSTREAM_LEN: self._downstream_default,
                                 self._DICTKEY_STRIDE: self._stride_default,
                                 self._DICTKEY_SEED: 0}
        self.CONF_DICT = {self._DICTKEY_CONFIGURATION: dataset_configuration}

    def create_kmers(self):
        bases=['A','T','G','C']
        for i, p in enumerate(product(bases, repeat=self._k)):
            self.DNA_DICT[''.join(p)] = i

    def load_experiment(self):
        experiment_folder = self.CONF_DICT[self._DICTKEY_EXPERIMENT_FOLDER]
        config_file = os.path.join(experiment_folder, self._EXPERIMENT_CONFIGURATION_FILENAME)
        with open(config_file) as f:
            self.CONF_DICT = json.load(f)
        module_args_file = os.path.join(experiment_folder, self._EXPERIMENT_MODULE_ARGS_FILENAME)
        if os.path.isfile(module_args_file):
            with open(module_args_file) as f:
                self._NN_MODULE_ARGS = json.load(f)
        self._NN_MODULE_ARGS = {f'module__{k}': v for k, v in self._NN_MODULE_ARGS.items()}
        optimizer_args_file = os.path.join(experiment_folder, self._EXPERIMENT_OPTIMIZER_ARGS_FILENAME)
        if os.path.isfile(optimizer_args_file):
            with open(optimizer_args_file) as f:
                self._NN_OPTIMIZER_ARGS = json.load(f)
        self._NN_OPTIMIZER_ARGS = {f'optimizer__{k}': v for k, v in self._NN_OPTIMIZER_ARGS.items()}
        #TODO: validate loaded json

    def read_create_arguments(self):
        parser = argparse.ArgumentParser(description='Create a dataset')
        parser.add_argument('-i', f'--{self._DICTKEY_INPUT_FILE}', metavar='input file', required=True,
                            type=str, help='path to the input genome')
        parser.add_argument('-a', f'--{self._DICTKEY_ANNOTATIONS}', metavar='annotations file', required=True,
                            type=str, help='path to the TSS annotations for the genome')
        parser.add_argument('-t', f'--{self._DICTKEY_DATASET_TYPE}', metavar='dataset type', required=True,
                            type=str, help='Type of dataset to be created', choices=self._DATASET_CHOICES)
        parser.add_argument('-c', f'--{self._DICTKEY_CONFIGURATION}', metavar='configuration file', required=False,
                            type=str, help='path to the configuration file. If not supplied, defaults will be used')
        parser.add_argument('-e', f'--{self._DICTKEY_EXPERIMENT_FOLDER}', metavar='experiment folder', required=True,
                            type=str, help='path to the experiment folder')

        args = parser.parse_args(sys.argv[3:])

        input_file = args.__dict__[self._DICTKEY_INPUT_FILE]
        FastaValidator().validate(input_file)
        self.CONF_DICT[self._DICTKEY_INPUT_FILE] = os.path.relpath(input_file)

        annotations_file = args.__dict__[self._DICTKEY_ANNOTATIONS]
        AnnotationValidator().validate(annotations_file)
        self.CONF_DICT[self._DICTKEY_ANNOTATIONS] = os.path.relpath(annotations_file)

        self.validate_config(args.__dict__[self._DICTKEY_DATASET_TYPE], args.__dict__[self._DICTKEY_CONFIGURATION])

        experiment_folder = args.__dict__[self._DICTKEY_EXPERIMENT_FOLDER]
        FolderValidator().validate(experiment_folder)
        self.CONF_DICT[self._DICTKEY_EXPERIMENT_FOLDER] = os.path.abspath(experiment_folder)

    def read_train_arguments(self):
        parser = argparse.ArgumentParser(description='Train an experiment')
        parser.add_argument('-e', f'--{self._DICTKEY_EXPERIMENT_FOLDER}', metavar='experiment folder', required=True,
                            type=str, help='Path to the experiment folder')
        parser.add_argument('-x', f'--{self._DICTKEY_NN_MAX_EPOCHS}', metavar='max epochs', required=False, default=self._maxepochs_default,
                            type=int, help='Maximum number of epochs')
        parser.add_argument('-p', f'--{self._DICTKEY_NN_PATIENCE}', metavar='patience', required=False, default=self._patience_default,
                            type=int, help='Stop training after this many epochs without progress')
        parser.add_argument('-l', f'--{self._DICTKEY_NN_LEARNING_RATE}', metavar='learning rate', required=False, default=self._learningrate_default,
                            type=int, help='Value for the learning rate parameter')
        parser.add_argument('-b', f'--{self._DICTKEY_NN_BATCH_SIZE}', metavar='batch size', required=False, default=self._batchsize_default,
                            type=int, help='Value for the batch size parameter')
        parser.add_argument('-t', f'--{self._DICTKEY_NN_OPTIMIZER}', metavar='optimizer', required=False, default=self._NN_OPTIMIZERS[0],
                            type=str, help='Optimizer for training the neural network', choices=self._NN_OPTIMIZERS)
        parser.add_argument('-f', f'--{self._DICTKEY_NN_OUTPUT_FUNCTION}', metavar='output function', required=False, default=self._NN_OUTPUT_FUNCTIONS[0],
                            type=str, help='Function of the last layer of the neural network', choices=self._NN_OUTPUT_FUNCTIONS)
        parser.add_argument('-m', f'--{self._DICTKEY_NN_MODULE}', metavar='pytorch neural network module', required=False, default='dprom',
                            type=str, help='Pytorch neural network architecture module to train', choices=self._NN_MODULES)
        args = parser.parse_args(sys.argv[2:])
        experiment_folder = args.__dict__[self._DICTKEY_EXPERIMENT_FOLDER]
        FolderValidator().validate(experiment_folder)
        self.CONF_DICT[self._DICTKEY_EXPERIMENT_FOLDER] = experiment_folder
        for k in [self._DICTKEY_NN_MAX_EPOCHS, self._DICTKEY_NN_PATIENCE,
                self._DICTKEY_NN_LEARNING_RATE, self._DICTKEY_NN_BATCH_SIZE]:
            PositiveValidator().validate(args.__dict__[k])
        args.__dict__.pop(self._DICTKEY_EXPERIMENT_FOLDER)
        return args.__dict__

    def read_convert_arguments(self):
        parser = argparse.ArgumentParser(description='File conversion system')
        parser.add_argument('-i', f'--{self._DICTKEY_INPUT_FILE}', metavar='input file', required=True,
                            type=str, help='Path to the input file')
        parser.add_argument('-o', f'--{self._DICTKEY_OUTPUT_FILE}', metavar='output file', required=True,
                            type=str, help='Path to save the converted file')
        parser.add_argument('-t', f'--{self._DICTKEY_INPUT_TYPE}', metavar='input type', required=True, default=self._FILE_TYPES[0],
                            type=str, help='Type of the input file', choices=self._FILE_TYPES)
        parser.add_argument('-e', f'--{self._DICTKEY_OUTPUT_TYPE}', metavar='output type', required=True, default=self._FILE_TYPES[1],
                            type=str, help='Type of the output file', choices=self._FILE_TYPES)
        args = parser.parse_args(sys.argv[2:])
        input_file = args.__dict__[self._DICTKEY_INPUT_FILE]
        if input_file:
            IsFileValidator().validate(input_file)
        output_file = args.__dict__[self._DICTKEY_OUTPUT_FILE]
        if output_file:
            FileExistsValidator().validate(output_file)
        input_type = args.__dict__[self._DICTKEY_INPUT_TYPE]
        output_type = args.__dict__[self._DICTKEY_OUTPUT_TYPE]
        if input_type == output_type:
            raise RuntimeError(msg="No conversion necessary. Input type = Output type")
        self.CONF_DICT = args.__dict__

    def read_epd_download_arguments(self):
        parser = argparse.ArgumentParser(description='Download annotations from Eukaryotic Promoter Database (EPDnew)')
        parser.add_argument('-o', f'--{self._DICTKEY_OUTPUT_FILE}', metavar='output file', required=False,
                            type=str, help='Path where the downloaded file will be saved')
        parser.add_argument('-d', f'--{self._DICTKEY_EPD_DATABASE}', metavar='database', required=True, default=self._EPD_DATABASES[0],
                            type=str, help='Database (species) to query', choices=self._EPD_DATABASES)
        parser.add_argument('-t', f'--{self._DICTKEY_EPD_TATA_FILTER}', metavar='TATA motif filter', required=True, default=self._EPD_TATA_FILTERS[0],
                            type=str, help='Filter promoters by TATA motif', choices=self._EPD_TATA_FILTERS)
        args = parser.parse_args(sys.argv[3:])
        output_file = args.__dict__[self._DICTKEY_OUTPUT_FILE]
        if output_file:
            FolderValidator().validate(os.path.dirname(output_file))
        return args

    def read_f5_download_arguments(self):
        parser = argparse.ArgumentParser(description='Download annotations from Riken Fantom (5) project')
        parser.add_argument('-o', f'--{self._DICTKEY_OUTPUT_FILE}', metavar='output file', required=False,
                            type=str, help='Path where the downloaded file will be saved')
        parser.add_argument('-s', f'--{self._DICTKEY_F5_SPECIES}', metavar='species', required=True, default=self._F5_SPECIES[0],
                            type=str, help='Database (species) to query', choices=self._F5_SPECIES)
        args = parser.parse_args(sys.argv[3:])
        output_file = args.__dict__[self._DICTKEY_OUTPUT_FILE]
        if output_file:
            FolderValidator().validate(os.path.dirname(output_file))
        return args

    def validate_config(self, db_type, config):
        try:
            self.CONF_DICT[self._DICTKEY_DATASET_TYPE] = db_type
            with open(config) as f:
                data = json.load(f)
                configuration = data[self._DICTKEY_CONFIGURATION]
        except TypeError:
            print('No configuration file supplied. Default configuration will be used')
            self.insert_default_values(db_type)
            self.validate_seed()
            return
        except:
            ConfigValidator().validate(config)

        print('Checking dataset configuration:')
        for key in self.CONF_DICT[self._DICTKEY_CONFIGURATION].keys():
            try:
                PositiveValidator().validate(configuration[key])
            except (KeyError, UnboundLocalError):
                raise Exception('Incomplete configuration file detected')
            except ValidationError:
                if key == self._DICTKEY_SEED:
                    continue

        self.CONF_DICT[self._DICTKEY_CONFIGURATION] = configuration
        self.validate_seed()
        if(db_type == self._DATASET_CHOICES[0]): # IO
            self.validate_io(config)
        elif(db_type == self._DATASET_CHOICES[1]): # BIO
            #TODO
            raise Exception("Functionality currently unavailable")
        elif(db_type == self._DATASET_CHOICES[2]): # BME
            #TODO
            raise Exception("Functionality currently unavailable")
        elif(db_type == self._DATASET_CHOICES[3]): # CBG
            #TODO
            raise Exception("Functionality currently unavailable")
        elif(db_type == self._DATASET_CHOICES[4]): # CBPS
            #TODO
            raise Exception("Functionality currently unavailable")
        elif(db_type == self._DATASET_CHOICES[5]): # LITERATURE
            #TODO
            raise Exception("Functionality currently unavailable")

    def insert_default_values(self, db_type):
        if(db_type == self._DATASET_CHOICES[0]): # IO
            self.CONF_DICT[self._DICTKEY_CONFIGURATION][self._DICTKEY_ERROR_TYPE] = self._error_type_default
            self.CONF_DICT[self._DICTKEY_CONFIGURATION][self._DICTKEY_ERROR_MARGIN] = self._error_margin_default
        elif(db_type == self._DATASET_CHOICES[1]): # BIO
            #TODO
            raise Exception("Functionality currently unavailable")
        elif(db_type == self._DATASET_CHOICES[2]): # BME
            #TODO
            raise Exception("Functionality currently unavailable")
        elif(db_type == self._DATASET_CHOICES[3]): # CBG
            #TODO
            raise Exception("Functionality currently unavailable")
        elif(db_type == self._DATASET_CHOICES[4]): # CBPS
            #TODO
            raise Exception("Functionality currently unavailable")
        elif(db_type == self._DATASET_CHOICES[5]): # LITERATURE
            #TODO
            raise Exception("Functionality currently unavailable")

    def validate_io(self, config):
        try:
            print('Checking error type in configuration file:')
            if(self.CONF_DICT[self._DICTKEY_CONFIGURATION][self._DICTKEY_ERROR_TYPE] == self._IO_ERROR_TYPES[0]): # sequence overlap
                if(len(self.CONF_DICT[self._DICTKEY_CONFIGURATION][self._DICTKEY_ERROR_MARGIN]) != 2):
                    raise Exception('Configuration file error: Error margin is not valid')
            elif(self.CONF_DICT[self._DICTKEY_CONFIGURATION][self._DICTKEY_ERROR_TYPE] == self._IO_ERROR_TYPES[1]): # tss proximity
                if(len(self.CONF_DICT[self._DICTKEY_CONFIGURATION][self._DICTKEY_ERROR_MARGIN]) != 1):
                    raise Exception('Configuration file error: Error margin is not valid')
            elif(self.CONF_DICT[self._DICTKEY_CONFIGURATION][self._DICTKEY_ERROR_TYPE] == self._IO_ERROR_TYPES[2]): # seq proximity
                if(len(self.CONF_DICT[self._DICTKEY_CONFIGURATION][self._DICTKEY_ERROR_MARGIN]) != 1):
                    raise Exception('Configuration file error: Error margin is not valid')
            else:
                raise Exception('Configuration file error: Error type is missing or invalid')

            print('Checking error margins in configuration file:')
            for value in self.CONF_DICT[self._DICTKEY_CONFIGURATION][self._DICTKEY_ERROR_MARGIN]:
                PositiveValidator().validate(value)
        except KeyError:
            raise Exception('Incomplete configuration file detected')
        print('Configuration file check completed')

    def validate_seed(self):
        try:
            seed = self.CONF_DICT[self._DICTKEY_CONFIGURATION][self._DICTKEY_SEED]
            SeedValidator().validate(seed)
        except KeyError:
            print('Seed not provided in configuration file.')
            seed = 0
        self._set_seeds(seed)

    def save_experiment_config(self):
        save_file = os.path.join(self.CONF_DICT[self._DICTKEY_EXPERIMENT_FOLDER], self._EXPERIMENT_CONFIGURATION_FILENAME)
        self.save_file(save_file, Helper.save_json, json_obj=self.CONF_DICT)

    def ask_and_save_config_parameters(self):
        self.ask_dataset_config()
        self.ask_dataset_type()
        self.ask_error_type()
        save_file = os.path.join(os.getcwd(), 'configuration.json')
        self.save_file(save_file, Helper.save_json, json_obj=self.CONF_DICT)

    def ask_experiment_parameters(self):
        self.ask_config()
        self.ask_dataset_config()
        self.ask_dataset_type()
        self.ask_error_type()

    def ask_convert_parameters(self):
        input_questions = [
            {
                'type': 'input',
                'name': self._DICTKEY_INPUT_FILE,
                'message': 'Input file:',
                'filter': lambda val: os.path.relpath(val),
                'validate': IsFileValidator
            },
            {
                'type': 'list',
                'name': self._DICTKEY_INPUT_TYPE,
                'message': 'Input file type:',
                'choices': self._FILE_TYPES
            }
        ]
        for k, v in prompt(input_questions).items():
            self.CONF_DICT[k] = v
        out_file_types = self._FILE_TYPES.copy()
        out_file_types.remove(self.CONF_DICT[self._DICTKEY_INPUT_TYPE])
        output_questions = [
            {
                'type': 'input',
                'name': self._DICTKEY_OUTPUT_FILE,
                'message': 'Output file:',
                'filter': lambda val: os.path.relpath(val),
                'validate': FileExistsValidator
            },
            {
                'type': 'list',
                'name': self._DICTKEY_OUTPUT_TYPE,
                'message': 'Output file type:',
                'choices': out_file_types # Has the input type removed
            }
        ]
        for k, v in prompt(output_questions).items():
            self.CONF_DICT[k] = v

    def ask_config(self):
        questions = [
            {
                'type': 'input',
                'name': self._DICTKEY_INPUT_FILE,
                'message': 'Input file:',
                'filter': lambda val: os.path.relpath(val),
                'validate': FastaValidator
            },
            {
                'type': 'input',
                'name': self._DICTKEY_ANNOTATIONS,
                'message': 'Annotations file:',
                'filter': lambda val: os.path.relpath(val),
                'validate': AnnotationValidator
            },
            {
                'type': 'input',
                'name': self._DICTKEY_EXPERIMENT_FOLDER,
                'message': 'Experiment folder:',
                'default': os.path.dirname(os.path.abspath(sys.argv[0])),
                'filter': lambda val: os.path.abspath(val),
                'validate': FolderValidator
            }
        ]

        for k, v in prompt(questions).items():
            self.CONF_DICT[k] = v

    def ask_train_parameters(self):
        questions = [
            {
                'type': 'input',
                'name': self._DICTKEY_EXPERIMENT_FOLDER,
                'message': 'Experiment folder:',
                'default': os.path.dirname(os.path.abspath(sys.argv[0])),
                'filter': lambda val: os.path.abspath(val),
                'validate': FolderValidator
            },
            {
                'type': 'input',
                'name': self._DICTKEY_NN_OUTPUT_FUNCTION,
                'message': 'Neural network output function:',
                'default': self._NN_OUTPUT_FUNCTIONS[0],
                'validate': FunctionValidator
            },
            {
                'type': 'input',
                'name': self._DICTKEY_NN_MODULE,
                'message': 'Pytorch module (Neural network architecture):',
                'default': "dprom",
                'validate': ModuleValidator
            },
            {
                'type': 'input',
                'name': self._DICTKEY_NN_OPTIMIZER,
                'message': 'Pytorch optimizer for training:',
                'default': self._NN_OPTIMIZERS[0],
                'validate': OptimizerValidator
            },
            {
                'type': 'input',
                'name': self._DICTKEY_NN_MAX_EPOCHS,
                'message': 'Maximum number of epochs during training:',
                'default': self._maxepochs_default,
                'validate': PositiveValidator
            },
            {
                'type': 'input',
                'name': self._DICTKEY_NN_PATIENCE,
                'message': '(Patience)Stop after this many epochs without progress:',
                'default': self._patience_default,
                'validate': PositiveValidator
            },
            {
                'type': 'input',
                'name': self._DICTKEY_NN_LEARNING_RATE,
                'message': 'Learning rate:',
                'default': self._learningrate_default,
                'validate': PositiveValidator
            },
            {
                'type': 'input',
                'name': self._DICTKEY_NN_BATCH_SIZE,
                'message': 'Batch size:',
                'default': self._batchsize_default,
                'validate': PositiveValidator
            }
        ]
        train_args = {}
        self.CONF_DICT[self._DICTKEY_EXPERIMENT_FOLDER] = prompt(questions)[self._DICTKEY_EXPERIMENT_FOLDER]
        for k, v in prompt(questions).items():
            if k == self._DICTKEY_EXPERIMENT_FOLDER:
                continue
            train_args[k] = v
        return train_args

    def ask_dataset_config(self):
        questions = [
            {
                'type': 'input',
                'name': self._DICTKEY_SEQ_UPSTREAM_LEN,
                'message': 'Promoter upstream length:',
                'default': str(self._upstream_default),
                'validate': PositiveValidator,
                'filter': lambda val: int(val)
            },
            {
                'type': 'input',
                'name': self._DICTKEY_SEQ_DOWNSTREAM_LEN,
                'message': 'Promoter downstream length:',
                'default': str(self._downstream_default),
                'validate': PositiveValidator,
                'filter': lambda val: int(val)
            },
            {
                'type': 'input',
                'name': self._DICTKEY_STRIDE,
                'message': 'Stride:',
                'default': str(self._stride_default),
                'validate': PositiveValidator,
                'filter': lambda val: int(val)
            }
        ]
        self.CONF_DICT[self._DICTKEY_CONFIGURATION] = {}

        for k, v in prompt(questions).items():
            self.CONF_DICT[self._DICTKEY_CONFIGURATION][k] = v
        self.ask_seed()

    def ask_seed(self):
        question = [
            {
                'type': 'input',
                'name': self._DICTKEY_SEED,
                'message': 'Seed:',
                'default': '0',
                'validate': SeedValidator,
                'filter': lambda val: int(val)
            }
        ]
        self._set_seeds(prompt(question)[self._DICTKEY_SEED])

    def ask_dataset_type(self):
        questions = [
            {
                'type': 'list',
                'name': self._DICTKEY_DATASET_TYPE,
                'message': 'Choose the dataset type:',
                'choices': self._DATASET_CHOICES
            }
        ]

        for k, v in prompt(questions).items():
            self.CONF_DICT[k] = v

    def ask_error_type(self):
        if self.CONF_DICT[self._DICTKEY_DATASET_TYPE] == self._DATASET_CHOICES[0]:
            self.ask_io_error_type_config()
        elif self.CONF_DICT[self._DICTKEY_DATASET_TYPE] == self._DATASET_CHOICES[1]:
            #TODO
            self.ask_bio_error_type_config()

    def ask_io_error_type_config(self):
        questions = [
            {
                'type': 'list',
                'name': self._DICTKEY_ERROR_TYPE,
                'message': 'Select error type:',
                'choices': self._IO_ERROR_TYPES
            }
        ]
        answers = prompt(questions)

        for k, v in answers.items():
            self.CONF_DICT[self._DICTKEY_CONFIGURATION][k] = v

        if self.CONF_DICT[self._DICTKEY_CONFIGURATION][self._DICTKEY_ERROR_TYPE] == self._IO_ERROR_TYPES[0]: # overlap
            questions = [
                {
                    'type': 'input',
                    'name': 'upstream',
                    'message': 'Length of overlap from the 5\' end of promoter:',
                    'validate': PositiveValidator,
                    'filter': lambda val: int(val)
                },
                {
                    'type': 'input',
                    'name': 'downstream',
                    'message': 'Length of overlap from the 3\' end of promoter:',
                    'validate': PositiveValidator,
                    'filter': lambda val: int(val)
                }
            ]
            answers = prompt(questions)
            self.CONF_DICT[self._DICTKEY_CONFIGURATION][self._DICTKEY_ERROR_MARGIN] = [answers['upstream'], answers['downstream']]

        elif self.CONF_DICT[self._DICTKEY_CONFIGURATION][self._DICTKEY_ERROR_TYPE] == self._IO_ERROR_TYPES[1]: # tss proximity
            questions = [
                {
                    'type': 'input',
                    'name': 'proximity',
                    'message': 'Length of proximity to the TSS in promoter:',
                    'validate': PositiveValidator,
                    'filter': lambda val: int(val)
                }
            ]
            answers = prompt(questions)
            self.CONF_DICT[self._DICTKEY_CONFIGURATION][self._DICTKEY_ERROR_MARGIN] = [answers['proximity']]

        elif self.CONF_DICT[self._DICTKEY_CONFIGURATION][self._DICTKEY_ERROR_TYPE] == self._IO_ERROR_TYPES[2]: # tss proximity
            questions = [
                {
                    'type': 'input',
                    'name': 'proximity',
                    'message': 'Length of sequence proximity to the TSS:',
                    'validate': PositiveValidator,
                    'filter': lambda val: int(val)
                }
            ]
            answers = prompt(questions)
            self.CONF_DICT[self._DICTKEY_CONFIGURATION][self._DICTKEY_ERROR_MARGIN] = [answers['proximity']]

    def save_file(self, file, save_func, continuous=False, **kwargs):
        if continuous: # file must be buffer
            save_func(file, kwargs)
            return 0

        questions = [
            {
                'type': 'expand',
                'message': 'Conflict on `%s`: ' % file,
                'name': 'overwrite',
                'default': 'y',
                'choices': [
                    {
                        'key': 'y',
                        'name': 'Overwrite',
                        'value': 'overwrite'
                    },
                    {
                        'key': 'a',
                        'name': 'Overwrite this one and all next',
                        'value': 'overwrite_all'
                    },
                    {
                        'key': 'r',
                        'name': 'Rename the file',
                        'value': 'rename'
                    },
                    Separator(),
                    {
                        'key': 'x',
                        'name': 'Abort',
                        'value': 'abort'
                    }
                ]
            }
        ]

        if self.file_exists(file):
            if not self._overwrite:
                answers = prompt(questions)
            if answers['overwrite'] == 'overwrite_all':
                self._overwrite = True
            if answers['overwrite'] == 'overwrite' or self._overwrite:
                save_func(file, kwargs)
            elif answers['overwrite'] == 'rename':
                questions = [
                    {
                        'type': 'input',
                        'name': 'file',
                        'message': 'New file name:',
                        'validate': FileExistsValidator
                    }
                ]
                answers = prompt(questions)
                folder = os.path.dirname(os.path.abspath(file))
                save_func(os.path.join(folder, answers['file']), kwargs)
        else:
            save_func(file, kwargs)

    def _set_seeds(self, seed):
        if seed == 0:
            print("Seed configuration was set to 0 (This creates a random seed)")
            seed = random.randint(0, 2**32 - 1)
            print("Generating random seed: %d" % seed)
        self.CONF_DICT[self._DICTKEY_CONFIGURATION][self._DICTKEY_SEED] = seed
        random.seed(seed)
        np.random.seed(seed)
        torch.manual_seed(seed)
        torch.cuda.manual_seed(seed)
        torch.backends.cudnn.deterministic = True

    def get_modules(self):
        file_path = os.path.realpath(__file__)
        folder_path = os.path.dirname(file_path)
        modules_folder = os.path.join(folder_path, "modules")
        files = os.listdir(modules_folder)
        module_dict = {}
        for f in files:
            if f[-3:] == ".py" and not f.startswith("__"):
                f_name = f[:-3]
                f_path = os.path.join(modules_folder, f)
                spec = importlib.util.spec_from_file_location(f_name, f_path)
                module = importlib.util.module_from_spec(spec)
                spec.loader.exec_module(module)
                for x in dir(module):
                    cls = getattr(module, x)
                    if inspect.isclass(cls):
                        module_dict[f_name] = cls
        return module_dict

    @staticmethod
    def save_dataframe(file, kwargs):
        kwargs.get('dataframe').to_csv(file, sep='\t', index=False, header=False)

    @staticmethod
    def save_sld_line(buffer, kwargs):
        line = f"{kwargs.get('start')}\t{kwargs.get('end')}\t+{kwargs.get('pos_strand_label')}\t-{kwargs.get('neg_strand_label')}\n"
        buffer.write(line)

    @staticmethod
    def save_fasta_record(buffer, kwargs):
        buffer.write(f">{kwargs.get('record')}{kwargs.get('start')}:{kwargs.get('end')}={kwargs.get('lbl')}\n")
        record_data = kwargs.get('record_data')
        while(len(record_data) > 50):
            line = record_data[:50]
            record_data = record_data[50:]
            buffer.write(f"{line}\n")
        if len(record_data) > 1:
            buffer.write(f"{record_data}\n")

    @staticmethod
    def save_sldi_line(buffer, kwargs):
        line = f"{kwargs.get('record')}\t{kwargs.get('record_start')}\t{kwargs.get('record_end')}\n"
        buffer.write(line)

    @staticmethod
    def save_aliases_table(buffer, kwargs):
        for row in kwargs.get('aliases_table'):
            alias = row[0]
            chromosome = row[1]
            line = f"{alias}\t{chromosome}\n"
            buffer.write(line)

    @staticmethod
    def save_json(file, json_obj):
        with open(file, 'w') as json_file:
            json.dump(next(iter(json_obj.values())), json_file)

    @staticmethod
    def save_tsv(buffer, kwargs):
        line = f"{kwargs.get('seq')}\t{kwargs.get('label')}\n"
        buffer.write(line)

    @staticmethod
    def save_annotation(file, kwargs):
        resp = requests.get(kwargs.get('url'), stream=True)
        with tqdm.wrapattr(open(file, "wb"), "write", miniters=1,
                        total=int(resp.headers.get('content-length', 0)),
                        desc=file) as fout:
            for chunk in resp.iter_content(chunk_size=1024):
                fout.write(chunk)

    @staticmethod
    def save_header(buffer, kwargs):
        line = f"@{kwargs.get('filename')}\n"
        buffer.write(line)

    @staticmethod
    def save_record(buffer, kwargs):
        line = f">{kwargs.get('record')}\n"
        buffer.write(line)

    @staticmethod
    def file_exists(file):
        file_path = Path(file)
        return file_path.exists()

    conversion_table = str.maketrans("ACTGactgN", "TGACtgacN")
    def reverse_complement(self, seq):
        return seq.translate(self.conversion_table)[::-1]

    @staticmethod
    def get_number_of_lines(file):
        f = open(file)
        lines = 0
        buf_size = 1048576
        read_f = f.read
        buf = read_f(buf_size)
        while buf:
            lines += buf.count('\n')
            buf = read_f(buf_size)
        return lines

    def one_hot_encoder(self, seq):
        one_hot = np.zeros((len(self.DNA_DICT), len(seq)), dtype=np.float32)
        for i in range(0, len(seq), self._k):
            token = seq[i:i+self._k]
            one_hot[self.DNA_DICT[token], i] = 1
        return one_hot

    def download_epd_promoters(self, db='human', tata='all', save_path=''): #tata: all, with, without
        headers = {'accept': 'text/html',
                    'content-type': 'application/x-www-form-urlencoded',
                    'sec-ch-ua': '\" Not;A Brand\";v=\"99\", \"Chromium\";v=\"91\"',
                    'sec-ch-ua-mobile': '?0',
                    'sec-fetch-dest': 'document',
                    'sec-fetch-mode': 'navigate',
                    'sec-fetch-site': 'same-origin',
                    'sec-fetch-user': '?1',
                    'upgrade-insecure-requests': '1'}
        payload = {'select_db': db, 'idtype': 'epd', 'ids': '', 'tata': tata, 'inrAND': 'AND', 'inr': 'all', 'ccaatAND': 'AND', 'ccaat': 'all', 'gcAND': 'AND', 'gc': 'all', 'dispersionAND': 'AND', 'dispersion': 'all', 'eAverageAND': 'AND', 'eAverage': '', 'eSamplesAND': 'AND', 'eSamples': '', 'action': 'Select', 'database': 'epdnew'}
        base_url = "https://epd.epfl.ch"
        r = requests.post(base_url + "/get_promoters.php", data=payload, headers=headers)
        download_url = ""
        for line in r.text.split('\n'):
            if 'SGA file' in line:
                start = line.index('href=') + 6
                end = line.index('.sga') + 4
                download_url = base_url + line[start:end]
                break
        if not download_url.strip():
            raise Exception("Could not find download link")
        motifs_str = ""
        if tata == self._EPD_TATA_FILTERS[0]:
            motifs_str = "All motifs"
        elif tata == self._EPD_TATA_FILTERS[1]:
            motifs_str = "With TATA motifs (TATA)"
        else:
            motifs_str = "Without TATA motifs (non-TATA)"
        if not save_path:
            save_file = os.path.join(os.getcwd(), f"epd_{db}_tata-{tata}.sga")
        else:
            save_file = save_path
        print(f"Downloading EPD {db} promoters: {motifs_str}")
        self.save_file(save_file, Helper.save_annotation, url=download_url)

    def download_f5_promoters(self, db='human', save_path=''):
        base_url = "https://fantom.gsc.riken.jp/5"
        specie_genome_dict = {'human': 'hg38', 'mouse': 'mm10', 'rat': 'rn6', 'dog': 'canFam3',
                                'chicken': 'galGal5', 'rhesus': 'rheMac8'}
        download_url = ""
        if db in ['human', 'mouse']:
            download_url = base_url + f"/datafiles/reprocessed/{specie_genome_dict[db]}_latest/extra/CAGE_peaks/{specie_genome_dict[db]}_fair+new_CAGE_peaks_phase1and2.bed.gz"
        else:
            download_url = base_url + f"/datafiles/latest/extra/CAGE_peaks/{specie_genome_dict[db]}.cage_peak_coord.bed.gz"
        if not download_url.strip():
            raise Exception("Could not find download link")
        if not save_path:
            save_file = os.path.join(os.getcwd(), f"f5_{db}.bed.gz")
        else:
            save_file = os.path.abspath(save_path)
        print(f"Downloading FANTOM5 {db} promoters")
        self.save_file(save_file, Helper.save_annotation, url=download_url)


# test decision_tree_classification

import unittest
from numpy import float64
import pandas as pd
from unittest.mock import patch, Mock, MagicMock
from sklearn.tree import DecisionTreeClassifier
from new_decision_tree import Data
from new_decision_tree import read_file
from new_decision_tree import train_decision_tree
from new_decision_tree import test_decision_tree
import numpy as np

class TestDecisionTree(unittest.TestCase):

    test_size1 = 0.3
    random_state1 = 4
    max_depth1 = 4

    test_size2 = 0.6
    random_state2 = 4
    max_depth2 = 3

    def setUp(self):

        self.dataset = pd.DataFrame({"Age": [23, 47, 47], "Sex": ['F', 'M', 'M'], "BP": ['HIGH', 'LOW', 'LOW'],
                         "Cholesterol": ['HIGH', 'HIGH', 'HIGH'], "Na_to_K": [25.355, 13.093, 10.114], "Drug": ['drugY', 'drugC', 'drugC']})

    def tearDown(self):
        pass

##################################################

    def test_pandas_read_csv(self):
        """ Check the Read_file function(get csv file and give pd.DataFram)"""
        with patch("pandas.read_csv") as read_csv_mock:
            read_csv_mock.return_value = self.dataset
            my_data = read_file("file/path/file.csv")
            read_csv_mock.assert_called_with("file/path/file.csv", encoding="cp1252")
            self.assertTrue(my_data.equals(self.dataset))

    def test_Read_file_removing_NaN_value(self):
        """ Check that remove the rows including NAN value"""
        with patch("pandas.read_csv") as read_csv_mock:
            read_csv_mock.return_value = pd.DataFrame({"Age": [23, 47, 47], "Sex": ['F', np.nan, 'M'], "BP": ['HIGH', 'LOW', 'LOW'],
                         "Cholesterol": ['HIGH', 'HIGH', 'HIGH'], "Na_to_K": [25.355, 13.093, 10.114], "Drug": ['drugY', 'drugC', 'drugC']})
            data = read_file("file/path/file.csv")
            my_data = data.dropna()
            read_csv_mock.assert_called_with("file/path/file.csv", encoding="cp1252")
            my_data.equals(pd.DataFrame({"Age": [23, 47], "Sex": ['F', 'M'], "BP": ['HIGH', 'LOW'],
                         "Cholesterol": ['HIGH', 'HIGH'], "Na_to_K": [25.355, 10.114], "Drug": ['drugY', 'drugC']}))

##################################################

    def test_Data_init(self):
        """ Test input in __init__"""
        c = Data(self.dataset)
        self.assertTrue(c.dataset.equals(self.dataset))

#################################################

    def test_process_data_invalid_random_state(self):
        """ Test the value of random_state (random_state < 0)"""
        random_state3 = -1
        with self.assertRaises(ValueError) as ex:
            c = Data(self.dataset)
            c.process_data(self.test_size1, random_state3, self.max_depth1)
            self.assertEqual(str(ex.exception), "random_state must be between 0 and 2**32 - 1 and must be int")

    def test_process_data_invalid_test_size(self):
        """ Test the values of test size """
        test_size3 = 1.5
        test_size4 = 0
        with self.assertRaises(ValueError) as ex:
            c = Data(self.dataset)
            # test size >= 1
            c.process_data(test_size3, self.random_state1, self.max_depth1)
            self.assertEqual(str(ex.exception),"test_size should be either positive and float in the (0, 1) range")

        with self.assertRaises(ValueError) as exp:
            c = Data(self.dataset)
            # test size <= 0
            c.process_data(test_size4, self.random_state1, self.max_depth1)
            self.assertEqual(str(exp.exception), "test_size should be either positive and float in the (0, 1) range")

    def test_process_data_invalid_max_depth(self):
        """ Test the values of max_depth"""
        max_depth3 = 0
        max_depth4 = 6
        with self.assertRaises(ValueError) as ex:
            c = Data(self.dataset)
            # max_depth <= 0
            c.process_data(self.test_size1, self.random_state1, max_depth3)
            self.assertEqual(str(ex.exception), "max_depth must be positive and greater than zero")

        with self.assertRaises(ValueError) as exp:
            c = Data(self.dataset)
            # max_depth > length - 1 => (5)
            c.process_data(self.test_size1, self.random_state1, max_depth4)
            self.assertEqual(str(exp.exception), "max_depth should not be grater than the number of independent values")

##################################################

    def test_determine_inputData_outputData(self):
        "Check to divide columns correctly for inputData and outputData"
        c = Data(self.dataset)
        c.process_data(self.test_size1, self.random_state1, self.max_depth1)
        self.assertEqual(len(c.inputData.columns), 5)
        self.assertEqual(len(c.outputData.columns), 1)

##################################################

    def test_categorise_data_Sex_type(self):
        "Check Sex feature turn into numerical type"
        c = Data(self.dataset)
        c.process_data(self.test_size1, self.random_state1, self.max_depth1)
        c.categorise_data(self.dataset)
        x = str(self.dataset.Sex.dtype)
        if x == "int64":
               Sex_dtype = {x}
        elif x == "int32":
               Sex_dtype = {x}
        elif x == "int16":
             Sex_dtype = {x}
        else:
            pass
        myset = {"int64", "int32", "int16"}
        self.assertTrue(set(Sex_dtype).issubset(set(myset)))

    def test_categorise_data_BP_type(self):
        "Check BP feature turn into numerical type"
        c = Data(self.dataset)
        c.process_data(self.test_size1, self.random_state1, self.max_depth1)
        c.categorise_data(self.dataset)
        x = str(self.dataset.BP.dtype)
        if x == "int64":
               BP_dtype = {x}
        elif x == "int32":
               BP_dtype = {x}
        elif x == "int16":
             BP_dtype = {x}
        else:
            pass
        myset = {"int64", "int32", "int16"}
        self.assertTrue(set(BP_dtype).issubset(set(myset)))

    def test_categorise_data_Cholesterol_type(self):
        "Check Cholesterol feature turn into numerical type"
        c = Data(self.dataset)
        c.process_data(self.test_size1, self.random_state1, self.max_depth1)
        c.categorise_data(self.dataset)
        x = str(self.dataset.BP.dtype)
        if x == "int64":
               Cholesterol_dtype = {x}
        elif x == "int32":
               Cholesterol_dtype = {x}
        elif x == "int16":
             Cholesterol_dtype = {x}
        else:
            pass
        myset = {"int64", "int32", "int16"}
        self.assertTrue(set(Cholesterol_dtype).issubset(set(myset)))

    def test_categorise_data_value(self):
        """ Turn categorical features into numerical values """
        c = Data(self.dataset)
        c.process_data(self.test_size1, self.random_state1, self.max_depth1)
        c.inputData.equals(pd.DataFrame({"Age": [23, 47, 47], "Sex": [0, 1, 1], "BP": [0, 1, 1], "Cholesterol": [0, 0, 0], "Na_to_K": [25.355, 13.093, 10.114]}))

###############################################################

    def test_split_data_X_trainset(self):
         """ Check the number of rows for X_trainset depend on test_size"""
         c = Data(self.dataset)
         # Test_size(0.3)-> 0.3 for test and 0.7 for train
         c.process_data(self.test_size1, self.random_state1, self.max_depth1)
         self.assertEqual(len(c.X_trainset), 2)

         # Test_size(0.6)-> 0.6 for test and 0.4 for train
         c.process_data(self.test_size2, self.random_state2, self.max_depth2)
         self.assertEqual(len(c.X_trainset), 1)

    def test_split_data_Y_trainset(self):
         """ Check the number of rows for Y_trainset depend on test_size"""
         c = Data(self.dataset)
         # test_size(0.3)-> 0.3 for test and 0.7 for train
         c.process_data(self.test_size1, self.random_state1, self.max_depth1)
         self.assertEqual(len(c.X_trainset), 2)

         # test_size(0.6)-> 0.6 for test and 0.4 for train
         c.process_data(self.test_size2, self.random_state2, self.max_depth2)
         self.assertEqual(len(c.X_trainset), 1)

    def test_split_data_X_testset(self):
         """ Check the number of rows for X_testset depend on test_size"""
         c = Data(self.dataset)
         # test_size(0.3)-> 0.3 for test and 0.7 for train
         c.process_data(self.test_size1, self.random_state1, self.max_depth1)
         self.assertEqual(len(c.X_testset), 1)

         # test_size(0.6)-> 0.6 for test and 0.4 for train
         c.process_data(self.test_size2, self.random_state2, self.max_depth2)
         self.assertEqual(len(c.X_testset), 2)

    def test_split_data_Y_testset(self):
         """ Check the number of rows for X_testset depend on test_size"""
         c = Data(self.dataset)
         # test_size(0.3)-> 0.3 for test and 0.7 for train
         c.process_data(self.test_size1, self.random_state1, self.max_depth1)
         self.assertEqual(len(c.Y_testset), 1)

         # test_size(0.6)-> 0.6 for test and 0.4 for train
         c.process_data(self.test_size2, self.random_state2, self.max_depth2)
         self.assertEqual(len(c.Y_testset), 2)

###################################################################

    def test_train_decision_tree(self):
        "Check train_decision_tree's output"
        c = Data(self.dataset)
        c.process_data(self.test_size1, self.random_state1, self.max_depth1)

        with patch("new_decision_tree.DecisionTreeClassifier") as mock_decision_tree:
            a = train_decision_tree(c, self.max_depth1)

            mock_decision_tree.assert_called_with(
                criterion="entropy", max_depth=self.max_depth1
            )
            a.fit.assert_called_with(c.X_trainset, c.Y_trainset)

###################################################################

    def test_test_decision_tree_accuracy_by_one_predict(self):
        """" Check the accuracy"""
        c = Data(self.dataset)
        c.process_data(self.test_size1, self.random_state1, self.max_depth1)
        dt = train_decision_tree(c, self.max_depth1)
        dt = MagicMock()
        # With one Y_testset(because of test_size = 0.3) and false predict
        dt.predict.return_value = ['drugY']
        accuracy = test_decision_tree(c, dt)
        dt.predict.assert_called_with(c.X_testset)
        self.assertEqual(accuracy, 0.0)

        # With one Y_testset(because of test_size = 0.3) and true predict
        dt.predict.return_value = ['drugC']
        accuracy = test_decision_tree(c, dt)
        dt.predict.assert_called_with(c.X_testset)
        self.assertEqual(accuracy, 1.0)

    def test_test_decision_tree_accuracy_by_two_predict(self):
        """" Check the accuracy """
        c = Data(self.dataset)
        c.process_data(self.test_size2, self.random_state2, self.max_depth2)
        dt = train_decision_tree(c, self.max_depth2)
        dt = MagicMock()
        # With two Y_testset(because of test_size = 0.6), one false predict and one true predict
        dt.predict.return_value = ['drugC', 'drugC']
        accuracy = test_decision_tree(c, dt)
        dt.predict.assert_called_with(c.X_testset)
        self.assertEqual(accuracy, 0.5)

if __name__ == '__main__':
    unittest.main()


# protonateMyPDB.py
#
# This example script takes the 3gb1_testing.pdb pdbfile as input, and outputs
# a PDB file with the protonation state at a desired pH calculated by MCCE.
#
# In this case, the PDB used for testing is the first 13 residues of 3gb1, the NMR structure
# of the B1 domain from the staphloccocus IgG-binding protein G.
#
# The parameters used in the MCCE run can be found in the prmfile specified below
#
# Vincent Voelz
# May 19 2007
#

import mmtools.mccetools.mcce as mcce
import os, sys

### INPUT LINES ####
#
# Specify the input and output PDB filenames
# NOTE: pdbfile and outpdbfile should be local (not absolute) paths for this project
pdbfile = '3gb1_testing.pdb'
outpdbfile = os.path.join(os.curdir,'3gb1_testing_protonated.pdb')

# Specify the pH
pH = 7.0

# Specify a MCCE parameter file with the desired set of parameters for calculating the pKa
prmfile = '../prmfiles/run.prm.quick'
prmfile = os.path.abspath(prmfile)

# Specify additional or different parameters than prmfile,  if desired.
# xtraprms = {'TITR_PH0':'3.0'}

# Perform titration.
### work is done in a temporary dir; Setting cleanup=True will erase these temporary files
mcce.protonatePDB(pdbfile, outpdbfile, pH, os.environ['MCCE_LOCATION'], cleanup=False, prmfile=prmfile)