#!/usr/bin/env python3

"""This script has the function of linking all the reads inside the sample sheet,
and to create the sample objects to be put inside the configuration."""

from ..abstract import ShortSample, LongSample
import os
import csv
import re

def parse_samplesheet(samplesheet, configuration):

    """This function will parse the samplesheet, create the sample objects, and put them in the configuration itself."""

    configuration["long_reads"] = dict()
    configuration["short_reads"] = dict()
    outdir = os.path.join(configuration["outdir"], "inputs", "reads")

    if not os.path.exists(outdir):
        os.makedirs(outdir)
    elif os.path.exists(outdir) and not os.path.isdir(outdir):
        raise OSError("Read directory is not a directory at all: {}".format(outdir))

    if samplesheet is None:
        return configuration

    with open(samplesheet) as sheet:
        for line in csv.reader(sheet, delimiter="\t"):
            if line[0].lstrip().startswith("#"):
                continue  # Ignore comments!
            label, read1, read2, type, strandedness = line
            label = re.sub("\s", "_", label)  # Remove spaces!
            if type in ("illumina", "short"):
                sample = ShortSample(read1, read2, label, outdir, strandedness=strandedness)
                tag = "short_reads"
            else:
                sample = LongSample(read1, label, outdir, strandedness, type)
                tag = "long_reads"
            if label in configuration[tag]:  # Double label!
                raise KeyError(
                    "{short_tag} read label {label} was found at least twice in the sample sheet. Please recheck it.".format(
                        short_tag=tag.split("_")[0].capitalize(),
                        **locals()))
            configuration[tag][label] = sample

    return configuration


import cv2
import pathlib

# tutor: https://www.youtube.com/watch?v=5cg_yggtkso

print("Face detection with OpenCV")
cascade_path = pathlib.Path(cv2.__file__).parent.absolute() / "data/haarcascade_frontalface_default.xml"
print("Classifier file path: " + str(cascade_path))

clf = cv2.CascadeClassifier(str(cascade_path))

camera = cv2.VideoCapture(0)
# camera = cv2.VideoCapture('video.mp4')

while True:
  _, frame = camera.read()
  gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
  faces = clf.detectMultiScale(
    gray,
    scaleFactor = 1.1,
    minNeighbors = 5,
    minSize = (100, 100),
    flags = cv2.CASCADE_SCALE_IMAGE
  )

  for (x, y, w, h) in faces:
    cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 0, 255), 2)

  cv2.imshow('frame', frame)

  if cv2.waitKey(1) & 0xFF == ord('q'):
    break

camera.release()
cv2.destroyAllWindows()


Test_case = int(input())

for _ in range(Test_case):
    stack = []
    VPS = input()
    for index in range(len(VPS)):
        if VPS[index] == '(':
            stack.append(index)

        elif VPS[index] == ')':
            try:
                stack.pop()
            except:
                print('NO')
                break
        if index == len(VPS)-1:
            if len(stack):
                print('NO')
            else:
                print('YES')

1.자료구조/괄호.py


from rest_framework.fields import SerializerMethodField
from rest_framework.relations import HyperlinkedIdentityField
from rest_framework.serializers import ModelSerializer
from account.serializers import UserOtherDetailsSerializer
from answers.models import Answer
from comments.models import Comment

class CommentDetailsSerializer(ModelSerializer):
    writer = UserOtherDetailsSerializer(read_only=True)
    parent_answer = SerializerMethodField()
    replies = HyperlinkedIdentityField(view_name='children_comment')
    replies_count = SerializerMethodField()

    class Meta:
        model = Comment
        fields = [
            'id',
            'writer',
            'body',
            'no_of_upvotes',
            'timestamp',
            'replies',
            'replies_count',
            'parent_answer',
        ]
        read_only_fields = [
            'no_of_upvotes',
        ]

    def get_replies(self, obj):
        if obj.is_parent:
            return CommentChildSerializer(obj.children(), many=True).data
        return None

    def get_replies_count(self, obj):
        if obj.is_parent:
            return obj.children().count()
        return 0

    def get_parent_answer(self, obj):
        if obj.parent_answer is not None:
            return CommentParentAnswerSerializer(obj.parent_answer).data
        return None

class CommentChildSerializer(ModelSerializer):
    writer = UserOtherDetailsSerializer(read_only=True)
    replies = HyperlinkedIdentityField(view_name='children_comment')
    replies_count = SerializerMethodField()

    class Meta:
        model = Comment
        fields = [
            'id',
            'writer',
            'body',
            'no_of_upvotes',
            'parent',
            'replies',
            'replies_count'
        ]

        read_only_fields = [
            'no_of_upvotes'
        ]

    def get_replies(self, obj):
        if obj.is_parent:
            return CommentChildSerializer(obj.children(), many=True).data
        return None

    def get_replies_count(self, obj):
        try:
            return obj.children().count()
        except:
            return 0

class CommentParentAnswerSerializer(ModelSerializer):
    writer = UserOtherDetailsSerializer(read_only=True)

    class Meta:
        model = Answer
        fields = [
            'writer',
            'body'
        ]


import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from mlxtend.frequent_patterns import apriori
from mlxtend.frequent_patterns import association_rules

#Loading the dataset
orders= pd.read_csv('Instacart//orders.csv')
orderproducts= pd.read_csv('Instacart//order_products__train.csv')
products= pd.read_csv('Instacart//products.csv')
aisles= pd.read_csv('Instacart//aisles.csv')
departments= pd.read_csv('Instacart//departments.csv')

#Displays the column of orders dataset
orders.columns
#Displays the column of orders_products dataset
orderproducts.columns
#Displays the column of products dataset
products.columns
#Displays the column of aisles dataset
aisles.columns
#Displays the column of departments dataset
departments.columns

#Merges Orders dataset with orderproducts using order_id column
data = orders.merge(orderproducts, how='inner', on='order_id')
data.shape
#Merges new dataset dataset with products using product id
data = data.merge(products,how='inner', on='product_id')
data.shape
#Merges new dataset with aisles usimg aisle id
data = data.merge(aisles, how='inner', on='aisle_id')
data.shape
#Merges new dataset with departments using departent id
data = data.merge(departments, how='inner', on='department_id')
#Shape and First Five rows of merged dataframe
data.shape
data.head()

#Null Values
data.isnull().sum()

#grouping the dataset based on product id and  product name and aggregates with count
product_freq=data.groupby(['product_id','product_name']).agg({'product_name':'count'})
#Displays the number of time each product is ordered
product_freq
#Grouping the data by department id and counts the orderid in each department
dept_order=data.groupby('department_id').agg({'order_id':'count'})
dept_order.head(5)

#bar plot
plt.figure(figsize=(18,8))
sns.barplot(dept_order.index,dept_order.order_id);
plt.xlabel("Department ID")
plt.ylabel('Number of Orders')
plt.show();

#Groups the data by order day and counts the order id
weekday=data.groupby('order_dow').agg({'order_id':'count'})

#bar plot
plt.figure(figsize=(15,8))
sns.barplot(weekday.index,weekday.order_id);
plt.xlabel('Day of Week')
plt.ylabel('Number of Orders')
plt.show()

#Groups the data by order hour and counts the order id
hour_order=data.groupby('order_hour_of_day').agg({'order_id':'count'})

#bar plot
plt.figure(figsize=(15,8))
sns.barplot(hour_order.index,hour_order.order_id);
plt.xlabel('Hour')
plt.ylabel('Number of Orders')
plt.show()

order_reorder = data.groupby('reordered')['reordered'].aggregate('count')
order_reorder

plt.pie(order_reorder, labels=order_reorder.index,autopct='%1.1f%%', startangle=90);
plt.title('Reordered vs Not-Reordered');

hmdata=heatmapp.pivot('order_dow', 'order_hour_of_day', 'reordered')
plt.figure(figsize=(18,8))
sns.heatmap(hmdata);
plt.title('Order Day vs Order Hour vs Reorder')
plt.xlabel('Hour of the day')
plt.ylabel('Day of the week');

#Creating Basket
order_freq = data.drop(['user_id', 'eval_set', 'order_dow', 'order_hour_of_day', 'days_since_prior_order', 'add_to_cart_order', 'reordered','aisle_id', 'department_id', 'aisle', 'department'], axis=1)
mostfreq_products = order_freq.groupby('product_name').agg({'product_id':'count'}).sort_values(by='product_id', ascending=False).head(100).reset_index()
mostfreq_dataset = data[data['product_name'].isin(mostfreq_products['product_name'])]
basket = mostfreq_dataset.pivot_table(index='order_id',columns='product_name',values='reordered', aggfunc=np.sum, fill_value=0)
basket

def hot_encode(x):
    if(x<= 0):
        return 0
    if(x>= 1):
        return 1

basket = basket.applymap(hot_encode)
basket

freq_items = apriori(basket, min_support = 0.1, use_colnames = True)
freq_items
mba_rules = association_rules(freq_items, metric ="lift", min_threshold = 1.0)
mba_rules = rules.sort_values(['confidence', 'lift'], ascending =[False, False])


import keras
from keras.models import Sequential
from keras.preprocessing.image import ImageDataGenerator
from keras.datasets import cifar10
from keras.regularizers import l2
from keras.layers import Dense, Dropout, Activation, BatchNormalization, Conv2D, MaxPooling2D, Flatten, AveragePooling2D
from keras.initializers import he_normal
from keras.utils import plot_model
from keras import backend as K
#from utils import save_obj, load_obj

import numpy as np

import matplotlib.pyplot as plt
plt.ion()

import tensorflow as tf
# Set Memory allocation in tf/keras to Growth
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
K.set_session(sess)

# Plot callback
class Plot_Callback(keras.callbacks.Callback):
    def on_train_begin(self, logs={}):
        self.losses = []
        self.accuracies = []
        self.val_losses = []
        self.val_accuracies = []
        return

    def on_train_end(self, logs={}):
        plt.ioff()
        plt.show()

    def on_epoch_end(self, epoch, logs={}):
        self.losses.append(logs.get('loss'))
        self.accuracies.append(logs.get('acc'))
        self.val_losses.append(logs.get('val_loss'))
        self.val_accuracies.append(logs.get('val_acc'))

        ax = plt.gca()
        ax.clear()
        ax.plot(range(1,len(self.losses)+1),self.losses,color='r',label='train_loss')
        ax.plot(range(1,len(self.losses)+1),self.val_losses,color='b',label='val_loss')
        ax.set_ylim([0,1])
        ax.legend(loc=2)
        ax.set_xlabel('Epoch')
        ax.set_ylabel('Loss')
        plt.draw()
        plt.pause(0.01)
        return

if __name__ == "__main__":
    # Params
    use_datagen = True
    batch_size = 128
    epochs = 20

    # Load Data
    (x_train, y_train), (x_test, y_test) = cifar10.load_data()
    num_classes = int(max(y_train)) + 1
    x_train = x_train.astype('float32')/255
    x_test = x_test.astype('float32')/255
    y_train = keras.utils.to_categorical(y_train, num_classes)
    y_test = keras.utils.to_categorical(y_test, num_classes)

    model = Sequential()

    model.add(Conv2D(32, (3, 3), padding='same', use_bias=False, input_shape=x_train.shape[1:]))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Conv2D(32, (3, 3), padding='same', use_bias=False))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    #model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

    model.add(Conv2D(64, (3, 3), strides=(2,2), padding='same', use_bias=False))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Conv2D(64, (3, 3), padding='same', use_bias=False))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    #model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

    model.add(Conv2D(128, (3, 3), strides=(2,2), padding='same', use_bias=False))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Conv2D(128, (3, 3), padding='same', use_bias=False))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

#==============================================================================
#     model.add(AveragePooling2D(pool_size=(8,8), strides=(1,1)))
#==============================================================================

    model.add(Flatten())

    model.add(Dense(512, use_bias=False))
    model.add(Activation('relu'))
    model.add(BatchNormalization())
    model.add(Dropout(0.5))

    model.add(Dense(num_classes, use_bias=False))
    model.add(BatchNormalization())
    model.add(Activation('softmax'))

#==============================================================================
#     model = Sequential()
#
#     model.add(Conv2D(32, (3, 3), padding='same', use_bias=False, input_shape=x_train.shape[1:]))
#     model.add(BatchNormalization())
#     model.add(Activation('relu'))
#     model.add(Conv2D(32, (3, 3), padding='same', use_bias=False))
#     model.add(BatchNormalization())
#     model.add(Activation('relu'))
#     model.add(MaxPooling2D(pool_size=(2, 2)))
#     model.add(Dropout(0.25))
#
#     model.add(Conv2D(64, (3, 3), padding='same', use_bias=False))
#     model.add(BatchNormalization())
#     model.add(Activation('relu'))
#     model.add(Conv2D(64, (3, 3), padding='same', use_bias=False))
#     model.add(BatchNormalization())
#     model.add(Activation('relu'))
#     model.add(MaxPooling2D(pool_size=(2, 2)))
#     model.add(Dropout(0.25))
#
#     model.add(Conv2D(128, (3, 3), padding='same', use_bias=False))
#     model.add(BatchNormalization())
#     model.add(Activation('relu'))
#     model.add(Conv2D(128, (3, 3), padding='same', use_bias=False))
#     model.add(BatchNormalization())
#     model.add(Activation('relu'))
#     model.add(MaxPooling2D(pool_size=(2, 2)))
#     model.add(Dropout(0.25))
#
#     model.add(AveragePooling2D(pool_size=(4,4), strides=(1,1)))
#
#     model.add(Flatten())
#
# #==============================================================================
# #     model.add(Dense(512, use_bias=False))
# #     model.add(Activation('relu'))
# #     model.add(BatchNormalization())
# #     model.add(Dropout(0.5))
# #==============================================================================
#
#     model.add(Dense(num_classes, use_bias=False))
#     model.add(BatchNormalization())
#     model.add(Activation('softmax'))
#==============================================================================

    # Compile model with optimizer
    #opt = keras.optimizers.Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
    #opt = keras.optimizers.SGD(lr=0.1, momentum=0.9, nesterov=True)
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

    #Print/Plot Model
    model.summary()
    plot_model(model, show_shapes=True, show_layer_names=False)

    # Fit
    plot = Plot_Callback()
    if use_datagen:
        datagen = ImageDataGenerator(
               width_shift_range=0.1,
               height_shift_range=0.1,
               #zoom_range=0.2,
               horizontal_flip=True)
        datagen.fit(x_train)
        model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size), steps_per_epoch=x_train.shape[0] // batch_size, epochs=epochs, validation_data=(x_test, y_test), callbacks=[plot])
    else:
        model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_data=(x_test, y_test), shuffle=True, callbacks=[plot])


import json
import yaml
import os
import shutil

import deepspeed
import torch
from accelerate import Accelerator
from beartype import beartype
from beartype.typing import Tuple
from einops import rearrange
from peft import get_peft_model, LoraConfig, TaskType
from torch.utils.data import DataLoader, Dataset
from transformers import (
    AutoModelForCausalLM,
    AutoTokenizer,
)

from chatllama.rlhf.config import ConfigActor
from chatllama.rlhf.model_list import (
    hf_models_causal_lm,
    llama_models,
    hf_models,
)

from chatllama.rlhf.model_loader import ModelLoader
from chatllama.rlhf.utils import TrainingStats

class ActorModel(torch.nn.Module):
    """Actor model that generates the augmented prompt from the initial
    user_input. The aim is to train this model to generate better prompts.

    Attributes:
        model: The model from LLaMA to be used
        tokenizer: The LLaMA tokenizer
        config (ConfigActor): Configuration for the actor model

    Methods:
        load: Load the model from a path
        save: Save the model to a path
        forward: Compute the action logits for a given sequence.
        generate: Generate a sequence from a given prompt
    """

    def __init__(self, config: ConfigActor) -> None:
        super().__init__()

        # save config
        self.config = config

        # initialize the self.model
        if config.model in llama_models:
            # llama module might not be present when HF models are used
            from chatllama.llama_model import (
                load_model,
                setup_model_parallel,
            )  # noqa

            local_rank, world_size = setup_model_parallel()

            # use load_model_test for testing
            self.model, self.tokenizer = load_model(
                ckpt_dir=config.model_folder,
                tokenizer_path=config.tokenizer_path,
                local_rank=local_rank,
                world_size=world_size,
                froze_embeddings=config.froze_embeddings,
                use_fairscale=config.use_fairscale,
                max_batch_size=config.batch_size,
            )
        elif config.model in hf_models_causal_lm:
            self.tokenizer = self.load_tokenizer(config)
            self.model = AutoModelForCausalLM.from_pretrained(
                config.model,
            )

            # Setup PEFT model
            if config.peft_enable:

                # check that the peft config exist
                if os.path.exists(config.peft_config_path):
                    # Read the peft config from yaml
                    with open(config.peft_config_path, "r") as c:
                        config_peft = yaml.safe_load(c)
                else:
                    raise ValueError(
                        f"PEFT config {config.peft_config_path} not found"
                    )

                print(config_peft)
                # define lora config for peft
                peft_config = LoraConfig(
                    task_type=TaskType.CAUSAL_LM, **config_peft
                )

                # create peft model
                self.model = get_peft_model(
                    model=self.model,
                    peft_config=peft_config,
                )

            self.model.to(config.device)

        else:
            raise ValueError(f"Model {config.model} not supported")

        # load the model from model_folder
        self.load()

    @beartype
    def load(self) -> None:
        """Load the model from the path"""
        # check if there is a model to load
        path = ModelLoader.check_model_path(
            config=self.config,
            is_checkpoint=False,
            current_epoch=None,
        )

        # if there is a model to load
        if path is not None:

            # load the model
            print("Loading ...")
            model_dict = torch.load(path)
            self.model.load_state_dict(model_dict.get("state_dict") or model_dict.get("model"))

    @beartype
    def save(self) -> None:
        """Save the model to the path"""
        # get the path to save the model
        model_folder, model_name, path = ModelLoader.get_model_path(
            config=self.config,
            is_checkpoint=False,
            current_epoch=None,
        )

        # save the model
        print(f"Saving model to {path} ...")
        torch.save(
            {"state_dict": self.model.state_dict()},
            path,
        )

    @staticmethod
    def load_tokenizer(config: ConfigActor):
        """Load the tokenizer from the model name"""
        if config.model in hf_models:
            # load the tokenizer from HF
            tokenizer = AutoTokenizer.from_pretrained(
                config.model,
                padding_side="left",
                padding=True,
                truncation=True,
                model_max_length=config.max_sequence_length,
            )

            # add eos token if not present
            if tokenizer.eos_token is None:
                tokenizer.eos_token = "</s>"
                tokenizer.eos_token_id = 2  # OPT eos-token-id

            # add pad token if not present
            if tokenizer.pad_token is None:
                tokenizer.pad_token = tokenizer.eos_token
                tokenizer.pad_token_id = tokenizer.eos_token_id
        elif config.model in llama_models:

            # llama module might not be present when HF models are used
            from chatllama.llama_model import (
                load_tokenizer,
            )  # noqa

            tokenizer = load_tokenizer(config.tokenizer_path)
        return tokenizer

    def parameters(self):
        """Return the parameters of the model"""
        return self.model.parameters()

    @beartype
    def forward(
        self, sequences: torch.Tensor, sequences_mask: torch.Tensor
    ) -> torch.Tensor:
        """Generate logits to have probability distribution over the vocabulary
            of the actions

        Args:
            sequences (torch.Tensor): Sequences of states and actions used to
                    compute token logits for the whole list of sequences
            attention_mask (torch.Tensor): Mask for the sequences attention

        Returns:
            logits (torch.Tensor): Logits for the actions taken
        """
        model_output = self.model.forward(
            sequences, attention_mask=sequences_mask
        )
        # need to return logits for the actions
        if self.config.model in hf_models_causal_lm:
            model_output = model_output.logits
        if self.config.debug:
            print("ActorModel.forward")
            print("model_output_logits shape", model_output.shape)
            print("model_output logits", model_output)
        return model_output

    @beartype
    @torch.no_grad()
    def generate(
        self, states: torch.Tensor, state_mask: torch.Tensor
    ) -> Tuple:
        """Generate actions and sequences=[states, actions] from state
            (i.e. input of the prompt generator model)

        Args:
            state (torch.Tensor): the input of the user
            state_mask (torch.Tensor): Mask for the state input (for padding)

        Returns:
            actions (torch.Tensor): Actions generated from the state
            sequences (torch.Tensor): Sequences generated from the
                state as [states, actions]
        """
        # temperature for the actor
        temperature = self.config.temperature

        # max sequence length for the actor (i.e. prompt + completion)
        max_sequence_length = self.config.max_sequence_length

        # max and min number of tokens to generate
        max_tokens = self.config.max_tokens
        min_tokens = self.config.min_tokens

        # max generation possible given the state and the max sequence length
        max_generation_possible = max_sequence_length - states.shape[1]
        if max_generation_possible < min_tokens:
            raise ValueError(
                f"The prompt is too long w.r.t the "
                f"model sequence length \n"
                f"max_sequence_length={max_sequence_length}\n"
                f"state_length={states.shape[1]}\n"
                f"min_tokens={min_tokens}\n"
                f"max_tokens={max_tokens}\n"
                f"max_generation_possible={max_generation_possible}\n"
            )

        # take the minimum the max_tokens and the max_generation_possible
        max_completion = min(max_tokens, max_generation_possible)

        sequences = self.model.generate(
            input_ids=states,
            attention_mask=state_mask,
            temperature=temperature,
            max_new_tokens=max_completion,
            no_repeat_ngram_size=3,
        )
        actions = sequences[:, states.shape[1] :]  # noqa E203
        if self.config.debug:
            print(
                f"input length {states.shape[1]} \n"
                f"max sequence length {max_sequence_length} \n"
                f"max completion {max_completion} \n"
                f"generated sequence {sequences.shape[1]} \n"
            )
            print("ActorModel.generate")
            print("state", states)
            print("state shape", states.shape)
            print("sequence shape", sequences.shape)
            print("sequence", sequences)
            print("actions shape", actions.shape)
            print("actions", actions)
        return actions, sequences

class ActorDataset(Dataset):
    """Dataset for the pretraining of the actor model
    read a json file with the following format:
    [
        {
            "user_input": "..."
            "completion": "..."
        },
        ...
    ]
    Where:
        user_input: the input of the user
        completion: the output of the user
    """

    def __init__(
        self,
        path: str,
    ) -> None:
        self.path = path
        with open(path, "r") as f:
            data = json.load(f)
        self.data = [d["user_input"] + d["completion"] for d in data]

    def __getitem__(self, idx):
        return self.data[idx]

    def __len__(
        self,
    ):
        return len(self.data)

class ActorTrainer:
    """Used to pre-train the actor model to generate better prompts.

    Args:
        config (ConfigActor): Configuration for the actor model

    Attributes:
        config (ConfigActor): Configuration for the actor model
        model (ActorModel): Actor model
        loss_function (torch.nn.CrossEntropyLoss): Loss function
        optimizer (torch.optim.Adam): Optimizer
        validation_flag (bool): Flag to indicate if the validation dataset
            is provided
        train_dataset (ActorDataset): Training dataset
        train_dataloader (DataLoader): Training dataloader
        validation_dataset (ActorDataset): Validation dataset
        validation_dataloader (DataLoader): Validation dataloader
        scheduler (torch.optim.lr_scheduler): Learning rate scheduler
        training_stats (TrainingStats): Training statistics
        model_engine (ModelEngine): Model engine for deepspeed training
        accelerator (Accelerator): Accelerator for accelerate training

    Methods:
        train: Train the actor model
        load_checkpoint: Load a checkpoint
        save_checkpoint: Save a checkpoint
    """

    def __init__(self, config: ConfigActor) -> None:

        # store config
        self.config = config

        # load the model
        self.actor = ActorModel(config)

        # define loss function
        self.loss_function = torch.nn.CrossEntropyLoss()

        # define optimizer
        self.optimizer = torch.optim.AdamW(
            self.actor.parameters(), lr=config.lr, weight_decay=1e-5
        )

        # check if validation dataset is provided
        self.validation_flag = False
        if config.validation_dataset_path is not None:
            self.validation_flag = True

        # create dataset and dataloaders
        self.train_dataset = ActorDataset(config.train_dataset_path)
        self.train_dataloader = DataLoader(
            self.train_dataset, batch_size=config.batch_size
        )
        if self.validation_flag:
            self.eval_dataset = ActorDataset(config.validation_dataset_path)
            self.validation_dataloader = DataLoader(
                self.eval_dataset, batch_size=config.batch_size
            )

        # define scheduler for the learning rate
        # learning rate is decreased until 10% of the initial value
        self.scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(
            self.optimizer,
            T_0=len(self.train_dataset) // config.batch_size,
            T_mult=1,
            eta_min=config.lr * 0.1,
        )

        # define training statistics
        stat_path = ModelLoader.get_training_stats_path(config)
        self.training_stats = TrainingStats(stat_path)

        # consistency check between accelerate and deepspeed
        if config.accelerate_enable and config.deepspeed_enable:
            raise ValueError(
                "Both DeepSpeed and Accelerate are enabled for the Actor."
                "Please choose one of them."
            )

        # initialize deepspeed
        self.model_engine = None
        if config.deepspeed_enable is True:
            if config.deepspeed_config_path is None:
                raise ValueError(
                    "DeepSpeed config path is None, but deepspeed is enabled"
                )
            if os.path.exists(config.deepspeed_config_path) is False:
                raise ValueError(
                    f"DeepSpeed config path {config.deepspeed_config_path}"
                    f"does not exist"
                )
            (
                self.model_engine,
                self.optimizer,
                self.train_dataloader,
                _,
            ) = deepspeed.initialize(
                args=None,
                model=self.actor,
                model_parameters=self.actor.parameters(),
                training_data=self.train_dataset,
                config=self.config.deepspeed_config_path,
            )
            print("Training with DeepSpeed")

        # initialize accelerate
        self.accelerator = None
        if config.accelerate_enable is True:
            self.accelerator = Accelerator()
            (
                self.actor,
                self.optimizer,
                self.train_dataloader,
                self.scheduler,
            ) = self.accelerator.prepare(
                self.actor,
                self.optimizer,
                self.train_dataloader,
                self.scheduler,
            )
            print("Training with Accelerate")

    @beartype
    def save_checkpoint(
        self,
        current_epoch: int,
        current_step: int,
        max_epochs: int,
        max_steps: int,
    ) -> None:
        """Save the current checkpoint

        Args:
            current_epoch (int): Current epoch
            current_step (int): Current step
            max_epochs (int): Maximum number of epochs
            max_steps (int): Maximum number of steps
        """

        print(
            f"Saving checkpoint for epoch {current_epoch + 1}, "
            f"step {current_step + 1} ..."
        )
        # look for path to save the checkpoint
        model_folder, model_name, path = ModelLoader.get_model_path(
            config=self.config,
            is_checkpoint=True,
            current_epoch=current_epoch,
            current_step=current_step,
            max_epochs=max_epochs,
            max_steps=max_steps,
        )

        # remove the checkpoint if it already exists
        if os.path.exists(path):
            if self.config.deepspeed_enable:
                shutil.rmtree(path)
            else:
                os.remove(path)

        if self.config.deepspeed_enable:
            client_state = {
                "epoch": current_epoch,
                "step": current_step,
            }
            self.model_engine.save_checkpoint(path, client_state=client_state)
        else:
            # save the checkpoint
            torch.save(
                {
                    "state_dict": self.actor.model.state_dict(),
                    "optim_state_dict": self.optimizer.state_dict(),
                    "training_stats": self.training_stats,
                    "epoch": current_epoch,
                    "step": current_step,
                },
                path,
            )

        # remove old checkpoints
        n_checkpoints_to_keep = self.config.n_checkpoints_to_keep
        ModelLoader.delete_old_checkpoints(
            model_folder, model_name, n_checkpoints_to_keep
        )

    @beartype
    def load_checkpoint(
        self,
    ) -> Tuple[int, int]:
        """Load a checkpoint from the model folder

        Returns:
            Tuple[int, int]: Current epoch and current step to resume
                training
        """

        print("Looking for checkpoints...")
        # look for a checkpoint
        path = ModelLoader.check_model_path(
            config=self.config,
            is_checkpoint=True,
            current_epoch=None,
        )

        # if there is a checkpoint
        if path is not None:
            print("Loading ...")

            if self.config.deepspeed_enable:
                # try to load the checkpoint
                try:
                    _, client_state = self.model_engine.load_checkpoint(path)
                except Exception:
                    print(
                        "Checkpoint corrupted!"
                        "Try to remove the last checkpoint."
                        "Now Starting from epoch 0, step 0"
                    )
                    return 0, 0
                # load epoch and step to resume loops
                epoch = client_state["epoch"]
                step = client_state["step"]
            else:
                # try to load the checkpoint
                try:
                    checkpoint = torch.load(path)
                except Exception:
                    print(
                        "Checkpoint corrupted!"
                        "Try to remove the last checkpoint."
                        "Now Starting from epoch 0, step 0"
                    )
                    return 0, 0

                # assing the checkpoint to the model
                epoch = checkpoint["epoch"]
                self.actor.model.load_state_dict(checkpoint["state_dict"])
                self.optimizer.load_state_dict(checkpoint["optim_state_dict"])
                self.trainign_stats = checkpoint["training_stats"]
                step = checkpoint["step"]
                return epoch, step + 1  # return the next episode to train
        return 0, 0

    def add_eos_token(
        self, tokens: torch.Tensor, mask: torch.Tensor
    ) -> Tuple[torch.Tensor, torch.Tensor]:
        # given tokens and mask, add eos token to the end of each sequence
        # and update the mask
        batch_size, seq_len = tokens.shape
        eos_token = self.actor.tokenizer.eos_token_id

        # see if i can append 1 token
        n_tokens_to_append = min(self.config.max_sequence_length - seq_len, 1)
        n_tokens_to_append = max(n_tokens_to_append, 0)

        # concatenate eos to tokens and mask
        if n_tokens_to_append > 0:
            tokens = torch.cat(
                [
                    tokens,
                    torch.ones(batch_size, n_tokens_to_append)
                    .long()
                    .to(tokens.device)
                    * eos_token,
                ],
                dim=1,
            )
            mask = torch.cat(
                [
                    mask,
                    torch.ones(batch_size, n_tokens_to_append)
                    .long()
                    .to(mask.device),
                ],
                dim=1,
            )
        return tokens, mask

    def train(
        self,
    ) -> None:
        """Train the model"""
        print("Start Actor Model Pretraining")

        # get config parameters
        if self.config.deepspeed_enable:
            batch_size = self.train_dataloader.batch_size
        else:
            batch_size = self.config.batch_size
        epochs = self.config.epochs
        device = self.config.device
        checkpoint_steps = self.config.checkpoint_steps

        # compute the number of iterations
        n_iter = int(len(self.train_dataset) / batch_size)

        # load model_checkpoint
        start_epoch, start_step = self.load_checkpoint()

        if start_epoch == 0 and start_step == 0:
            self.training_stats.clear()

        # counter for the checkpoint
        cnt_checkpoint = 1

        # traing loop
        for epoch in range(start_epoch, epochs):
            self.actor.train()
            for i, input_text in enumerate(self.train_dataloader):

                # skip the first steps if we are resuming training
                if i < start_step:
                    continue

                # tokenize input
                with torch.no_grad():
                    input_tokenized = self.actor.tokenizer(
                        input_text,
                        return_tensors="pt",
                        truncation=True,
                        padding=True,
                    )

                    # split tokens and mask
                    input_tokenized_id = input_tokenized["input_ids"]
                    input_tokenized_mask = input_tokenized["attention_mask"]

                    # add eos token
                    (
                        input_tokenized_id,
                        input_tokenized_mask,
                    ) = self.add_eos_token(
                        input_tokenized_id,
                        input_tokenized_mask,
                    )

                    # split into input and output
                    training_output = input_tokenized_id[:, 1:]
                    training_input = input_tokenized_id[:, :-1]
                    attention_mask = input_tokenized_mask[:, :-1]

                    # move to device
                    training_output = training_output.to(device)
                    training_input = training_input.to(device)
                    attention_mask = attention_mask.to(device)

                # forward pass
                if self.config.deepspeed_enable:
                    est_output = self.model_engine(
                        training_input, attention_mask
                    )
                else:
                    est_output = self.actor(training_input, attention_mask)

                # compute loss
                est_output = rearrange(est_output, "b s v -> (b s) v")
                training_output = rearrange(training_output, "b s -> (b s)")
                loss = self.loss_function(est_output, training_output)
                self.training_stats.training_loss.append(loss.item())

                # backward pass
                if self.config.deepspeed_enable:
                    self.model_engine.backward(loss)
                    self.model_engine.step()
                elif self.config.accelerate_enable:
                    self.optimizer.zero_grad()
                    self.accelerator.backward(loss)
                    self.optimizer.step()
                    self.scheduler.step()
                else:
                    self.optimizer.zero_grad()
                    loss.backward()
                    self.optimizer.step()
                    self.scheduler.step()

                # print progress
                if i % self.config.iteration_per_print == 0:
                    print(
                        f"Epoch: {epoch+1}/{epochs}, "
                        f"Iteration: {i+1}/{n_iter}, "
                        f"Training Loss: {loss}"
                    )

                # save checkpoint periodically
                if cnt_checkpoint % checkpoint_steps == 0:
                    self.save_checkpoint(epoch, i, epochs, n_iter)
                    self.training_stats.save()
                    cnt_checkpoint = 1
                else:
                    cnt_checkpoint += 1

            # Validation
            if self.validation_flag:
                self.actor.eval()
                with torch.no_grad():
                    for i, input_text in enumerate(self.validation_dataloader):

                        # tokenize input
                        input_tokenized = self.actor.tokenizer(
                            input_text, return_tensors="pt", padding=True
                        )
                        validation_output = input_tokenized["input_ids"][:, 1:]
                        validation_input = input_tokenized["input_ids"][:, :-1]
                        attention_mask = input_tokenized["attention_mask"][
                            :, :-1
                        ]

                        # forward pass
                        est_output = self.actor.forward(
                            validation_input, attention_mask
                        )
                        validation_output = rearrange(
                            validation_output, "b s -> (b s)"
                        )

                        # compute loss
                        est_output = rearrange(est_output, "b s v -> (b s) v")
                        loss = self.loss_function(
                            est_output, validation_output
                        )
                        self.training_stats.validation_loss.append(loss.item())

                        # print progress
                        if i % self.config.iteration_per_print == 0:
                            print(
                                f"Epoch: {epoch+1}/{epochs}, "
                                f"Iteration: {i+1}/{n_iter}, "
                                f"Validation Loss: {loss}"
                            )
            # reset start_step after training is resumed
            start_step = 0

        # save the model
        self.actor.save()
        print("Training Finished ")


import pickle
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.transforms as mtransforms

expFolderName = '/home/zhangzeqing/ur5_ws/src/ur_water_bucket/data'
fileName = '/bucket_targeted_amount_0.6_saved_trajs.pkl'
save_path = expFolderName+'/fig'

f = open(expFolderName+fileName,'rb')
data = pickle.load(f)
# print(data)
# print(data['loader_type'])

paths_xyz = data['loader_pos_trajs']
paths_theta = data['loader_rot_trajs']
num_path = np.shape(paths_xyz)[0]
num_traj = data['num_episodes']
path_length = data['episode_length']

# region
# for path_id in range(num_path):
#     fig = plt.figure(figsize = (10,10))
#     ax = plt.axes(projection='3d')

#     cur_path_xyz = paths_xyz[path_id,:,:]
#     # plot path
#     ax.plot3D(cur_path_xyz[:,0],cur_path_xyz[:,2],cur_path_xyz[:,1])
#     # start/goal
#     ax.scatter(cur_path_xyz[0,0],cur_path_xyz[0,2],cur_path_xyz[0,1], marker="o", color="red", s = 40,label="start")
#     ax.scatter(cur_path_xyz[-1,0],cur_path_xyz[-1,2],cur_path_xyz[-1,1], marker="x", color="red", s = 40,label="goal")

#     ax.set_title(f'{path_id}-th path')
#     # Set axes label
    # ax.axis('scaled')
    # ax.set_xlabel('x', labelpad=5)
    # ax.set_ylabel('y', labelpad=5)
    # ax.set_zlabel('z', labelpad=5)
    # ax.view_init(0 , 180)
#     plt.legend()
#     plt.show()
#     # plt.savefig(save_path+f'/{path_id}th_path' + '.png')
# endregion

width = 0.05
height = 0.025
position_x = .1
position_y = .4
# extent = [position_x, position_x+width, position_y, position_y+height] # left, right, bottom, top
init_color_matrix = [[1.,0.], [1.,0.]]

for path_id in range(num_path):
    fig = plt.figure(figsize = (10,10))
    ax = plt.axes()
    # ax = plt.axes(projection='3d')
    # ax.view_init(0 , 180)

    # path xyz R^{75*3}
    cur_path_xyz = paths_xyz[path_id,:,:]
    # theta (rad) R^{75*1}
    rot_traj = paths_theta[path_id,:,:]

    for i in range(path_length):
        # in Shiyu's frame
        # 3D plot path x,y,z
        # ax.plot3D(cur_path_xyz[i,0],cur_path_xyz[i,2],cur_path_xyz[i,1],'.')
        # 2D plot path y,z
        ax.plot(cur_path_xyz[i,2],cur_path_xyz[i,1],'.')

        # plot bucket model (rectangle)
        position_x = cur_path_xyz[i,2]
        position_y = cur_path_xyz[i,1]
        angle_deg = np.degrees(rot_traj[i,0])
        extent = [position_x, position_x+width, position_y-height/2, position_y+height/2] # left, right, bottom, top
        car_img = ax.imshow(init_color_matrix, cmap = plt.cm.Greens, interpolation = 'bicubic', extent=extent)
        trans_data = mtransforms.Affine2D().rotate_deg_around(position_x, position_y, angle_deg) + ax.transData
        car_img.set_transform(trans_data)

        # start/goal
        ax.scatter(cur_path_xyz[0,2],cur_path_xyz[0,1], marker="o", color="red", s = 40,label="start")
        ax.scatter(cur_path_xyz[-1,2],cur_path_xyz[-1,1], marker="x", color="red", s = 40,label="goal")

        ax.axis('scaled')
        ax.set_xlim([-0.2,0.3])
        ax.set_ylim([0.1,0.7])
        ax.set_xlabel('z', labelpad=5)
        ax.set_ylabel('y', labelpad=5)


# bmi formula
# (weight in pounds x 703) / (height in inches x height in inches)

# bmi chart
# <16.0 Severely underweight
# 16.0 - 18.4 Underweight
# 18.5 - 24.9 Normie
# 25.0 - 29.9 Overweight
# 30.0 - 34.9 Moderately Obese
# 35.0 - 39.9 Severely Obese
# >39.9 Morbidly Obese

# my attempt

# inches = int(input("Enter your height in inches: "))
# pounds = int(input("Enter your weight in pounds: "))

# # weight conversion

# # unit = print(int(input("Which unit of weight are you using (p or k)? ")))
# # if unit == "k":
# #     print(int(input("Enter your weight in kilograms: ")))
# #     conversion = round(unit * 2.20462)
# # else:
# #     pounds = print(int(input("Enter your weight in pounds: ")))

# # vvv BROKEN (pseudo-code??) vvv
# bmi = round((pounds * 703) / (inches * inches), 2)

# if bmi < 16.0:
#     rating = "Severely underweight"
# elif bmi >= 16.0 or bmi <= 18.4:
#     rating = "Underweight"
# elif bmi >= 18.5 or bmi <= 24.9:
#     rating = "Normie"
# elif bmi >= 25.0 or bmi <= 29.9:
#     rating = "Overweight"
# elif bmi >= 30.0 or bmi <= 34.9:
#     rating = "Moderately Obese"
# elif bmi >= 35.0 or bmi <= 39.9:
#     rating = "Severely Obese"
# else:
#     rating = "Morbidly Obese"

# print(f"Your BMI of {bmi} makes you {rating}.")

# the solution

height = float((input("Enter your height in inches: ")))
weight = float((input("Enter your weight in pounds: ")))
bmi = weight * 703 / height ** 2
bmi = round(bmi, 1)

if bmi < 16.0:
    rating = "Severely underweight"
elif bmi >= 16.0 or bmi <= 18.4:
    rating = "Underweight"
elif bmi >= 18.5 or bmi <= 24.9:
    rating = "Normie"
elif bmi >= 25.0 or bmi <= 29.9:
    rating = "Overweight"
elif bmi >= 30.0 or bmi <= 34.9:
    rating = "Moderately Obese"
elif bmi >= 35.0 or bmi <= 39.9:
    rating = "Severely Obese"
else:
    rating = "Morbidly Obese"

print(f"Your BMI of {bmi} makes you {rating}.")


import argparse
import glob
import logging
import os
import sys
from functools import partial

import numpy as np
import pandas as pd

sys.path += [os.path.abspath('.'), os.path.abspath('..')]  # Add path to root
from birl.utilities.data_io import create_folder, load_landmarks_csv, save_landmarks_csv
from birl.utilities.dataset import compute_bounding_polygon, inside_polygon, list_sub_folders, parse_path_scale
from birl.utilities.experiments import is_iterable, iterate_mproc_map, parse_arg_params
from birl.utilities.registration import estimate_affine_transform, transform_points
from bm_dataset.rescale_tissue_images import DEFAULT_SCALES, FOLDER_TEMPLATE, NB_WORKERS

def arg_parse_params():
    """ argument parser from cmd

    :return dict:
    """
    # SEE: https://docs.python.org/3/library/argparse.html
    parser = argparse.ArgumentParser()
    parser.add_argument('-a', '--path_annots', type=str, required=False, help='path to folder with annotations')
    parser.add_argument('-d', '--path_dataset', type=str, required=False, help='path to the output directory - dataset')
    parser.add_argument(
        '--scales',
        type=int,
        required=False,
        nargs='*',
        help='generated scales for the dataset',
        default=DEFAULT_SCALES
    )
    parser.add_argument(
        '--nb_selected', type=float, required=False, default=None, help='number ot ration of selected landmarks'
    )
    parser.add_argument(
        '--nb_total', type=int, required=False, default=None, help='total number of generated landmarks'
    )
    parser.add_argument(
        '--nb_workers', type=int, required=False, default=NB_WORKERS, help='number of processes in parallel'
    )
    args = parse_arg_params(parser)
    if not is_iterable(args['scales']):
        args['scales'] = [args['scales']]
    return args

def load_largest_scale(path_set):
    """ in given set find the largest scale and load all landmarks in full size

    :param str path_set: path to image/landmark set
    :return dict: dictionary of ndarray with loaded landmarks in full scale
    """
    scales_folders = [(parse_path_scale(p), os.path.basename(p)) for p in list_sub_folders(path_set)]
    if not scales_folders:
        return
    scale, folder = sorted(scales_folders, reverse=True)[0]

    paths_csv = glob.glob(os.path.join(path_set, folder, '*.csv'))
    scaling = 100. / scale
    names_lnds = {os.path.basename(p): load_landmarks_csv(p) * scaling for p in paths_csv}
    return names_lnds

def generate_random_points_inside(ref_points, nb_extras):
    """ generate some extra points inside the tissue boundary polygon

    :param ref_points: point of the tissue
    :param int nb_extras: number of point to be added
    :return [(int, int)]: extra points
    """
    # tighter approximation, not all tissue is really convex
    convex_polygon = compute_bounding_polygon(ref_points)
    poly_mins = np.min(convex_polygon, axis=0)
    poly_size = np.max(convex_polygon, axis=0) - poly_mins
    # generate sample points inside polygon
    points_extra = []
    for i in range(int(1e5)):
        point = (np.random.random(2) * poly_size + poly_mins).astype(int)
        if inside_polygon(convex_polygon, point):
            points_extra.append(point)
            if len(points_extra) > nb_extras:
                logging.debug('particular polygon generated %f inside', nb_extras / float(i))
                break
    else:  # in case the loop ended regularly without break
        logging.warning('something went wrong with ')
    return points_extra

def expand_random_warped_landmarks(names_lnds, names_lnds_new, nb_total):
    """ add some extra point which are randomly sampled in the first sample
    and warped to the other images using estimated affine transform

    :param dict names_lnds: mapping to ndarray of the original landmarks
    :param dict names_lnds_new: mapping to ndarray of the generated landmarks
    :param int nb_total: total number of point - landmarks
    :return dict: mapping to ndarray
    """
    # estimate then number of required points
    nb_min_new = min(map(len, names_lnds_new.values()))
    nb_extras = nb_total - nb_min_new
    if nb_extras <= 0:
        return names_lnds_new

    ref_name = sorted(names_lnds)[0]
    ref_points = names_lnds[ref_name]
    points_extra = generate_random_points_inside(ref_points, nb_extras)

    for name in filter(lambda n: n != ref_name, names_lnds):
        # prepare the points
        nb_common = min([len(names_lnds[ref_name]), len(names_lnds[name])])
        pts1 = names_lnds[ref_name][:nb_common]
        pts2 = names_lnds[name][:nb_common]
        # estimate the internal affine transformation
        matrix, _, _, _ = estimate_affine_transform(pts1, pts2)
        points_warp = transform_points(points_extra, matrix)
        # insert the warped points
        names_lnds_new[name] = np.vstack([names_lnds_new[name][:nb_min_new], points_warp])
    # insert also the reference sample
    names_lnds_new[ref_name] = np.vstack([names_lnds_new[ref_name][:nb_min_new], points_extra])

    # reorder landmarks but equally in all sets
    reorder = list(range(nb_total))
    np.random.shuffle(reorder)
    names_lnds_new = {n: names_lnds_new[n][reorder] for n in names_lnds_new}
    return names_lnds_new

def extend_landmarks(path_set, path_dataset, nb_selected=None, nb_total=None):
    """ select and extend the original set of landmarks

    :param str path_set: path to the particular set if images/landmarks
    :param str path_dataset: root path to generated dataset
    :param float|int|None nb_selected: portion of selected points,
        if None use all original landmarks
    :param int|None nb_total: add extra points up to total number,
        if None, no adding extra points
    :return:
    """
    logging.debug('> processing: %s', path_set)

    # search form mas scale in set and load all related landmarks
    names_lnds = load_largest_scale(path_set)
    if not names_lnds:
        logging.warning('no landmarks was loaded for "%s"', path_set)
        return

    # select subset of selected landmarks
    names_lnds_new = {}
    if nb_selected is not None:
        if nb_selected < 0:
            raise ValueError('number of selected has to be positive')
        lens = list(map(len, names_lnds.values()))
        # transform the relative count to absolute number
        if nb_selected < 1:
            nb_selected = np.ceil(nb_selected * max(lens)).astype(int)
        # perform the selection
        indexes = list(range(min(lens)))
        np.random.shuffle(indexes)
        # just a required subset
        indexes = indexes[:nb_selected]
        for name in names_lnds:
            names_lnds_new[name] = names_lnds[name][indexes]
    else:
        names_lnds_new = names_lnds

    if nb_total is not None:
        names_lnds_new = expand_random_warped_landmarks(names_lnds, names_lnds_new, nb_total)

    # export the landmarks
    path_set_scale = os.path.join(path_dataset, os.path.basename(path_set), FOLDER_TEMPLATE % 100)
    create_folder(path_set_scale)
    for name, val in names_lnds_new.items():
        save_landmarks_csv(os.path.join(path_set_scale, name), val)

def dataset_expand_landmarks(path_annots, path_dataset, nb_selected=None, nb_total=None, nb_workers=NB_WORKERS):
    """ select and expand over whole dataset

    :param str path_annots: root path to original dataset
    :param str path_dataset: root path to generated dataset
    :param float|int|None nb_selected: portion of selected points
    :param int|None nb_total: add extra points up to total number
    :param int nb_workers: number of jobs running in parallel
    :return list(int):
    """
    list_sets = list_sub_folders(path_annots)
    logging.info('Found sets: %i', len(list_sets))

    _wrap_extend = partial(extend_landmarks, path_dataset=path_dataset, nb_selected=nb_selected, nb_total=nb_total)
    counts = list(iterate_mproc_map(_wrap_extend, sorted(list_sets), nb_workers=nb_workers, desc='expand landmarks'))
    return counts

def scale_set_landmarks(path_set, scales=DEFAULT_SCALES):
    """ scale the updated (generated) landmarks

    the scales are created within the same path set

    :param str path_set: path to the image/landmark set
    :param list(int) scales: created scales
    :return dict:
    """
    logging.debug('> processing: %s', path_set)
    path_scale100 = os.path.join(path_set, FOLDER_TEMPLATE % 100)
    if not os.path.isdir(path_scale100):
        logging.error('missing base scale 100pc in "%s"', path_scale100)
        return
    list_csv = glob.glob(os.path.join(path_scale100, '*.csv'))
    logging.debug('>> found landmarks: %i', len(list_csv))
    dict_lnds = {os.path.basename(p): pd.read_csv(p, index_col=0) for p in list_csv}
    set_scales = {}
    for sc in (sc for sc in scales if sc not in [100]):  # drop the base scale
        folder_name = FOLDER_TEMPLATE % sc
        path_scale = create_folder(os.path.join(path_set, folder_name))
        for name in dict_lnds:
            df_scale = dict_lnds[name] * (sc / 100.)
            df_scale.to_csv(os.path.join(path_scale, name))
        set_scales[sc] = len(dict_lnds)
    dict_lens = {os.path.basename(path_set): set_scales}
    return dict_lens

def dataset_scale_landmarks(path_dataset, scales=DEFAULT_SCALES, nb_workers=NB_WORKERS):
    """ generate several scales within the same dataset

    :param str path_dataset: path to the source/generated dataset
    :param [inr] scales: created scales
    :param int nb_workers: number of jobs running in parallel
    :return:
    """
    list_sets = list_sub_folders(path_dataset)
    logging.info('Found sets: %i', len(list_sets))

    _wrap_scale = partial(scale_set_landmarks, scales=scales)
    counts = list(iterate_mproc_map(_wrap_scale, sorted(list_sets), nb_workers=nb_workers, desc='scaling sets'))
    return counts

def main(path_annots, path_dataset, scales, nb_selected=None, nb_total=None, nb_workers=NB_WORKERS):
    """ main entry point

    :param str path_annots: root path to original dataset
    :param str path_dataset: root path to generated dataset
    :param list(int) scales: generated scales
    :param float|int|None nb_selected: portion of selected points
    :param int|None nb_total: add extra points up to total number
    :param int nb_workers: number of jobs running in parallel
    :return tuple(int,int):
    """
    count_gene = dataset_expand_landmarks(path_annots, path_dataset, nb_selected, nb_total, nb_workers=nb_workers)
    count_scale = dataset_scale_landmarks(path_dataset, scales=scales, nb_workers=nb_workers)
    return count_gene, count_scale

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)

    arg_params = arg_parse_params()
    logging.info('running...')
    main(**arg_params)
    logging.info('DONE')


"""Start Bot"""
import logging

from butler_G.core import start_bot
from butler_G.constants import DEV_MODE

if __name__ == "__main__":
    logger = logging.getLogger()
    shndlr, *_ = logger.handlers
    shndlr.setLevel(logging.INFO)
    shndlr.setFormatter(
        logging.Formatter(
            "[{}] %(asctime)s - %(levelname)s : %(message)s".format(
                "DEV" if DEV_MODE else "PRD"
            )
        )
    )

    start_bot()


import argparse

from graph import DirectGraph

#receiving the file with the archs
parser = argparse.ArgumentParser()
parser.add_argument('-file', help='File containing the archs and costs')
name_file = parser.parse_args().file
assert name_file is not None, 'Please pass a proper file'

def remove_vals(lista, val):
    for i in range(len(lista)):
        while val in lista[i]: lista[i].remove(val)
    return lista
def find_negatives(lista):
    for arc in lista:
        if len(arc) > 3:
            if '-' in arc:
                arc.remove('-')
                arc[2] = -int(arc[2])
            else:
                arc[2] = "".join(arc[2:])
                arc.pop(3)
    return lista
#reading from the file
file = open(name_file, 'r')
graph = []
for line in file:
    graph.append(list(line))
graph = remove_vals(graph, '\n')
graph = find_negatives(graph)
print(graph)

dg = DirectGraph(graph)
dg.PrintsGraph()
# dg.Dijkstra()
# print('---Dijkstra---')
# dg.PrintShortPath()
# dg.FIFO()
# print('---FIFO---')
# dg.PrintShortPath()
dg.Floyd()


import pygame
from game2 import Game2
from image2 import *

pygame.init()
pygame.display.set_caption("Tower of the Arcane Realms")

font = pygame.font.SysFont("assets/utilitaire/8bitlimo.ttf", 32)
text_color = (0, 0, 0)
input_rect = pygame.Rect(50, 50, 200, 32)

game = Game2()
running = True
text = ""
active = True

while running:
    for event in pygame.event.get():

        if event.type == pygame.QUIT:
            running = False
            pygame.quit()
            print("fermeture du jeu")
        elif event.type == pygame.KEYDOWN:
            game.pressed[event.key] = True
        elif event.type  == pygame.KEYUP:
            game.pressed[event.key] = False

        elif event.type == pygame.MOUSEMOTION:
            if valider_buton_rect.collidepoint(event.pos):
                valider_buton_affichee = valider_buton_grossie
                valider_buton_affichee_rect = valider_buton_grossie_rect
            else:
                valider_buton_affichee = valider_buton
                valider_buton_affichee_rect = valider_buton_rect
            if play_rect.collidepoint(event.pos) and game.background == 1:
                play_affichee = play_grossie
                play_affichee_rect = play_grossie_rect
            else:
                play_affichee = play_buton
                play_affichee_rect = play_rect
            if play_rect.collidepoint(event.pos) and game.background == 3:
                arc_affichee = arc_grossie
                arc_affichee_rect = arc_grossie_rect
            else:
                arc_affichee = arc
                arc_affichee_rect = arc_rect
            if female_rect.collidepoint(event.pos) and game.background == 2 and selction_sex == False:
                female_affichee = female_grossie
                female_affichee_rect = female_grossie_rect
            else:
                female_affichee = female
                female_affichee_rect = female_rect
            if male_rect.collidepoint(event.pos) and game.background == 2 and selction_sex == False:
                male_affichee = male_grossie
                male_affichee_rect = male_grossie_rect
            else:
                male_affichee = male
                male_affichee_rect = male_rect
            if livre_rect.collidepoint(event.pos) and game.background == 3:
                livre_affichee = livre_grossie
                livre_affichee_rect = livre_grossie_rect
            else:
                livre_affichee = livre
                livre_affichee_rect = livre_rect
            if bouclier_rect.collidepoint(event.pos) and game.background == 3:
                bouclier_affichee = bouclier_grossie
                bouclier_affichee_rect = bouclier_grossie_rect
            else:
                bouclier_affichee = bouclier
                bouclier_affichee_rect = bouclier_rect
            if epee_rect.collidepoint(event.pos) and game.background == 3:
                epee_affichee = epee_grossie
                epee_affichee_rect = epee_grossie_rect
            else:
                epee_affichee = epee
                epee_affichee_rect = epee_rect
            if arc_rect.collidepoint(event.pos) and game.background == 3:
                arc_affichee = arc_grossie
                arc_affichee_rect = arc_grossie_rect

            else:
                arc_affichee = arc
                arc_affichee_rect = arc_rect

        elif event.type == pygame.MOUSEBUTTONDOWN:
            if play_rect.collidepoint(event.pos) and game.background == 1:
                game.is_playing = True
                game.background += 1
            elif quit_rect.collidepoint(event.pos) and game.background == 1:
                running = False
                pygame.quit()
            elif valider_buton_rect.collidepoint(event.pos) and game.background == 2 and selction_sex:
                game.background += 1
            elif valider_buton_rect.collidepoint(event.pos) and game.background == 3 and selction_class :
                game.background += 1
            elif female_rect.collidepoint(event.pos) and game.background == 2:
                female_affichee = female_grossie
                female_affichee_rect = female_grossie_rect
                game.sex = 2
                selction_sex = True
            elif male_rect.collidepoint(event.pos) and game.background == 2:
                male_affichee = male_grossie
                male_affichee_rect = male_grossie_rect
                game.sex = 1
                selction_sex = True
            elif livre_rect.collidepoint(event.pos) and game.background == 3:
                livre_affichee  = livre_grossie
                livre_affichee_rect = livre_grossie_rect
                game.classe = 1
                selction_class = True
            elif epee_rect.collidepoint(event.pos) and game.background == 3:
                epee_affichee  = epee_grossie
                epee_affichee_rect = epee_grossie_rect
                game.classe = 2
                selction_class = True
            elif arc_rect.collidepoint(event.pos) and game.background == 3:
                arc_affichee  = arc_grossie
                arc_affichee_rect = arc_grossie_rect
                game.classe = 3
                selction_class = True
            elif bouclier_rect.collidepoint(event.pos) and game.background == 3:
                bouclier_affichee  = bouclier_grossie
                bouclier_affichee_rect = bouclier_grossie_rect
                game.classe = 4
                selction_class = True

    if game.background == 1 or 2 or 3 or 4:
        screen.blit(background, (0, 0))

    if game.background == 1:
        screen.blit(titre, titre_rect)
        screen.blit(play_affichee, play_affichee_rect)
        screen.blit(quit_buton, (quit_rect.x, quit_rect.y))
    elif game.is_playing and game.background == 2:
        screen.blit(sexe_text, sexe_text_rect)
        screen.blit(female_affichee, female_affichee_rect)
        screen.blit(male_affichee, male_affichee_rect)
        screen.blit(valider_buton, valider_buton_rect)
        if selction_sex:
            screen.blit(valider_buton_affichee, valider_buton_affichee_rect)
        else:
            screen.blit(valider_buton_non, valider_buton_non_rect)
    elif game.is_playing and game.background == 3:
        screen.blit(class_text, class_text_rect)
        screen.blit(livre_affichee, livre_affichee_rect)
        screen.blit(epee_affichee, epee_affichee_rect)
        screen.blit(arc_affichee, arc_affichee_rect)
        screen.blit(bouclier_affichee, bouclier_affichee_rect)
        if selction_class:
            screen.blit(valider_buton_affichee, valider_buton_affichee_rect)
        else:
            screen.blit(valider_buton_non, valider_buton_non_rect)
    elif game.is_playing and game.background == 4:
        pygame.draw.rect(screen, (255, 255, 255), input_rect)
        rendered_text = font.render(text, True, text_color)
        screen.blit(rendered_text, input_rect)

    elif game.is_playing and game.background == 6:
        game.update(screen)

    pygame.display.update()


import logging
import numa
import bisect
import numpy as np
from resources.gpu import GPU
from resources.server_config import DefaultServerConfig
'''
Creates a server with the specified number of GPUs.
If unspecified, creates a server with 8 homogenous GPUs.
Assumes that GPUs in a server are always homogenous.
If CPU, DRAM, and disk_bw not specificed, uses the default values
of those objects.
'''

class Server():

    def __init__(self,
                 rack_id=None,
                 server_id=None,
                 server_config=None,
                 start_gpu=None,
                 ip=None,
                 port=None,
                 start_gpu_deploy=0,
                 start_cpu_deploy=0,
                 numa_aware=False):
        self.logger = logging.getLogger(__name__)
        self.server_id = 0
        self.rack_id = 0
        self.start_cpu_deploy = int(start_cpu_deploy)
        self.start_gpu_deploy = int(start_gpu_deploy)
        self.numa_aware = int(numa_aware)

        if server_config is not None:
            self.server_config = server_config
        else:
            self.server_config = DefaultServerConfig()

        if server_id is not None:
            self.server_id = server_id

        if rack_id is not None:
            self.rack_id = rack_id

        # If nopt simulation, the IP, port and rpc client of the worker
        self.ip = ip
        if port is not None:
            self.port = int(port)
        else:
            self.port = port
        self.rpc_client = None

        # Max resource availability in the server
        self.num_cpus = self.server_config.cpu
        self.memory = self.server_config.mem
        self.storage_speed = self.server_config.sspeed
        self.num_gpus = self.server_config.gpu
        self.network = self.server_config.net

        self.cpus_available = list(range(self.start_cpu_deploy + 0, self.start_cpu_deploy + self.num_cpus))
        self.cpu_map_numa = dict()
        self.cpu_map_numa_ref = dict()
        if self.numa_aware:
            self.init_numa()

        # Resources allocated in the server
        self.gpu_used = 0
        self.cpu_used = 0
        self.mem_used = 0
        self.sspeed_used = 0
        self.net_used = 0

        # Resources on hold during recursive allocation
        self.gpu_on_hold = 0
        self.cpu_on_hold = 0
        self.mem_on_hold = 0
        self.sspeed_on_hold = 0
        self.net_on_hold = 0

        # Resources actually utilized by the jobs its allocated to
        # This is updated by the running jobs
        self.cpu_true_utilization = 0
        self.mem_true_utilization = 0
        self.sspeed_true_utilization = 0
        self.net_true_utilization = 0

        # Resources demanded by the jobs
        # This is updated by the running jobs
        self.cpu_demand = 0
        self.mem_demand = 0
        self.sspeed_demand = 0
        self.net_demand = 0

        self.gpus = []
        self.allocated_gpus = []
        self.gpus = self.get_gpus(start_gpu)

    def init_numa(self):
        # Populate self.cpu_map_numa - one list entry for each numa node
        num_numa_nodes = numa.get_max_node() + 1
        for i in range(0, num_numa_nodes):
            self.cpu_map_numa[i] = list(numa.node_to_cpus(i))
            self.cpu_map_numa_ref[i] = list(numa.node_to_cpus(i))

        self.logger.info("Numa nodes = {}".format(num_numa_nodes))
        self.logger.info("Numa node CPUs = {}".format(self.cpu_map_numa))

    def get_cpus(self, num_cpus):
        if not self.numa_aware:
            if len(self.cpus_available)  < num_cpus:
                return None
            cpus = self.cpus_available[:num_cpus]
            del self.cpus_available[:num_cpus]
            self.logger.info("Get {} CPUs : {}".format(num_cpus, cpus))
            return cpus
        else:
            # FInd the numa node with fewest available CPUs
            # that can satisy the current demands
            candidate_nodes = [n for n, cpus in self.cpu_map_numa.items()\
                          if len(cpus) >= num_cpus]
            if len(candidate_nodes) > 0:
                target_numa = sorted(candidate_nodes, key = lambda n : len(self.cpu_map_numa[n]))[0]
                cpus = self.cpu_map_numa[target_numa][:num_cpus]
                del self.cpu_map_numa[target_numa][:num_cpus]
                self.logger.info("Get {} CPUs : {}".format(num_cpus, cpus))
                return cpus

            sorted_numa_list = sorted(self.cpu_map_numa.keys(), key = lambda n : len(self.cpu_map_numa[n]))
            cpus = []
            i = 0
            while len(cpus) < num_cpus:
                cpus_needed = num_cpus - len(cpus)
                numa_id = sorted_numa_list[i]
                if cpus_needed < len(self.cpu_map_numa[numa_id]):
                    cpu_ids = self.cpu_map_numa[numa_id][:cpus_needed]
                    del self.cpu_map_numa[numa_id][:cpus_needed]
                    cpus.extend(cpu_ids)
                else:
                    cpu_ids = self.cpu_map_numa[numa_id]
                    self.cpu_map_numa[numa_id] = []
                    cpus.extend(cpu_ids)
                i += 1
            return cpus

    def remove_cpus_available(self, cpu_ids):
        if self.numa_aware:
            for cpu in cpu_ids:
                for numa_id, cpu_list in self.cpu_map_numa.items():
                    if cpu in cpu_list:
                        cpu_list.remove(cpu)
                        self.cpu_map_numa[numa_id] = cpu_list

        else:
            for cpu in cpu_ids:
                self.cpus_available.remove(cpu)

    def add_cpus_available(self, cpu_ids):
        if self.numa_aware:
            for cpu in cpu_ids:
                for numa_id, cpu_list in self.cpu_map_numa_ref.items():
                    if cpu in cpu_list:
                        bisect.insort(self.cpu_map_numa[numa_id], cpu)

        else:
            for cpu in cpu_ids:
                bisect.insort(self.cpus_available, cpu)

    def print_cpus_available(self):
        if self.numa_aware:
            return self.cpu_map_numa
        else:
            return self.cpus_available

    def get_gpus(self, start_gpu):
        if start_gpu is None:
            start_gpu = 0
        job_id  = -1
        tenant_id = -1

        gpus = []

        for gpu_id in np.arange(start_gpu, start_gpu+self.num_gpus):
            gpu = GPU(self.rack_id, self.server_id, gpu_id, job_id,
                     tenant_id, server_handle=self)
            gpus.append(gpu)
        return gpus

    # Update allocation of cpu, gpu, mem in the server
    def allocate(self, res_map):
        #print(res_map)
        #print(type(res_map))
        if not isinstance(res_map, dict):
            raise ValueError("Resource map is invalid")

        # No strict allocation restriction on storage speed
        for res in res_map:
            if 'cpu' in res:
                assert(res_map[res] <= self.cpu_available)
                self.cpu_used += res_map[res]
                self.cpu_true_utilization += res_map[res]
                self.cpu_demand += res_map[res]
                assert(int(self.cpu_true_utilization) <= int(self.cpu_used))
                assert(int(self.cpu_true_utilization) >= 0)
            if 'mem' in res:
                assert(res_map[res] <= self.mem_available)
                self.mem_used += res_map[res]
                self.mem_true_utilization += res_map[res]
                self.mem_demand += res_map[res]
                assert(int(self.mem_true_utilization) <= int(self.mem_used))
                assert(int(self.mem_true_utilization) >= 0)
            if 'gpu' in res:
                assert(res_map[res] <= self.gpu_available)
                self.gpu_used += res_map[res]
            if 'sspeed' in res:
                self.sspeed_used += res_map[res]
                self.sspeed_true_utilization += res_map[res]
                self.sspeed_demand += res_map[res]
                assert(int(self.sspeed_true_utilization) <= int(self.sspeed_used))
                assert(int(self.sspeed_true_utilization) >= 0)
        self.logger.debug("Allocate [{}] : gpus:{}, cpus:{}, mem:{}, sspeed:{}".format(self.id, self.gpu_available, self.cpu_available, self.mem_available, self.sspeed_available))

    def deallocate(self, res_map):
        if not isinstance(res_map, dict):
            raise ValueError("Resource map is invalid")
        #print(res_map)
        for res in res_map:
            if 'cpu' in res:
                assert(self.cpu_used >= res_map[res])
                self.cpu_used -= res_map[res]
            if 'mem' in res:
                assert(res_map[res] <= self.mem_used)
                self.mem_used -= res_map[res]
            if 'gpu' in res:
                assert(res_map[res] <= self.gpu_used)
                self.gpu_used -= res_map[res]
            if 'sspeed' in res:
                self.sspeed_used -= res_map[res]
        self.logger.debug("Deallocate [{}] : gpus:{}, cpus:{}, mem:{}, sspeed:{}".format(self.id, self.gpu_available, self.cpu_available, self.mem_available, self.sspeed_available))

    @property
    def id(self):
        return self.server_id

    @property
    def cpu_available(self):
        return self.num_cpus - self.cpu_used

    @property
    def mem_available(self):
        return self.memory - self.mem_used

    @property
    def sspeed_available(self):
        return self.storage_speed - self.sspeed_used

    @property
    def gpu_available(self):
        return self.num_gpus - self.gpu_used

    @property
    def net_available(self):
        return self.network - self.net_used

    def alloc_stats(self, percent=False):
        # Gpu, cpu, mem, sspeed, net used
        if percent:
            return (self.gpu_used/self.num_gpus*100,
                self.cpu_used/self.num_cpus*100,
                self.mem_used/self.memory*100,
                self.sspeed_used/self.storage_speed*100,
                self.net_used/self.network*100)
        else:
            return (self.gpu_used,
                self.cpu_used,
                self.mem_used,
                self.sspeed_used,
                self.net_used)

    def utilization_stats(self, percent=False):
        # Gpu, cpu, mem, sspeed, net true utilization
        if percent:
            return (self.gpu_used/self.num_gpus*100,
                self.cpu_true_utilization/self.num_cpus*100,
                self.mem_true_utilization/self.memory*100,
                self.sspeed_true_utilization/self.storage_speed*100,
                self.net_true_utilization/self.network*100)
        else:
            return (self.gpu_used,
                self.cpu_true_utilization,
                self.mem_true_utilization,
                self.sspeed_true_utilization,
                self.net_true_utilization)

    def demand_stats(self, percent=False):
        # Gpu, cpu, mem, sspeed, net demand at this server
        if percent:
            return (self.gpu_used/self.num_gpus*100,
                self.cpu_demand/self.num_cpus*100,
                self.mem_demand/self.memory*100,
                self.sspeed_demand/self.storage_speed*100,
                self.net_demand/self.network*100)
        else:
            return (self.gpu_used,
                self.cpu_demand,
                self.mem_demand,
                self.sspeed_demand,
                self.net_demand)

    def availability_stats(self, percent=False):
        # Gpu, cpu, mem, sspeed, net demand at this server
        if percent:
            return (self.gpu_available/self.num_gpus*100,
                self.cpu_available/self.num_cpus*100,
                self.mem_available/self.memory*100,
                self.sspeed_available/self.storage_speed*100,
                self.net_available/self.network*100)
        else:
            return (self.gpu_available,
                self.cpu_available,
                self.mem_available,
                self.sspeed_available,
                self.net_available)

    def availability_stats_with_hold(self, percent=False):
        # Gpu, cpu, mem, sspeed, net demand at this server
        return (self.gpu_available - self.gpu_on_hold,
                self.cpu_available  - self.cpu_on_hold,
                self.mem_available - self.mem_on_hold,
                self.sspeed_available - self.sspeed_on_hold,
                self.net_available - self.net_on_hold)

    def release_held_resources(self):
        self.gpu_on_hold = 0
        self.cpu_on_hold = 0
        self.mem_on_hold = 0
        self.sspeed_on_hold = 0
        self.net_on_hold = 0

    def hold_resources(self, demand_vec):
        self.gpu_on_hold += demand_vec[0]
        self.cpu_on_hold += demand_vec[1]
        self.mem_on_hold += demand_vec[2]
        self.sspeed_on_hold += demand_vec[3]
        self.net_on_hold += demand_vec[4]

    def __getstate__(self):
        d = self.__dict__.copy()
        if 'logger' in d:
            d['logger'] = d['logger'].name
        return d

    def __setstate__(self, d):
        if 'logger' in d:
            d['logger'] = logging.getLogger(d['logger'])
        self.__dict__.update(d)

    @property
    def gpu_ids(self):
        gpu_ids = []
        for gpu in self.gpus:
            gpu_ids.append(gpu.gpu_id)

        return gpu_ids

    @property
    def gpu_list(self):
        return self.gpus

    @property
    def free_gpu_list(self):
        free = []
        for gpu in self.gpus:
            if gpu not in self.allocated_gpus:
                free.append(gpu)

        return free

    def __cmp__(self,other):
        return cmp(self.server_id, other.server_id)

    def __lt__(self,other):
        return self.server_id < other.server_id

    def print_server_stats(self):
        for gpu in self.gpus:
            print("{}".format(gpu))


# 209. Minimum Size Subarray Sum (leetcode)
class Solution:
    def minSubArrayLen(self, target: int, nums: List[int]) -> int:
        l=0
        r=0
        s=0
        res=len(nums)+1
        while(r<len(nums)):
            s=s+nums[r]
            while(s>=target):
                res=min([res,r-l+1])
                s=s-nums[l]
                l=l+1
            r=r+1
        return res if res<=len(nums) else 0


from mytts import ResultReason  # type: ignore

SSML_MODEL = """<speak xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="http://www.w3.org/2001/mstts" xmlns:emo="http://www.w3.org/2009/10/emotionml" version="1.0" xml:lang="en-US"><voice name="zh-CN-XiaohanNeural"><prosody rate="18%" pitch="0%">{}</prosody></voice></speak>"""

GET_SHELF = "http://{}/getBookshelf"
GET_CHAPTER_LIST = "http://{}/getChapterList?url={}"
GET_CONTENT = "http://{}/getBookContent?url={}&index={}"

MODE_CHOOSE = """\
Choose running mode:
1. Basic
2. Fix
3. Concat
"""

CHOOSEBOOK = """
[b bright_red]No.     %02d[/b bright_red]
[orange1]name:   %s[/orange1]
[chartreuse3]author: %s[/chartreuse3]
[bright_cyan]now:    %s[/bright_cyan]
"""

DEFAULT_CONFIG = {
    "MAX_RETRY": 5,
    "MAX_TASK": 10,
    "MAX_CHAR": 1500,
    "WAIT_TIME": 5,
    "RETRY_SUB": 2,
    "MAX_WAIT": 20,
    "TRANS_MODE": 2,
    "TIMEOUT": 3,
    "OPT_DIR": "Output",
    "SAVE_LOG": True,
    "SHOW_DEBUG": False,
    "SAVE_REQ": False,
    "TO_CONSOLE": True,
    "bracket": ["“\"'【","”\"'】"],
    "LANG_FILE": "zh",
    "ip": ""
}

DEFAULT_TYPE = {
    "MAX_RETRY": (int,),
    "MAX_TASK": (int,),
    "MAX_CHAR": (int,),
    "WAIT_TIME": (int, float),
    "RETRY_SUB": (int, float),
    "MAX_WAIT": (int,),
    "TRANS_MODE": (int,),
    "TIMEOUT": (int, float),
    "OPT_DIR": (str,),
    "SHOW_DEBUG": (bool,),
    "SAVE_LOG": (bool,),
    "SAVE_REQ": (bool,),
    "bracket": (list,),
    "TO_CONSOLE": (bool,),
    "LANG_FILE": (str,),
    "ip": (str,)
}

def _debug_trans(data:dict) -> bool:
    debug = data["DEBUG"]
    if not isinstance(debug,bool):
        return False
    data["SHOW_DEBUG"] = debug
    data["SAVE_LOG"] = debug
    data.pop("DEBUG")
    return True

OLD_DATA_TRANSFORM={
    "DEBUG": _debug_trans
}

TTS_SUC = ResultReason.SynthesizingAudioCompleted
TTS_CANCEL = ResultReason.Canceled