#!/usr/bin/python3

import random
import itertools

import numpy as np
import pandas as pd

from numpy.random import beta
from typing import List

from pandas.core.frame import DataFrame  # class
from pandas.core.series import Series  # class

from history.aHistory import AHistory #class

from aggregation.negImplFeedback.aPenalization import APenalization #class

from sklearn.preprocessing import normalize

class PenalUsingReduceRelevance(APenalization):

    def __init__(self, penalPositionFnc, argumentsPositionList:List,
                 penalHistoryFnc, argumentsHistoryList:List,
                 lengthOfHistory:int):
        if type(argumentsPositionList) is not list:
            raise ValueError("Type of argumentsPositionList isn't list.")
        if type(argumentsHistoryList) is not list:
            raise ValueError("Type of argumentsHistoryList isn't list.")
        if type(lengthOfHistory) is not int:
            raise ValueError("Argument lengthOfHistory isn't type int.")

        self._penalPositionFnc = penalPositionFnc
        self._argumentsPositionList = argumentsPositionList
        self._penalHistoryFnc = penalHistoryFnc
        self._argumentsHistoryList = argumentsHistoryList

        self._lengthOfHistory = lengthOfHistory

    # methodsResultDict:dict[int,Series[int,str]]
    def runPenalization(self, userID:int, methodsResultDict:dict, history:AHistory):

        if type(userID) is not int and type(userID) is not np.int64:
            print(type(userID))
            raise ValueError("Type of userID isn't int.")
        if type(methodsResultDict) is not dict:
            raise ValueError("Type of methodsResultDict isn't dict.")
        if not isinstance(history, AHistory):
            raise ValueError("Type of history isn't AHistory.")

        itemIDs:List[int] = list(set(itertools.chain(*[rI.index for mI, rI in methodsResultDict.items()])))

        # dictionary of float penalization indexed by itemIDs
        penalties:dict = self.__getPenaltiesOfItemIDs(userID, itemIDs, history)

        penalizedResultsDict:dict = {}

        methodIdI:str
        resultsI:Series
        for methodIdI, resultsI in methodsResultDict.items():
            penalizedResultDictI:dict = {}

            itemIdJ:int
            ratingJ:int
            for itemIdJ, ratingJ in resultsI.items():
                #print("candidateIdJ: " + str(candidateIdJ))
                #print("votesOfCandidateJ: " + str(votesOfCandidateJ))

                penalizedResultDictI[itemIdJ] = ratingJ / (1 + penalties.get(itemIdJ, 0))

            penalizedResultsDict[methodIdI] = Series(penalizedResultDictI, name="rating")

        return penalizedResultsDict

    def runOneMethodPenalization(self, userID:int, methodsResultSrs:Series, history:AHistory):

        if type(userID) is not int and type(userID) is not np.int64:
            raise ValueError("Type of userID isn't int.")
        if type(methodsResultSrs) is not Series:
            raise ValueError("Type of methodsResultSrs isn't Series.")
        if not isinstance(history, AHistory):
            raise ValueError("Type of history isn't AHistory.")

        itemIDs:List[int] = methodsResultSrs.keys()

        # dictionary of float penalization indexed by itemIDs
        penalties:dict = self.__getPenaltiesOfItemIDs(userID, itemIDs, history)

        penalizedRatings:List[float] = []
        itemIdI:int
        ratingI:int
        for itemIdI, ratingI in methodsResultSrs.items():
            penalizedRatings.append(ratingI / (1 + penalties.get(itemIdI, 0)))

        penalizedItemIDswithRatingsSrs:Series = Series(penalizedRatings, index=methodsResultSrs.keys())

        sortedPenalizedItemIDswithRatingsSrs:Series = penalizedItemIDswithRatingsSrs.sort_values(
            axis=0, ascending=False, inplace=False, kind='quicksort', na_position='last')

        normalizedNewMethodsResultSrs:Series = Series(
            normalize(sortedPenalizedItemIDswithRatingsSrs.values[:, np.newaxis], axis=0).ravel(),
            index=sortedPenalizedItemIDswithRatingsSrs.index)

        return normalizedNewMethodsResultSrs
        # return Series<(rating:int, itemID:int)>

    def getPenaltiesOfItemIDs(self, userID:int, history:AHistory):
        return self.__getPenaltiesOfItemIDs(userID, None, history)

    def __getPenaltiesOfItemIDs(self, userID:int, itemIDs:List[int], history:AHistory):
        prevRecomendations:List[tuple] = history.getPreviousRecomOfUser(userID, self._lengthOfHistory)
        i:int = len(prevRecomendations)   #self._lengthOfHistory#
        print("PenaltyReduceRelevance, item penalties:")
        penalties:dict = {}
        for indexJ, userIdJ, itemIdJ, positionJ, clickedJ, timestampJ in prevRecomendations:
            actPenalty = penalties.get(itemIdJ, 0) #init with one
            penaltyPositionJ:float = self._penalPositionFnc(positionJ, *self._argumentsPositionList)
            penaltyHistoryJ:float = self._penalHistoryFnc(i, *self._argumentsHistoryList)

            prob_implicit_rejection:float = self._penalPositionFnc(positionJ, *self._argumentsPositionList)
            prob_stable_pref:float = self._penalHistoryFnc(i, *self._argumentsHistoryList)
            prob_couldBeRelevant = (1-(prob_implicit_rejection * prob_stable_pref)  )

            penalties[itemIdJ]  =  actPenalty + (penaltyPositionJ * penaltyHistoryJ)

            i = i-1
        print(penalties)
        return penalties

    """
    def __getPenaltiesOfItemIDs(self, userID:int, itemIDs:List[int], history:AHistory):

        penalties:dict = {}
        for itemIdI in itemIDs:
            prevRecomendations:List[tuple] = history.getPreviousRecomOfUserAndItem(userID, itemIdI, self._lengthOfHistory)
            prevRecomendations.reverse()

            penaltyI: float = 0
            i:int = 0
            #print(prevRecomendations)
            for indexJ, userIdJ, itemIdJ, positionJ, clickedJ, timestampJ in prevRecomendations:
                penaltyPositionJ:float = self._penalPositionFnc(positionJ, *self._argumentsPositionList)

                penaltyHistoryJ:float = self._penalHistoryFnc(indexJ, *self._argumentsHistoryList)

                penaltyI += penaltyPositionJ * penaltyHistoryJ
                i += 1
            penalties[itemIdI] = penaltyI

        return penalties
     """

def penaltyStatic(xDistance:int, value:float):
    return value

def penaltyLinear(xDistance:int, maxPenaltyValue:float, minPenaltyValue:float, lengthOfInterval:int):
    if (xDistance < 0):
        raise ValueError("xDistance must be greater than zero")
    if (maxPenaltyValue < minPenaltyValue):
        raise ValueError("maxPenaltyValue must be greater than or equal to minPenaltyValue")

    if xDistance > lengthOfInterval:
        return 0
    slope: float = (maxPenaltyValue - minPenaltyValue) / lengthOfInterval
    return maxPenaltyValue - slope * xDistance

def __getPenaltyPosition(position:int, maxPenalty:float):
    """
    Simpler version of penalty for 1/1+p position discount
    """
    return maxPenalty / (1 + position)

def __getPenaltyLinear(timeDiff:float, minTimeDiff:float, maxTimeDiff:float, minPenalty:float, maxPenalty:float):
    '''
    computes linear penalty based on a line equation
    y = m*x + c   ~   penalty = m*timeDiff + c
    the line is defined by two points:
    (minTimeDiff, maxPenalty), (maxTimeDiff, minPenalty)
    :param timeDiff: time since the recommendation (in seconds)
    :param minTimeDiff: penalty starts to decrease after this time
    :param maxTimeDiff: penalty remains minimal after this time
    :param minPenalty: minimal penalty given (for timeDiff >= maxTimeDiff)
    :param maxPenalty: maximal penalty given (for timeDiff <= minTimeDiff)
    :return: computed penalty
    '''
    if (timeDiff <= 0):
        raise ValueError("timeDiff must not be negative")
    if (minTimeDiff < 0):
        raise ValueError("minTimeDiff must not be negative")
    if (maxTimeDiff <= minTimeDiff):
        raise ValueError("maxTimeDiff must be greater than minTimeDiff")
    if (minPenalty < 0):
        raise ValueError("minPenalty must not be negative")
    if (maxPenalty < minPenalty):
        raise ValueError("maxPenalty must be greater than or equal to minPenalty")

    m = (minPenalty - maxPenalty) / (maxTimeDiff - minTimeDiff)
    c = maxPenalty - (m * maxTimeDiff)
    res = (m * timeDiff) + c
    if res < minPenalty:
        res = minPenalty
    elif res > maxPenalty:
        res = maxPenalty

    return res


from datetime import datetime
from decimal import Decimal

from pcapi.core.users.external.models import UserAttributes
from pcapi.core.users.models import Credit
from pcapi.core.users.models import DomainsCredit

common_user_attributes = UserAttributes(
    booking_count=4,
    booking_categories=["CINEMA", "LIVRE"],
    booking_subcategories=["ABO_LIVRE_NUMERIQUE", "CARTE_CINE_ILLIMITE", "CINE_PLEIN_AIR"],
    date_created=datetime(2021, 2, 6),
    date_of_birth=datetime(2003, 5, 6),
    departement_code="12",
    deposit_activation_date=None,
    deposit_expiration_date=None,
    domains_credit=DomainsCredit(
        all=Credit(initial=Decimal("500"), remaining=Decimal("480.00")),
        digital=Credit(initial=Decimal("200"), remaining=Decimal("200")),
        physical=Credit(initial=200, remaining=Decimal("180.00")),
    ),
    eligibility="age-18",
    first_name="First name",
    has_completed_id_check=True,
    is_beneficiary=True,
    is_eligible=True,
    is_email_validated=True,
    is_pro=False,
    last_booking_date=datetime(2021, 5, 6),
    last_favorite_creation_date=None,
    last_name="Last name",
    last_visit_date=None,
    marketing_email_subscription=True,
    marketing_push_subscription=True,
    postal_code=None,
    products_use_date={"product_brut_x_use": datetime(2021, 5, 6)},
    user_id=1,
)


from training_algorithm import Training
import numpy as np
import matplotlib.pyplot as plt
from pathlib import Path

class Logic_of_training:
    def __init__(self,file_name:str='syn',number_of_column:int=0) -> None:
        self.file_name=file_name
        self.number_of_column=number_of_column
        self.start_component={
            'syn':[4,4,4,4],
            'asmund':[7,5,7,6,7,7],
            'dorrit':[5,5,4,4],
            'marat':[3,3,4]
        }
        self.names={
            'syn':['first','second','third','fourth'],
            'asmund':['catechol','hydroquinone','indole','resorcinol','tryptophane','tyrosine'],
            'dorrit':['hydroquinone','tryptophane','phenylalanine','DOPA'],
            'marat':['humic','tyrosine','tryptophane']
        }

    def l2(self,n_comp:int):
        mm=Training(file_name=self.file_name,number_of_column=self.number_of_column,
                    number_of_components=[n_comp],l2_coefs=np.logspace(-3,15,19))
        response=mm.main()
        return response

    def comp(self,l2:np.ndarray,n_comp:int):
        mm=Training(file_name=self.file_name,number_of_column=self.number_of_column,
                    number_of_components=[i for i in range(1,n_comp+1)],l2_coefs=np.array([l2]))
        response=mm.main()
        return response

    def img_save_file_name(self):
        return 'loadings/'+self.file_name+'/'+self.names[self.file_name][self.number_of_column]

    def easy_plot(self,resp):
        fig = plt.figure(figsize=(17, 10.5),constrained_layout=True)
        plt.title(self.file_name+" "+self.names[self.file_name][self.number_of_column],loc='center')
        spec = fig.add_gridspec(ncols=2, nrows=1)
        row=0
        for col in range(2):
            ax = fig.add_subplot(spec[row, col])
            if col==0:
                for i in range(resp[2]['n_components']):
                    ax.plot(resp[3].w_i[i,:,0],label=str(i))
                ax.set_title('Excitation')
            elif col==1:
                for i in range(resp[2]['n_components']):
                    ax.plot(resp[3].w_k[i,:,0],label=str(i))
                ax.set_title('Emission')
            ax.legend()
        const_part_name:str=self.img_save_file_name()
        Path(const_part_name).mkdir(parents=True, exist_ok=True)
        # plt.savefig( 'loadings/syn/first.jpg')
        plt.savefig(const_part_name+'/'+self.names[self.file_name][self.number_of_column]+'.pdf', format='pdf', dpi=300,bbox_inches='tight')
        plt.savefig(const_part_name+'/'+self.names[self.file_name][self.number_of_column]+'.jpg', format='jpg', dpi=300,bbox_inches='tight')
        plt.savefig(const_part_name+'/'+self.names[self.file_name][self.number_of_column]+".svg", format="svg",bbox_inches='tight')
        plt.show()

    def main(self):
        resp=self.l2(self.start_component[self.file_name][self.number_of_column])
        l2_best=resp[3].a
        resp2=self.comp(l2=l2_best,n_comp=self.start_component[self.file_name][self.number_of_column])
        n_comp_best=resp2[3].n_components
        self.easy_plot(resp=resp2)
        return [l2_best,n_comp_best,np.sqrt(resp2[1]['mean_test_mse'][n_comp_best-1])/resp2[-2],
                resp2[1]['mean_test_r2'][n_comp_best-1]]


from __future__ import division
import os
from scipy import io as scio
import copy
from PIL.Image import Resampling
import sys
import random
import cv2
import torch
import numpy as np
from PIL import Image
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms
from torchvision.transforms import Compose, ColorJitter, ToTensor, Normalize

def get_citypersons(root_dir='data/cityperson', mode='train'):
    all_img_path = os.path.join(root_dir, 'images')
    all_anno_path = os.path.join(root_dir, 'annotations')
    rows, cols = 1024, 2048

    anno_path = os.path.join(all_anno_path, 'anno_' + mode + '.mat')
    image_data = []
    annos = scio.loadmat(anno_path)
    index = 'anno_' + mode + '_aligned'
    valid_count = 0
    iggt_count = 0
    box_count = 0

    for l in range(len(annos[index][0])):
        anno = annos[index][0][l]
        cityname = anno[0][0][0][0]
        imgname = anno[0][0][1][0]
        gts = anno[0][0][2]
        img_path = os.path.join(all_img_path, mode + '/' + cityname + '/' + imgname)
        boxes = []
        ig_boxes = []
        vis_boxes = []
        for i in range(len(gts)):
            label, x1, y1, w, h = gts[i, :5]
            x1, y1 = max(int(x1), 0), max(int(y1), 0)
            w, h = min(int(w), cols - x1 - 1), min(int(h), rows - y1 - 1)
            xv1, yv1, wv, hv = gts[i, 6:]
            xv1, yv1 = max(int(xv1), 0), max(int(yv1), 0)
            wv, hv = min(int(wv), cols - xv1 - 1), min(int(hv), rows - yv1 - 1)

            if label == 1 and h >= 50:
                box = np.array([int(x1), int(y1), int(x1) + int(w), int(y1) + int(h)])
                boxes.append(box)
                vis_box = np.array([int(xv1), int(yv1), int(xv1) + int(wv), int(yv1) + int(hv)])
                vis_boxes.append(vis_box)
            else:
                ig_box = np.array([int(x1), int(y1), int(x1) + int(w), int(y1) + int(h)])
                ig_boxes.append(ig_box)
        boxes = np.array(boxes)
        vis_boxes = np.array(vis_boxes)
        ig_boxes = np.array(ig_boxes)

        if len(boxes) > 0:
            valid_count += 1
        annotation = {}
        annotation['filepath'] = img_path
        box_count += len(boxes)
        iggt_count += len(ig_boxes)
        annotation['bboxes'] = boxes
        annotation['vis_bboxes'] = vis_boxes
        annotation['ignoreareas'] = ig_boxes
        image_data.append(annotation)

    return image_data

def _brightness(image, min=0.5, max=2.0):
    '''
    Randomly change the brightness of the input image.
    Protected against overflow.
    '''
    hsv = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)

    random_br = np.random.uniform(min, max)

    # To protect against overflow: Calculate a mask for all pixels
    # where adjustment of the brightness would exceed the maximum
    # brightness value and set the value to the maximum at those pixels.
    mask = hsv[:, :, 2] * random_br > 255
    v_channel = np.where(mask, 255, hsv[:, :, 2] * random_br)
    hsv[:, :, 2] = v_channel

    return cv2.cvtColor(hsv, cv2.COLOR_HSV2RGB)

def resize_image(image, gts, igs, scale=(0.4, 1.5)):
    height, width = image.shape[0:2]
    ratio = np.random.uniform(scale[0], scale[1])
    # if len(gts)>0 and np.max(gts[:,3]-gts[:,1])>300:
    #     ratio = np.random.uniform(scale[0], 1.0)
    new_height, new_width = int(ratio * height), int(ratio * width)
    image = cv2.resize(image, (new_width, new_height))
    if len(gts) > 0:
        gts = np.asarray(gts, dtype=float)
        gts[:, 0:4:2] *= ratio
        gts[:, 1:4:2] *= ratio

    if len(igs) > 0:
        igs = np.asarray(igs, dtype=float)
        igs[:, 0:4:2] *= ratio
        igs[:, 1:4:2] *= ratio

    return image, gts, igs

def random_crop(image, gts, igs, crop_size, limit=8):
    img_height, img_width = image.shape[0:2]
    crop_h, crop_w = crop_size

    if len(gts) > 0:
        sel_id = np.random.randint(0, len(gts))
        sel_center_x = int((gts[sel_id, 0] + gts[sel_id, 2]) / 2.0)
        sel_center_y = int((gts[sel_id, 1] + gts[sel_id, 3]) / 2.0)
    else:
        sel_center_x = int(np.random.randint(0, img_width - crop_w + 1) + crop_w * 0.5)
        sel_center_y = int(np.random.randint(0, img_height - crop_h + 1) + crop_h * 0.5)

    crop_x1 = max(sel_center_x - int(crop_w * 0.5), int(0))
    crop_y1 = max(sel_center_y - int(crop_h * 0.5), int(0))
    diff_x = max(crop_x1 + crop_w - img_width, int(0))
    crop_x1 -= diff_x
    diff_y = max(crop_y1 + crop_h - img_height, int(0))
    crop_y1 -= diff_y
    cropped_image = np.copy(image[crop_y1:crop_y1 + crop_h, crop_x1:crop_x1 + crop_w])
    # crop detections
    if len(igs) > 0:
        igs[:, 0:4:2] -= crop_x1
        igs[:, 1:4:2] -= crop_y1
        igs[:, 0:4:2] = np.clip(igs[:, 0:4:2], 0, crop_w)
        igs[:, 1:4:2] = np.clip(igs[:, 1:4:2], 0, crop_h)
        keep_inds = ((igs[:, 2] - igs[:, 0]) >= 8) & \
                    ((igs[:, 3] - igs[:, 1]) >= 8)
        igs = igs[keep_inds]
    if len(gts) > 0:
        ori_gts = np.copy(gts)
        gts[:, 0:4:2] -= crop_x1
        gts[:, 1:4:2] -= crop_y1
        gts[:, 0:4:2] = np.clip(gts[:, 0:4:2], 0, crop_w)
        gts[:, 1:4:2] = np.clip(gts[:, 1:4:2], 0, crop_h)

        before_area = (ori_gts[:, 2] - ori_gts[:, 0]) * (ori_gts[:, 3] - ori_gts[:, 1])
        after_area = (gts[:, 2] - gts[:, 0]) * (gts[:, 3] - gts[:, 1])

        keep_inds = ((gts[:, 2] - gts[:, 0]) >= limit) & \
                    (after_area >= 0.5 * before_area)
        gts = gts[keep_inds]

    return cropped_image, gts, igs

def random_pave(image, gts, igs, pave_size, limit=8):
    img_height, img_width = image.shape[0:2]
    pave_h, pave_w = pave_size
    # paved_image = np.zeros((pave_h, pave_w, 3), dtype=image.dtype)
    paved_image = np.ones((pave_h, pave_w, 3), dtype=image.dtype) * np.mean(image, dtype=int)
    pave_x = int(np.random.randint(0, pave_w - img_width + 1))
    pave_y = int(np.random.randint(0, pave_h - img_height + 1))
    paved_image[pave_y:pave_y + img_height, pave_x:pave_x + img_width] = image
    # pave detections
    if len(igs) > 0:
        igs[:, 0:4:2] += pave_x
        igs[:, 1:4:2] += pave_y
        keep_inds = ((igs[:, 2] - igs[:, 0]) >= 8) & \
                    ((igs[:, 3] - igs[:, 1]) >= 8)
        igs = igs[keep_inds]

    if len(gts) > 0:
        gts[:, 0:4:2] += pave_x
        gts[:, 1:4:2] += pave_y
        keep_inds = ((gts[:, 2] - gts[:, 0]) >= limit)
        gts = gts[keep_inds]

    return paved_image, gts, igs

def augment(img_data, c, img):
    assert 'filepath' in img_data
    assert 'bboxes' in img_data
    img_data_aug = copy.deepcopy(img_data)
    if img is None:
        img = cv2.imread(img_data_aug['filepath'])
    img_height, img_width = img.shape[:2]

    # random brightness
    if c.brightness and np.random.randint(0, 2) == 0:
        img = _brightness(img, min=c.brightness[0], max=c.brightness[1])
    # random horizontal flip
    if c.use_horizontal_flips and np.random.randint(0, 2) == 0:
        img = cv2.flip(img, 1)
        if len(img_data_aug['bboxes']) > 0:
            img_data_aug['bboxes'][:, [0, 2]] = img_width - img_data_aug['bboxes'][:, [2, 0]]
        if len(img_data_aug['ignoreareas']) > 0:
            img_data_aug['ignoreareas'][:, [0, 2]] = img_width - img_data_aug['ignoreareas'][:, [2, 0]]

    gts = np.copy(img_data_aug['bboxes'])
    igs = np.copy(img_data_aug['ignoreareas'])

    img, gts, igs = resize_image(img, gts, igs, scale=(0.4, 1.5))
    if img.shape[0] >= c.size_train[0]:
        img, gts, igs = random_crop(img, gts, igs, c.size_train, limit=16)
    else:
        img, gts, igs = random_pave(img, gts, igs, c.size_train, limit=16)

    img_data_aug['bboxes'] = gts
    img_data_aug['ignoreareas'] = igs

    img_data_aug['width'] = c.size_train[1]
    img_data_aug['height'] = c.size_train[0]

    return img_data_aug, img

class CityPersons(Dataset):
    def __init__(self, path, mode, config, transform=None):
        if transform is None:
            transform = transforms.Compose(
                [transforms.ColorJitter(brightness=0.5), transforms.ToTensor(),
                 transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])
        self.dataset = get_citypersons(root_dir=path, mode=mode)
        self.dataset_len = len(self.dataset)
        self.mode = mode

        if self.mode == 'train' and config.train_random:
            random.shuffle(self.dataset)
        self.config = config
        self.transform = transform

        if self.mode == 'train':
            self.preprocess = RandomResizeFix(size=config.size_train, scale=(0.4, 1.5))
        else:
            self.preprocess = None

    def __getitem__(self, item):

        # input is RGB order, and normalized
        img_data = self.dataset[item]
        img = Image.open(img_data['filepath'])

        if self.mode == 'train':
            gts = img_data['bboxes'].copy()
            igs = img_data['ignoreareas'].copy()

            x_img, gts, igs = self.preprocess(img, gts, igs)

            y_center, y_height, y_offset = self.calc_gt_center(gts, igs, radius=2, stride=self.config.down)

            if self.transform is not None:
                x_img = self.transform(x_img)

            return x_img, [y_center, y_height, y_offset]

        else:
            if self.transform is not None:
                x_img = self.transform(img)
            else:
                x_img = img

            return x_img

    def __len__(self):
        return self.dataset_len

    def calc_gt_center(self, gts, igs, radius=2, stride=4):

        def gaussian(kernel):
            sigma = ((kernel - 1) * 0.5 - 1) * 0.3 + 0.8
            s = 2 * (sigma ** 2)
            dx = np.exp(-np.square(np.arange(kernel) - int(kernel / 2)) / s)
            return np.reshape(dx, (-1, 1))

        scale_map = np.zeros((2, int(self.config.size_train[0] / stride), int(self.config.size_train[1] / stride)))
        offset_map = np.zeros((3, int(self.config.size_train[0] / stride), int(self.config.size_train[1] / stride)))
        pos_map = np.zeros((3, int(self.config.size_train[0] / stride), int(self.config.size_train[1] / stride)))
        pos_map[1, :, :, ] = 1  # channel 1: 1-value mask, ignore area will be set to 0

        if len(igs) > 0:
            igs = igs / stride
            for ind in range(len(igs)):
                x1, y1, x2, y2 = int(igs[ind, 0]), int(igs[ind, 1]), int(np.ceil(igs[ind, 2])), int(
                    np.ceil(igs[ind, 3]))
                pos_map[1, y1:y2, x1:x2] = 0

        if len(gts) > 0:
            gts = gts / stride
            for ind in range(len(gts)):
                x1, y1, x2, y2 = int(np.ceil(gts[ind, 0])), int(np.ceil(gts[ind, 1])), int(gts[ind, 2]), int(
                    gts[ind, 3])
                c_x, c_y = int((gts[ind, 0] + gts[ind, 2]) / 2), int((gts[ind, 1] + gts[ind, 3]) / 2)

                dx = gaussian(x2 - x1)
                dy = gaussian(y2 - y1)
                gau_map = np.multiply(dy, np.transpose(dx))

                pos_map[0, y1:y2, x1:x2] = np.maximum(pos_map[0, y1:y2, x1:x2], gau_map)  # gauss map
                pos_map[1, y1:y2, x1:x2] = 1  # 1-mask map
                pos_map[2, c_y, c_x] = 1  # center map

                scale_map[0, c_y - radius:c_y + radius + 1, c_x - radius:c_x + radius + 1] = np.log(
                    gts[ind, 3] - gts[ind, 1])  # log value of height
                scale_map[1, c_y - radius:c_y + radius + 1, c_x - radius:c_x + radius + 1] = 1  # 1-mask

                offset_map[0, c_y, c_x] = (gts[ind, 1] + gts[ind, 3]) / 2 - c_y - 0.5  # height-Y offset
                offset_map[1, c_y, c_x] = (gts[ind, 0] + gts[ind, 2]) / 2 - c_x - 0.5  # width-X offset
                offset_map[2, c_y, c_x] = 1  # 1-mask

        return pos_map, scale_map, offset_map

class RandomResizeFix(object):
    """
    Args:
        size: expected output size of each edge
        scale: scale factor
        interpolation: Default: PIL.Image.BILINEAR
    """

    def __init__(self, size, scale=(0.4, 1.5), interpolation=Resampling.BILINEAR):
        self.size = size
        self.interpolation = interpolation
        self.scale = scale

    def __call__(self, img, gts, igs):
        # resize image
        w, h = img.size
        ratio = np.random.uniform(self.scale[0], self.scale[1])
        n_w, n_h = int(ratio * w), int(ratio * h)
        img = img.resize((n_w, n_h), self.interpolation)
        gts = gts.copy()
        igs = igs.copy()

        # resize label
        if len(gts) > 0:
            gts = np.asarray(gts, dtype=float)
            gts *= ratio

        if len(igs) > 0:
            igs = np.asarray(igs, dtype=float)
            igs *= ratio

        # random flip
        w, h = img.size
        if np.random.randint(0, 2) == 0:
            img = img.transpose(Image.FLIP_LEFT_RIGHT)
            if len(gts) > 0:
                gts[:, [0, 2]] = w - gts[:, [2, 0]]
            if len(igs) > 0:
                igs[:, [0, 2]] = w - igs[:, [2, 0]]

        if h >= self.size[0]:
            # random crop
            img, gts, igs = self.random_crop(img, gts, igs, self.size, limit=16)
        else:
            # random pad
            img, gts, igs = self.random_pave(img, gts, igs, self.size, limit=16)

        return img, gts, igs

    @staticmethod
    def random_crop(img, gts, igs, size, limit=8):
        w, h = img.size
        crop_h, crop_w = size

        if len(gts) > 0:
            sel_id = np.random.randint(0, len(gts))
            sel_center_x = int((gts[sel_id, 0] + gts[sel_id, 2]) / 2.0)
            sel_center_y = int((gts[sel_id, 1] + gts[sel_id, 3]) / 2.0)
        else:
            sel_center_x = int(np.random.randint(0, w - crop_w + 1) + crop_w * 0.5)
            sel_center_y = int(np.random.randint(0, h - crop_h + 1) + crop_h * 0.5)

        crop_x1 = max(sel_center_x - int(crop_w * 0.5), int(0))
        crop_y1 = max(sel_center_y - int(crop_h * 0.5), int(0))
        diff_x = max(crop_x1 + crop_w - w, int(0))
        crop_x1 -= diff_x
        diff_y = max(crop_y1 + crop_h - h, int(0))
        crop_y1 -= diff_y
        cropped_img = img.crop((crop_x1, crop_y1, crop_x1 + crop_w, crop_y1 + crop_h))

        # crop detections
        if len(igs) > 0:
            igs[:, 0:4:2] -= crop_x1
            igs[:, 1:4:2] -= crop_y1
            igs[:, 0:4:2] = np.clip(igs[:, 0:4:2], 0, crop_w)
            igs[:, 1:4:2] = np.clip(igs[:, 1:4:2], 0, crop_h)
            keep_inds = ((igs[:, 2] - igs[:, 0]) >= 8) & ((igs[:, 3] - igs[:, 1]) >= 8)
            igs = igs[keep_inds]

        if len(gts) > 0:
            before_area = (gts[:, 2] - gts[:, 0]) * (gts[:, 3] - gts[:, 1])
            gts[:, 0:4:2] -= crop_x1
            gts[:, 1:4:2] -= crop_y1
            gts[:, 0:4:2] = np.clip(gts[:, 0:4:2], 0, crop_w)
            gts[:, 1:4:2] = np.clip(gts[:, 1:4:2], 0, crop_h)

            after_area = (gts[:, 2] - gts[:, 0]) * (gts[:, 3] - gts[:, 1])

            keep_inds = ((gts[:, 2] - gts[:, 0]) >= limit) & (after_area >= 0.5 * before_area)
            gts = gts[keep_inds]

        return cropped_img, gts, igs

    @staticmethod
    def random_pave(img, gts, igs, size, limit=8):
        img = np.asarray(img)
        h, w = img.shape[0:2]
        pave_h, pave_w = size
        # paved_image = np.zeros((pave_h, pave_w, 3), dtype=image.dtype)
        paved_image = np.ones((pave_h, pave_w, 3), dtype=img.dtype) * np.mean(img, dtype=int)
        pave_x = int(np.random.randint(0, pave_w - w + 1))
        pave_y = int(np.random.randint(0, pave_h - h + 1))
        paved_image[pave_y:pave_y + h, pave_x:pave_x + w] = img
        # pave detections
        if len(igs) > 0:
            igs[:, 0:4:2] += pave_x
            igs[:, 1:4:2] += pave_y
            keep_inds = ((igs[:, 2] - igs[:, 0]) >= 8) & ((igs[:, 3] - igs[:, 1]) >= 8)
            igs = igs[keep_inds]

        if len(gts) > 0:
            gts[:, 0:4:2] += pave_x
            gts[:, 1:4:2] += pave_y
            keep_inds = ((gts[:, 2] - gts[:, 0]) >= limit)
            gts = gts[keep_inds]

        return Image.fromarray(paved_image), gts, igs

class Config(object):
    def __init__(self):
        self.gpu_ids = [0, 1]
        self.onegpu = 4
        self.num_epochs = 150
        self.add_epoch = 0
        self.iter_per_epoch = 2000
        self.init_lr = 2e-4
        self.alpha = 0.999

        # dataset
        self.train_path = './data/citypersons'
        self.train_random = True

        # setting for network architechture
        self.network = 'resnet50'  # or 'mobilenet'
        self.point = 'center'  # or 'top', 'bottom
        self.scale = 'h'  # or 'w', 'hw'
        self.num_scale = 1  # 1 for height (or width) prediction, 2 for height+width prediction
        self.offset = True  # append offset prediction or not
        self.down = 4  # downsampling rate of the feature map for detection
        self.radius = 2  # surrounding areas of positives for the scale map

        # setting for data augmentation
        self.use_horizontal_flips = True
        self.brightness = (0.5, 2, 0.5)
        self.size_train = (640, 1280)
        self.size_test = (1024, 2048)

        # image channel-wise mean to subtract, the order is BGR
        self.img_channel_mean = [103.939, 116.779, 123.68]

        # use teacher
        self.teacher = True

        self.test_path = './data/citypersons'

        # whether or not to do validation during training
        self.val = True
        self.val_frequency = 1


'''
Next Permutation

Implement the next permutation, which rearranges numbers into the numerically next greater permutation of numbers for a given array A of size N.
If such arrangement is not possible, it must be rearranged as the lowest possible order i.e., sorted in an ascending order.

Constraints:
1 <= N <= 5 * 105
1 <= A[i] <= 109

input: [1, 2, 3]
output: [1, 3, 2]

input: [3, 2, 1]
output: [1, 2, 3]
'''

def nextPermutation(a):
    n = len(a)

    # if len <= 2 then we either have a asc order arr or desc. either way we return reverse(a)
    if len(a) <= 2:
        return a.reverse()

    # find where the desc slope from the back ends
    i = n-2
    while i >= 0 and a[i] >= a[i+1]:
        i -= 1

    # if the whole arr is in dec order
    if i == -1:
        return a.reverse()

    # swap just greater number than a[i] from the dec order and reverse the right side
    for j in range(n-1, i, -1):
        if a[i] < a[j]:
            a[i], a[j] = a[j], a[i]
            break
    a[i+1:] = reversed(a[i+1:])

    return a


import os
import unittest
from pathlib import Path

from Main.kmer.simplekmercounter.SimpleKmerCounter import SimpleKmerCounter

class MyTestCase(unittest.TestCase):

    def test_kmer_count(self):
        self.__write_input_files()
        self.__clear_output_path(self.__get_out_path())
        path = self.__get_path()
        kmer_count = SimpleKmerCounter(path,3)
        kmer_count.process(self.__get_out_path())
        self.__check_out_file(self.__get_out_path(),"id,ACU,AUC,CCG,CCU,CGG,CUC,CUG,GAU,GCC,GGC,GGG,GGU,GUU,UCC,UGA,"
                                                    "UGC,UUG\nfile1,1,0,1,0,1,1,0,0,0,0,0,1,0,1,0,0,0\nfile2,0,1,1,1,"
                                                    "1,0,1,1,1,1,1,1,1,1,1,1,1\n")

    def __get_path(self):
        project_root = Path(os.path.abspath(os.path.dirname(__file__)))
        project_root = project_root.parent.absolute()
        path = os.path.join(project_root, "resource")
        path = os.path.join(path, "test_algorithm")
        return path

    def __get_out_path(self):
        project_root = Path(os.path.abspath(os.path.dirname(__file__)))
        project_root = project_root.parent.absolute()
        path = os.path.join(project_root, "resource")
        path = os.path.join(path, "output")
        path = os.path.join(path, "out.csv")
        return path

    def __check_out_file(self, out_path,expected):
        with open(out_path,"rb") as file:
            actual = file.read().decode('utf-8')
            file.close()
        self.assertEqual(expected, actual)

    def __clear_output_path(self, path):
        if not os.path.exists(path):
            with open(path,"x") as file:
                file.close()
        else:
            with open(path, "wb+") as file:
                file.close()

    def __write_input_files(self):
        f1 = os.path.join(self.__get_path(),"file1.db")
        f2 = os.path.join(self.__get_path(),"file2.db")
        with open(f1,"wb+")as file:
            s = "name: file1\nL\nL\nL\nACUCCGGU\n"
            file.write(s.encode('utf-8'))
            file.close()
        with open(f2,"wb+")as file:
            s = "name: file2\nL\nL\nL\nGGUUGAUCCUGCCGGGC\n"
            file.write(s.encode('utf-8'))
            file.close()

if __name__ == '__main__':
    unittest.main()


# =======================
# -*- coding: utf-8 -*-
# author: LONGFEI XU
# Try your best
# ============
import sys
sys.path.append('../../')

import numpy as np
import pandas as pd

from .. import base_few
import Config
from .wuliu_ai import OrderSimilar

class OrderProcesser():
    '''
    处理相似订单合并
    '''
    def __init__(self):
        pass

    def combine_order(self, dataSaver):
        '''
        进行相似订单的合并
        '''
        similarList = [[]]
        orderDic = dataSaver.orderDic
        riderDic = dataSaver.riderFrame
        nowTime = dataSaver.time
        # 计算相似度矩阵
        similarMatrix = self.similar_matrix(orderDic, dataSaver)
        # 计算并单
        orderList = list(orderDic.keys())
        for orderId in orderList:
            jumpId = False
            for thisGroup in similarList:
                if jumpId:
                    break
                for compareId in thisGroup:
                    score = similarMatrix[compareId][orderId]
                    if score >= Config.combine_score_thres:
                        if len(thisGroup) < Config.combine_thres:
                            thisGroup.append(orderId)
                            jumpId = True
                            break
            if not jumpId:
                similarList.append([orderId])
        # 选出有组合的list
        outList = []
        for orderSet in similarList:
            if len(orderSet) > 1:
                outList.append(orderSet)
        return outList

    def similar_matrix(self, orderDic, dataSaver):
        '''
        相似度矩阵
        '''
        similarDic = {}
        orderList = list(orderDic.keys())
        orderNum = len(orderList)
        for i in range(orderNum-1):
            if orderList[i] not in similarDic:
                similarDic[orderList[i]] = {}
            for j in range(i+1, orderNum):
                scoreValue = OrderSimilar.cal_order_similar(
                        orderDic[orderList[i]],
                        orderDic[orderList[j]],
                        dataSaver
                        )
                similarDic[orderList[i]][orderList[j]] = scoreValue
                if orderList[j] not in similarDic:
                    similarDic[orderList[j]] = {}
                similarDic[orderList[j]][orderList[i]] = scoreValue
        return similarDic


class Solution:
    # @param {integer} rowIndex
    # @return {integer[]}
    def getRow(self, rowIndex):
        result = []
        for i in range(rowIndex + 1):
            result.append(1)
            for j in range(i - 1, 0, -1):
                result[j] += result[j - 1]
        return result


import os
import pickle
import time
import warnings

import config
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from data_utils import get_loader, eta, user_friendly_time, progress_bar, time_since
from model import Seq2seq

warnings.filterwarnings("ignore")

class Trainer(object):
    def __init__(self, model_path=None, level="para"):

        train_file = config.train_file
        train_adj = config.train_adj_file

        dev_file = config.dev_file
        dev_adj = config.dev_adj_file

        # load dictionary and embedding file
        with open(config.embedding, "rb") as f:
            embedding = pickle.load(f, encoding='iso-8859-1')
            device = torch.device(config.device)
            embedding = torch.Tensor(embedding).to(device)
        with open(config.word2idx_file, "rb") as f:
            word2idx = pickle.load(f, encoding='iso-8859-1')

        # train, dev loader
        print("load train data")
        self.train_loader = get_loader(train_file, train_adj,
                                       word2idx,
                                       use_tag=config.use_tag,
                                       batch_size=config.batch_size,
                                       debug=config.debug)
        print("load dev data")
        self.dev_loader = get_loader(dev_file, dev_adj,
                                     word2idx,
                                     use_tag=config.use_tag,
                                     batch_size=128,
                                     debug=config.debug)

        train_dir = os.path.join("./save", config.level)
        self.model_dir = os.path.join(train_dir, "train_%s" % config.level)
        if not os.path.exists(self.model_dir):
            os.makedirs(self.model_dir)

        self.model = Seq2seq(embedding, model_path=model_path)
        params = list(self.model.encoder.parameters()) \
                 + list(self.model.decoder.parameters())

        self.lr = config.lr
        self.optim = optim.SGD(params, self.lr, momentum=0.8)
        # self.optim = optim.Adam(params)
        self.criterion = nn.CrossEntropyLoss(ignore_index=0)

    def save_model(self, loss, epoch):
        state_dict = {
            "epoch": epoch,
            "current_loss": loss,
            "encoder_state_dict": self.model.encoder.state_dict(),
            "decoder_state_dict": self.model.decoder.state_dict()
        }
        loss = round(loss, 2)
        model_save_path = os.path.join(self.model_dir, str(epoch) + "_" + str(loss))
        torch.save(state_dict, model_save_path)

    def train(self):
        batch_num = len(self.train_loader)
        print("Epoch :", config.num_epochs)
        self.model.train_mode()
        best_loss = 1e10
        for epoch in range(1, config.num_epochs + 1):
            print("epoch {}/{} :".format(epoch, config.num_epochs), end="\r")
            start = time.time()
            # halving the learning rate after epoch 8
            if epoch >= 8 and epoch % 2 == 0:
                self.lr *= 0.5
                state_dict = self.optim.state_dict()
                for param_group in state_dict["param_groups"]:
                    param_group["lr"] = self.lr
                self.optim.load_state_dict(state_dict)

            for batch_idx, train_data in enumerate(self.train_loader, start=1):
                batch_loss = self.step(train_data)

                self.optim.zero_grad()
                batch_loss.backward()
                # gradient clipping
                nn.utils.clip_grad_norm_(self.model.encoder.parameters(), config.max_grad_norm)
                nn.utils.clip_grad_norm_(self.model.decoder.parameters(), config.max_grad_norm)
                self.optim.step()
                batch_loss = batch_loss.detach().item()
                msg = "{}/{} {} - ETA : {} - loss : {:.4f}" \
                    .format(batch_idx, batch_num, progress_bar(batch_idx, batch_num),
                            eta(start, batch_idx, batch_num), batch_loss)
                print(msg, end="\r")

            val_loss = self.evaluate(msg)
            if val_loss <= best_loss:
                best_loss = val_loss
                self.save_model(val_loss, epoch)

            print("Epoch {} took {} - final loss : {:.4f} - val loss :{:.4f}"
                  .format(epoch, user_friendly_time(time_since(start)), batch_loss, val_loss))

    def mask_matrix(self, m):
        mm = []
        for i, seq in enumerate(m):
            mm.append([])
            for token in seq:
                if token > 0:
                    mm[i].append(1)
                else:
                    mm[i].append(0)
        return mm

    def step(self, train_data):

        src_seq, ext_src_seq, src_len, trg_seq, ext_trg_seq, trg_len, tag_seq, _, ans_seq, ans_len, key_seq, key_len, adjs = train_data

        src_len = torch.LongTensor(src_len)
        ans_len = torch.LongTensor(ans_len)
        key_len = torch.LongTensor(key_len)
        enc_zeros = torch.zeros_like(src_seq)
        ans_zeros = torch.zeros_like(ans_seq)
        key_zeros = torch.zeros_like(key_seq)
        #enc_mask = torch.BoolTensor(src_seq == enc_zeros)
        enc_mask=(src_seq == 0).byte()

        adjs = torch.FloatTensor(adjs)
        if config.use_gpu:
            src_seq = src_seq.to(config.device)
            ext_src_seq = ext_src_seq.to(config.device)
            src_len = src_len.to(config.device)
            trg_seq = trg_seq.to(config.device)
            ext_trg_seq = ext_trg_seq.to(config.device)
            enc_mask = enc_mask.to(config.device)

            ans_seq = ans_seq.to(config.device)
            ans_len = ans_len.to(config.device)

            key_seq = key_seq.to(config.device)
            key_len = key_len.to(config.device)
            adjs = adjs.to(config.device)

            if config.use_tag:
                tag_seq = tag_seq.to(config.device)
            else:
                tag_seq = None

        enc_outputs, enc_states = self.model.encoder(src_seq, src_len, tag_seq, ans_seq, ans_len, key_seq, key_len,
                                                     adjs)
        sos_trg = trg_seq[:, :-1]
        eos_trg = trg_seq[:, 1:]

        if config.use_pointer:
            eos_trg = ext_trg_seq[:, 1:]
        logits = self.model.decoder(sos_trg, ext_src_seq, enc_states, enc_outputs, enc_mask)
        batch_size, nsteps, _ = logits.size()
        preds = logits.view(batch_size * nsteps, -1)
        targets = eos_trg.contiguous().view(-1)

        loss = self.criterion(preds, targets)
        return loss

    def evaluate(self, msg):
        self.model.eval_mode()
        num_val_batches = len(self.dev_loader)
        val_losses = []
        for i, val_data in enumerate(self.dev_loader, start=1):
            with torch.no_grad():
                val_batch_loss = self.step(val_data)
                val_losses.append(val_batch_loss.item())
                msg2 = "{} => Evaluating :{}/{}".format(msg, i, num_val_batches)
                print(msg2, end="\r")
        # go back to train mode
        self.model.train_mode()
        val_loss = np.mean(val_losses)

        return val_loss


import torch
from utils.functional import create_path

class Checkpoint:

    """
    A class used for storing and loading the dynamic states of modules and metrics
    during the training of the Transformer.

    Args:
        model (model.transformer.Transformer): The transformer network.
        optimizer (torch.optim.Optimizer, optional): The optimizer object assigned to the transformer network. Default is None.
        scheduler (torch.optim.lr_scheduler.ReduceLROnPlateau, optional): The scheduler assigned to the optimizer. Default is None.
        frequency (int, optional): The frequency at which the checkpoint will make a save. Default is None.
        path (str, optional): The path to save the checkpoint. Default is None.
        overwrite (bool, optional): If True, overwrite the existing checkpoint at the same path. Default is False.

    Attributes:
        model (model.transformer.Transformer): The transformer network.
        optimizer (torch.optim.Optimizer): The optimizer object assigned to the transformer network.
        scheduler (torch.optim.lr_scheduler.ReduceLROnPlateau): The scheduler assigned to the optimizer.
        frequency (int): The frequency at which the checkpoint will make a save.
        path (str): The path to save the checkpoint.
        overwrite (bool): If True, overwrite the existing checkpoint at the same path.
        epoch (int): The current epoch count.
        dict (dict): A dictionary to store additional states.
    """

    def __init__(self, model, optimizer=None, scheduler=None, frequency=None, path=None, overwrite=False):
        self.model = model
        self.optimizer = optimizer
        self.scheduler = scheduler
        self.frequency = frequency
        self.path = path
        self.overwrite = overwrite
        self.epoch = 0
        self.dict = dict()

    def check(self, **kwargs):

        """
        Saves the current state if the current epoch is a multiple of the frequency.

        Returns:
            bool: True if a checkpoint is saved, False otherwise.
        """

        if self.frequency is None:
            self.frequency = 1
        # update metric states
        self.epoch += 1
        # save current state
        if self.epoch % self.frequency == 0:
            self.save(**kwargs)
            return True
        return False

    def save(self, **kwargs):

        """
        Saves the current state of the model, optimizer, and scheduler.
        Additional states can be saved by passing them as keyword arguments.

        Args:
            **kwargs: Arbitrary keyword arguments for additional states to save.
        """

        # create path (if non-existent)
        if self.path is None:
            self.path = "checkpoint.pt"
        create_path(self.path)
        path = f"{self.path[:-3]}-{self.epoch}{self.path[-3:]}" \
            if not self.overwrite else self.path

        # required states
        state_dict = {
            "model_params": self.model.state_dict(),
            "optimizer_params": self.optimizer.state_dict() \
                if self.optimizer is not None else None,
            "scheduler_params": self.scheduler.state_dict() \
                if self.scheduler is not None else None,
            "frequency": self.frequency,
            "path": self.path,
            "overwrite": self.overwrite,
            "epoch": self.epoch
            }

        # additional states
        self.dict = dict(state_dict, **kwargs)
        torch.save(self.dict, path)

    def load_checkpoint(self, path, verbose=True, device=None):

        """
        Loads the checkpoint from the specified path and sets the states of the model, optimizer, and scheduler.

        Args:
            path (str): The path where the checkpoint is stored.
            verbose (bool, optional): If True, print a message after loading the checkpoint. Default is True.
            device (torch.device, optional): The device to map the loaded checkpoint. Default is None.
        """

        # load checkpoint & set modules
        checkpoint = torch.load(path, map_location=device)
        self.model.load_state_dict(checkpoint["model_params"])
        self.frequency = checkpoint["frequency"]
        self.path = checkpoint["path"]
        self.epoch = checkpoint["epoch"]
        self.overwrite = checkpoint["overwrite"]
        self.dict = checkpoint

        # conditinal modules
        if self.optimizer is not None:
            self.optimizer.load_state_dict(checkpoint["optimizer_params"])
        if self.scheduler is not None:
            self.scheduler.load_state_dict(checkpoint["scheduler_params"])
            self.scheduler.optimizer = self.optimizer

        if verbose:
            print("Checkpoint loaded")

    def __getitem__(self, item):

        """
        Allows retrieval of model, optimizer, scheduler, and additional states via indexing.

        Args:
            item (str): The key of the state to be retrieved.

        Returns:
            any: The state corresponding to the provided key.
        """

        if item == "model":
            return self.model
        if item == "optimizer":
            return self.optimizer
        if item == "scheduler":
            return self.scheduler
        return self.dict[item]


import os
import xlrd
# outfile = open("out.txt","w+")
outfile = open("out.txt","a")
filelist = os.listdir(os.getcwd())
for name_book in filelist:
	if name_book[-1] == "s":
		wb = xlrd.open_workbook(name_book)
		name_sheet = wb.sheet_names()
		for sheetnames in name_sheet:
			table = wb.sheet_by_name(sheetnames)
			nrows = table.nrows
			for rownum in range(nrows):
				n5 = str(table.row_values(rownum))
				outfile.write(n5)
				outfile.write('\n')
				# print table.row_values(rownum)
outfile.close()
# 13122042	10	6	2	8	9 queshi yong li jin shuju buchong

list1 = open("out.txt","r").readlines()
orderfile = list(set(list1))
orderfile.sort()
for line in orderfile:
	line1 = line.replace("[","").replace("]","").replace(".0","")
	f = open('order.txt','a')
	f.writelines(line1)
f.close()


__author__ = 'Vit'
from bs4 import BeautifulSoup

from base_classes import UrlList,URL
from site_models.base_site_model import ParseResult,ControlInfo, ThumbInfo
from site_models.soup.base_soup_model import BaseSoupSite,_iter
from site_models.util import get_href,get_url,quotes

class CBPvideoSoupSite(BaseSoupSite):
    def start_button_name(self):
        return "CBPvid"

    def get_start_button_menu_text_url_dict(self):
        return dict(HD=URL('http://collectionofbestporn.com/tag/hd-porn*'),
                    Latest=URL('http://collectionofbestporn.com/most-recent*'),
                    TopRated=URL('http://collectionofbestporn.com/top-rated*'),
                    MostViewed=URL('http://collectionofbestporn.com/most-viewed*'),
                    Categories=URL('http://collectionofbestporn.com/channels/'),
                    Longest=URL('http://collectionofbestporn.com/longest*'))

    def startpage(self):
        return URL("http://collectionofbestporn.com/most-recent*")

    def can_accept_index_file(self, base_url=URL()):
        return base_url.contain('collectionofbestporn.com/')

    def parse_soup(self, soup: BeautifulSoup, result: ParseResult, base_url: URL):
        # parce video page
        video = soup.find('video')
        if video is not None:
            urls = UrlList()
            for source in _iter(video.find_all('source')):
                urls.add(source.attrs['res'], get_url(source.attrs['src'],base_url))
            result.set_video(urls.get_media_data(-1))

            user=soup.find('div', {'class':'pull-left user-container'})
            if user is not None:
                user_strings = [string for string in user.stripped_strings]
                label='"{0} {1}"'.format(user_strings[0],user_strings[1])
                href=user.find('a', href=lambda x: '#' not in x)
                result.add_control(ControlInfo(label, get_url(href.attrs['href']+'/videos',base_url)))

            for tag_container in _iter(soup.find_all('div', {'class':'tags-container'})):
                for href in _iter(tag_container.find_all('a')):
                    if href.string is not None:
                        result.add_control(ControlInfo(str(href.string), get_url(href.attrs['href'],base_url)))
            return result

        # parce thumbnail page
        for thumbnail in soup.find_all('div',{'class':'video-thumb'}):
            href=get_url(thumbnail.a.attrs['href'],base_url)
            description=thumbnail.a.img.attrs['alt']
            thumb_url = get_url(thumbnail.img.attrs['src'], base_url)

            duration = thumbnail.find('span', {'class': "time"})
            dur_time= '' if duration is None else str(duration.string)

            quality = thumbnail.find('span', {'class': "quality"})
            qual = '' if quality is None else str(quality.string)

            result.add_thumb(ThumbInfo(thumb_url=thumb_url, href=href, popup=description,
                                       labels=[{'text':dur_time, 'align':'top right'},
                                               {'text':description, 'align':'bottom center'},
                                               {'text':qual,'align':'top left'}]))

        tags=soup.find('ul', {'class': 'drop2 hidden-xs'})
        if tags is not None:
            for tag in tags.find_all('a'):
                result.add_control(ControlInfo(str(tag.string).strip(), get_url(tag.attrs['href'],base_url)))

        pagination=soup.find('ul', {'class': 'pagination'})
        if pagination is not None:
            for page in pagination.find_all('a'):
                if page.string.isdigit():
                    result.add_page(ControlInfo(page.string, get_url(page.attrs['href'],base_url)))

        return result

if __name__ == "__main__":
    pass


import asyncio
from aioanticaptcha.recaptchav3proxyless import *

async def main():
    async with recaptchaV3Proxyless() as solver:
        solver.set_key("YOUR_KEY")
        solver.set_website_url("https://website.com")
        solver.set_website_key("SITE_KEY")
        solver.set_page_action("home_page")
        solver.set_verbose(1)
        solver.set_min_score(0.9)

        g_response = await solver.solve_and_return_solution()
        if g_response != 0:
            print("g-response: " + g_response)
        else:
            print("task finished with error " + solver.error_code)

if __name__ == "__main__":
    asyncio.run(main())


from sklearn.metrics import mean_squared_error, mean_absolute_error
from PreProsses import PreProcess
from keras.models import Sequential
from keras.layers import Flatten, Conv1D, MaxPooling1D, BatchNormalization, Dropout, Dense, Conv2D, MaxPooling2D, \
    Reshape
from keras.optimizers import Adam
import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
import pandas as pd

class my_NN:
    def __init__(self, input_shape):
        self.input_shape = input_shape

    def createCNN(self):
        model = Sequential()
        # model.add(Reshape((1, input_shape, 1)))
        model.add(Conv1D(filters=8, kernel_size=5, activation="relu"))
        model.add(MaxPooling1D(pool_size=2))

        model.add(Conv2D(filters=16, kernel_size=3, activation="relu"))
        model.add(BatchNormalization())
        model.add(Conv2D(filters=16, kernel_size=3, activation="relu"))
        model.add(BatchNormalization())
        model.add(MaxPooling2D(pool_size=2))

        model.add(Conv2D(filters=32, kernel_size=3, padding="same", activation="relu"))
        model.add(BatchNormalization())
        model.add(Conv2D(filters=32, kernel_size=3, padding="same", activation="relu"))
        model.add(BatchNormalization())
        model.add(Flatten())

        model.add(Dropout(0.5))
        model.add(Dense(512, activation="relu"))
        model.add(Dense(1))
        return model

train, target = PreProcess('Tehran').process_input_data()

print(np.array(train).shape)
split = int(len(train.index) * 0.8)
train_x = np.array(train)[:split]
test_x = np.array(train)[split:]

train_y = np.array(target)[:split]
test_y = np.array(target)[split:]

print(train_x.shape, train_y.shape, test_x.shape, test_y.shape)

# dataset = tf.data.Dataset.from_tensor_slices((train.values, target.values))
# for feat, targ in dataset.take(5):
#     print('Features: {}, Target: {}'.format(feat, targ))
# print(tf.constant(train))
# train_dataset = dataset.shuffle(len(train)).batch(1)
# print(train_dataset)

input_shape = (10, 92, 1)
learning_rate = 0.001
epochs = 200
batch_size = 64
# model = my_NN(input_shape).createCNN()

model = Sequential()
model.add(Dense(300, activation="relu"))
model.add(Dense(11, activation="relu"))
model.add(Dense(9, activation="relu"))
model.add(Dense(7, activation="relu"))
model.add(Dense(500))
model.add(Dense(300))
model.add(Dense(100))
# model.add(Dense(7, activation="relu"))
# model.add(Dense(9, activation="relu"))
# model.add(Dense(11, activation="relu"))
model.add(Dropout(0.5))
model.add(Dense(1))

optimizer = Adam(lr=learning_rate, decay=learning_rate / (epochs))

model.compile(optimizer=optimizer, loss="mse", metrics=["mae"])
model_fit = model.fit(
    x=np.array(train_x),
    y=np.array(train_y),
    batch_size=batch_size,
    epochs=epochs,
    validation_split=0.2)

history_model = model_fit.history

plt.figure()
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.plot(history_model['loss'])
plt.plot(history_model['val_loss'])
plt.legend(['train', 'validation'])
plt.show()

test_loss = model.evaluate(test_x, test_y)
print("Test Loss: ", test_loss)
# make a prediction
y_predict = model.predict(test_x)
test_y = np.array(test_y).reshape(len(test_y))
y_predict = np.array(y_predict).reshape(len(y_predict))
print(np.array(test_y).reshape(len(test_y)).shape, len(y_predict))
df = pd.DataFrame({'Actual': test_y, 'Predicted': y_predict})
print(df)
print('Mean Squared Error:', mean_absolute_error(test_y, y_predict))


from nova import context
from nova import exception
from nova import test

class EC2APIErrorTestCase(test.TestCase):
    def test_return_valid_error(self):
        # without 'code' arg
        err = exception.EC2APIError('fake error')
        self.assertEqual(err.__str__(), 'fake error')
        self.assertEqual(err.code, None)
        self.assertEqual(err.msg, 'fake error')
        # with 'code' arg
        err = exception.EC2APIError('fake error', 'blah code')
        self.assertEqual(err.code, 'blah code')
        self.assertEqual(err.msg, 'fake error')

class FakeNotifier(object):
    """Acts like the nova.notifier.api module."""
    ERROR = 88

    def __init__(self):
        self.provided_publisher = None
        self.provided_event = None
        self.provided_priority = None
        self.provided_payload = None

    def notify(self, context, publisher, event, priority, payload):
        self.provided_publisher = publisher
        self.provided_event = event
        self.provided_priority = priority
        self.provided_payload = payload
        self.provided_context = context

def good_function(self, context):
    return 99

def bad_function_exception(self, context, extra, blah="a", boo="b", zoo=None):
    raise test.TestingException()

class WrapExceptionTestCase(test.TestCase):
    def test_wrap_exception_good_return(self):
        wrapped = exception.wrap_exception()
        self.assertEquals(99, wrapped(good_function)(1, 2))

    def test_wrap_exception_throws_exception(self):
        wrapped = exception.wrap_exception()
        self.assertRaises(test.TestingException,
                          wrapped(bad_function_exception), 1, 2, 3)

    def test_wrap_exception_with_notifier(self):
        notifier = FakeNotifier()
        wrapped = exception.wrap_exception(notifier, "publisher", "event",
                                           "level")
        ctxt = context.get_admin_context()
        self.assertRaises(test.TestingException,
                          wrapped(bad_function_exception), 1, ctxt, 3, zoo=3)
        self.assertEquals(notifier.provided_publisher, "publisher")
        self.assertEquals(notifier.provided_event, "event")
        self.assertEquals(notifier.provided_priority, "level")
        self.assertEquals(notifier.provided_context, ctxt)
        for key in ['exception', 'args']:
            self.assertTrue(key in notifier.provided_payload.keys())

    def test_wrap_exception_with_notifier_defaults(self):
        notifier = FakeNotifier()
        wrapped = exception.wrap_exception(notifier)
        self.assertRaises(test.TestingException,
                          wrapped(bad_function_exception), 1, 2, 3)
        self.assertEquals(notifier.provided_publisher, None)
        self.assertEquals(notifier.provided_event, "bad_function_exception")
        self.assertEquals(notifier.provided_priority, notifier.ERROR)

class NovaExceptionTestCase(test.TestCase):
    def test_default_error_msg(self):
        class FakeNovaException(exception.NovaException):
            message = "default message"

        exc = FakeNovaException()
        self.assertEquals(unicode(exc), 'default message')

    def test_error_msg(self):
        self.assertEquals(unicode(exception.NovaException('test')),
                          'test')

    def test_default_error_msg_with_kwargs(self):
        class FakeNovaException(exception.NovaException):
            message = "default message: %(code)s"

        exc = FakeNovaException(code=500)
        self.assertEquals(unicode(exc), 'default message: 500')

    def test_error_msg_exception_with_kwargs(self):
        class FakeNovaException(exception.NovaException):
            message = "default message: %(mispelled_code)s"

        exc = FakeNovaException(code=500, mispelled_code='blah')
        self.assertEquals(unicode(exc), 'default message: blah')

    def test_default_error_code(self):
        class FakeNovaException(exception.NovaException):
            code = 404

        exc = FakeNovaException()
        self.assertEquals(exc.kwargs['code'], 404)

    def test_error_code_from_kwarg(self):
        class FakeNovaException(exception.NovaException):
            code = 500

        exc = FakeNovaException(code=404)
        self.assertEquals(exc.kwargs['code'], 404)

    def test_cleanse_dict(self):
        kwargs = {'foo': 1, 'blah_pass': 2, 'zoo_password': 3, '_pass': 4}
        self.assertEquals(exception._cleanse_dict(kwargs), {'foo': 1})

        kwargs = {}
        self.assertEquals(exception._cleanse_dict(kwargs), {})

    def test_format_message_local(self):
        class FakeNovaException(exception.NovaException):
            message = "some message"

        exc = FakeNovaException()
        self.assertEquals(unicode(exc), exc.format_message())

    def test_format_message_remote(self):
        class FakeNovaException_Remote(exception.NovaException):
            message = "some message"

            def __unicode__(self):
                return u"print the whole trace"

        exc = FakeNovaException_Remote()
        self.assertEquals(unicode(exc), u"print the whole trace")
        self.assertEquals(exc.format_message(), "some message")

    def test_format_message_remote_error(self):
        class FakeNovaException_Remote(exception.NovaException):
            message = "some message %(somearg)s"

            def __unicode__(self):
                return u"print the whole trace"

        self.flags(fatal_exception_format_errors=False)
        exc = FakeNovaException_Remote(lame_arg='lame')
        self.assertEquals(exc.format_message(), "some message %(somearg)s")


import re

def check_ip_address(input_text):
    # IPv4 pattern
    ipv4_pattern = re.compile(
        r'^((25[0-5]|2[0-4][0-9]|[0-1]?[0-9]?[0-9])\.){3}(25[0-5]|2[0-4][0-9]|[0-1]?[0-9]?[0-9])$'
    )

    # IPv6 pattern
    ipv6_pattern = re.compile(
        r'^([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}$|^([0-9a-fA-F]{1,4}:){1,7}:$'
    )

    if ipv4_pattern.match(input_text):
        return "IPv4"
    elif ipv6_pattern.match(input_text):
        return "IPv6"
    else:
        return "Bukan IP Address"

# Input
N = int(input())
input_lines = [input() for _ in range(N)]

# Output
for line in input_lines:
    result = check_ip_address(line.strip())
    print(result)


import pandas as pd
import os

# Directory where CSV files are located
dir_path = "results"
all_files = [f for f in os.listdir(dir_path) if f.endswith('.csv')]

# Create a list to store dataframes
dfs = []

# Read each CSV file, add a new column for the currency name, and append it to the list
for file in all_files:
    # Extract currency name from the filename
    currency_name = file.split('_')[0]

    file_path = os.path.join(dir_path, file)
    df = pd.read_csv(file_path)
    df = df[(df['Profit'] > 0) & (df['Profitable Orders'] > df['Loss Orders'])]
    # Add a new column 'currency' to the dataframe and assign the currency_name to all its rows
    df['currency'] = currency_name
    dfs.append(df)

# Concatenate all dataframes in the list
merged_df = pd.concat(dfs, ignore_index=True)

# Save the merged dataframe to a new CSV file
merged_df.to_csv(os.path.join(dir_path, "merged.csv"), index=False)

print("All CSV files have been merged into 'merged.csv' with currency names added.")


"""
Only reports
No file changes
Should break build on any issues.
Expect few issues
All issues should be addressable immediately.
-------
Rarely more than 1 or two issues if you have already worked pylint.
"""

from navio_tasks import settings as settings
from navio_tasks.cli_commands import check_command_exists, execute, prepinform_simple

def do_flake8() -> str:
    """
    Flake8 Checks
    """
    command = "flake8"
    check_command_exists(command)
    command_text = f"flake8 --config {settings.CONFIG_FOLDER}/.flake8"
    command_text = prepinform_simple(command_text)
    execute(*(command_text.split(" ")))
    return "flake 8 succeeded"


# Document Detection - Transformation - Extraction basics
import os
import imutils
import cv2
from transform import perspective_transform
import numpy as np
from skimage.filters import threshold_local

image = cv2.imread('receipt.jpg')
ratio = image.shape[0] / 500.0
orig = image.copy()
image = imutils.resize(image,height=500)

gray = cv2.cvtColor(image,cv2.COLOR_BGR2GRAY)
blur = cv2.GaussianBlur(gray,(5,5),0)
edges = cv2.Canny(blur,75,200)

cv2.imshow("Edges",edges)
cv2.waitKey(0)

cnts = cv2.findContours(edges.copy(),cv2.RETR_LIST,cv2.CHAIN_APPROX_SIMPLE)
cnts = imutils.grab_contours(cnts)
cnts = sorted(cnts, key = cv2.contourArea, reverse=True)[:5]

for c in cnts:
    peri = cv2.arcLength(c,True)
    approx = cv2.approxPolyDP(c,0.02*peri, True)

    if len(approx)==4:
        screenCnt = approx
        break

cv2.drawContours(image,[screenCnt],-1,(0,255,0),3)
cv2.imshow("Bordered",image)

output = perspective_transform(orig, screenCnt.reshape(4, 2) * ratio)
cv2.imshow("Tranformed",output)
cv2.waitKey(0)

warped = cv2.cvtColor(output, cv2.COLOR_BGR2GRAY)
T = threshold_local(warped, 11, offset = 10, method = "gaussian")
warped = (warped > T).astype("uint8") * 255

cv2.imshow("Scanned", imutils.resize(warped, height = 650))
cv2.waitKey(0)
# Getting data from receipt

import pytesseract

data = pytesseract.image_to_string(output,lang="eng",config="--psm 6")
print(data)

cv2.waitKey(0)


class TOrder:
    def DFS_Loop(self,Graph):
        n = len(Graph)
        visited = [False for i in range(n + 1)]
        f_s = [n for i in range(n)]
        cur_label = n

        def DFS(G, s):
            nonlocal cur_label
            nonlocal f_s
            for v in G[s]:
                if not visited[v]:
                    DFS(G,v)
            visited[s] = True
            f_s[s - 1] = cur_label
            cur_label -= 1

        for vertex in Graph:
            if visited[vertex] == False:
                DFS(Graph, vertex)
        return f_s

file = open('TOrder.txt','r')
Graph = dict()
for line in file:
    cur = line.split()
    cur = map(int, cur)
    cur = list(cur)
    Graph[cur[0]] = set()
    for i in range(1, len(cur)):
        Graph[cur[0]].add(cur[i])

print(TOrder().DFS_Loop(Graph))


#
# Tests for training and evaluation of all models.
#

import logging
import os
import pytest
import numpy as np
from datetime import datetime
from src.models.data_manager import DataManager
from src.models import train_model
from src.models.predict_model import predict_seqtoseq, predict_seqtoseq_beam
from src.models.agent import PolicyAgent
from src.models.policy_model import PolicyGradientModel

#
# Configure logging
#
abs_path = os.path.abspath(os.path.dirname(__file__))
logs_dir = os.path.join(abs_path, '../../logs')
if not os.path.exists(logs_dir):
    os.mkdir(logs_dir)
    os.chmod(logs_dir, 0o777)
log_path = os.path.join(abs_path, '../../logs/run-{0}.log')
logging.basicConfig(filename=log_path.format(datetime.now().strftime('%Y%m%d-%H%M%S')),
                    level=logging.INFO,
                    format='%(asctime)s-%(process)d-%(name)s-%(levelname)s-%(message)s',
                    datefmt='%Y-%m-%d %H:%M:%S')

logging.info('Test Logger Created.')

class TestModel(object):

    def test_get_cornell_data(self):
        d = DataManager()
        x, y, a, b = d.get_cornell_data()
        assert x is not None
        assert y is not None
        assert a is not None
        assert b is not None

    def test_get_starting_prompts(self):
        questions = DataManager().get_cornell_starting_prompts()
        print(questions)
        assert len(questions) > 0

    def test_get_dull_responses(self):
        questions = DataManager().get_cornell_dull_responses()
        print(questions)
        assert len(questions) > 0

    def test_train_seqtoseq(self):
        train_model.train_seqtoseq(model_name='test-seqtoseq')

    def test_predict_seqtoseq(self):
        y = predict_seqtoseq('Hi!', model_name='test-seqtoseq')
        print(y)

    def test_train_policy(self):
        train_model.train_rl(model_name='test-rl')

    def test_predict_policy(self):
        y = predict_seqtoseq_beam('Hi!', model_name='test-seqtoseq-attn')
        print(y)

    def test_policy_reward(self):
        data_manager = DataManager()
        agent = PolicyAgent(agent_name='test-agent')
        last_response = data_manager.question_to_tokens('I am fine. How are you?')
        request = data_manager.question_to_tokens('Great. How is the weather today?')
        responses, probs, rewards = agent.play((last_response, request))
        logging.info(responses)
        logging.info(probs)
        logging.info(rewards)
        assert len(responses) > 0

    @pytest.mark.run_this
    def test_rl_train(self):
        batch_size = 1100
        model = PolicyGradientModel(seq2seq_model_name='test-policy', model_name='rl-policy')
        model_inputs = {
            'request': [[np.random.randint(0, 5000) for _ in range(10)]] * batch_size,
            'response': [[np.random.randint(0, 5000) for _ in range(15)]] * batch_size,
            'probability': [[np.random.random() for _ in range(15)]] * batch_size,
            'reward': [[np.random.random() for _ in range(15)]] * batch_size
        }

        model.train(model_inputs, None)


#coding:utf-8

from scrapy_plus.core.engine import Engine

# # 导入所有爬虫类
#from spiders.baidu import BaiduSpider
# from spiders.douban import DoubanSpider

# # 导入所有管道类
# from pipelines import BaiduPipeline1, BaiduPipeline2, DoubanPipeline1, DoubanPipeline2

# # 导入所有中间件类
# from middlewares import SpiderMiddleware1, SpiderMiddleware2, DownloaderMiddleware1, DownloaderMiddleware2

# import settings

# import importlib

def main():
    # 构建自定义的Spider对象
    # baidu_spider = BaiduSpider()
    # douban_spider = DoubanSpider()
    # # 将对象做为参数传给Engine

    #getattr(对象，属性)
    #getattr(文件绝对路径，文件里的类)

    # 构建需要执行的多个爬虫的字典
    # spiders = {
    #     BaiduSpider.name : BaiduSpider(),
    #     DoubanSpider.name : DoubanSpider()
    # }

    # # 自定义管道
    # pipelines = [
    #     BaiduPipeline1(),
    #     BaiduPipeline2(),
    #     DoubanPipeline1(),
    #     DoubanPipeline2(),
    # ]

    # # 自定义爬虫中间件
    # spider_middlewares = [
    #     SpiderMiddleware1(),
    #     SpiderMiddleware2()
    # ]

    # # 自定义下载中间件
    # downloader_middlewares = [
    #     DownloaderMiddleware1(),
    #     DownloaderMiddleware2()
    # ]

    # engine = Engine(
    #     spiders = spiders,
    #     pipelines = pipelines,
    #     spider_mids = spider_middlewares,
    #     downloader_mids = downloader_middlewares
    # )

    engine = Engine()
    engine.start()

if __name__ == "__main__":
    main()


import unidecode, random, os.path, glob

listOfChar = 'ABCDEFGHIJKLMNOPQRSTUVWXYZÀÂÇÈÉÊÎÔÙÛ\'-'
def estLettreFr(cara):
	cara = cara.upper()
	return cara in listOfChar

def premiereLettreMajuscule(texte):
	return texte[0].upper() + texte[1:].lower()

class couplesmots:
	def __init__(self, func):
		self.isLetter = func
		self.mesCouples = []

	def cleanWord(self, word):
		#word = unidecode.unidecode(word)
		newWord = ""
		for l in word:
			if self.isLetter(l):
				newWord += l
		return newWord.upper()

	def afficher(self):
		for i in range(len(self.mesCouples)):
			print(F"{self.mesCouples[i][0].capitalize()} {self.mesCouples[i][1].capitalize()} : {self.mesCouples[i][2]} pts")

	def ajouterCouple(self, mot1, mot2):
		mot1 = self.cleanWord(mot1)
		mot2 = self.cleanWord(mot2)
		if(not mot1 or not mot2):
			return 0
		found = False
		points = 1
		for i in range(len(self.mesCouples)):
			if(mot1.upper() == self.mesCouples[i][0] and mot2.upper() == self.mesCouples[i][1]):
				points = self.mesCouples[i][2] +1
				self.mesCouples[i][2] = points
				found = True
				# print("mot ajouté!")
				break
		if not found:
			self.mesCouples.append([mot1.upper(), mot2.upper(), 1])
		return points

	def ajouterSuiteChaine(self, motprec, texte):
		if(len(texte.split()) == 0):
			return
		first = texte.split()[0]
		tail = texte[len(first)+1:]
		if(len(motprec) > 0):
			self.ajouterCouple(motprec, first)
		self.ajouterSuiteChaine(first, tail)

	def ajouterChaine(self, texte):
		self.ajouterSuiteChaine("", texte)
		return texte.split()[-1:]

	def ajouterFichier(self, nomfic, lineLimit = -1):
		lastWord = None
		with open(nomfic, 'r', encoding="utf-8") as fichier:
			line = " "
			iLine = 0
			while(line and (lineLimit < 0 or iLine <= lineLimit)):
				iLine += 1
				# print(F"Ligne {iLine} ... :")
				line = fichier.readline()
				for l in line:
					if(not self.isLetter(l)):
						l = " "
				lastWord = self.ajouterChaine(line)
		return lastWord

	def lesSuivOcc(self, mot):
		mot = self.cleanWord(mot)
		listePossible = []
		for couple in self.mesCouples:
			if(self.cleanWord(couple[0]) == mot):
				listePossible.append(couple)

		return sorted(listePossible, key= lambda x: x[2], reverse=True)
	def lesSuivants(self, mot):
		listeOcc = self.lesSuivOcc(mot)
		liste = []
		for couple in listeOcc:
			liste.append(couple[1])

		return liste

	def unSuivant(self, mot):
		liste = self.lesSuivants(mot)
		return random.choice(liste)

	def meilleurSuivant(self, mot):
		listePossible = self.lesSuivants(mot)
		if listePossible:
			return listePossible[0]
		return None

	def phrase(self, mot, n):
		phrase = [self.cleanWord(mot)]
		for i in range(n):
			mot = self.meilleurSuivant(mot)
			phrase.append(mot)

		return " ".join(phrase)

	def phraseUnique(self, mot, n) :
		utilises = [self.cleanWord(mot)]
		phrase = [self.cleanWord(mot)]

		for i in range(n):
			mots = self.lesSuivants(mot)
			if(len(mots) == 0):
				break
			mot = mots[0]
			while(mots[0] in utilises):
				mots.pop(0)
			if(mots):
				mot = mots[0]
				utilises.append(mot)
				phrase.append(mot)

		return " ".join(phrase)

	def ajouterLangage(self, lang="fr"):
		if not os.path.exists(lang):
			print(F"'{lang}' n'est pas connu...")
			return False
		files = glob.glob(lang + "/*")
		for file in files:
			self.ajouterFichier(file)
		return True

enchainemots = couplesmots(estLettreFr)
enchainemots.ajouterLangage("fr")

print(premiereLettreMajuscule(enchainemots.phrase("Il", n=20)))
print(premiereLettreMajuscule(enchainemots.phraseUnique("Il", n=20)))


from flask import Flask, request, jsonify
import pickle

app = Flask(__name__)

model = pickle.load(open('models/model.pkl', 'rb'))

@app.route('/predict', methods=['POST'])
def predict():
   # Get the request body
   req_body = request.get_json()
   print(req_body)
   # Extract the value list
   value = req_body.get('value')
   print("Value: ", value)
   # Ensure the value list contains two numbers
   if len(value) != 2 or not all(isinstance(n, (int, float)) for n in value):
        return jsonify({'error': 'Invalid input format'}), 400

   # Perform some prediction with the given values
   prediction = model.predict([value])
   print("prediction: ", prediction)
   # Return the prediction result as a JSON object
   return jsonify({'prediction': prediction[0]}), 200

if __name__ == '__main__':
    app.run(debug=True)


def flowerbed(flowerbeds):
    flowebeds.sort(key=lambda x: x[0])
    # print(flowebeds)
    start = flowebeds[0][0]
    end = flowebeds[0][1]
    for i in flowebeds:
        if i[0] > end:
            print(start, end)
            start = i[0]
        if end < i[1]:
            end = i[1]
    print(start, end)

if __name__ == '__main__':
    gardener = int(input())
    flowebeds = []
    for i in range(gardener):
        array = [int(i) for i in input().split()]
        flowebeds.append(array)
    # print(flowebeds)
    flowerbed(flowebeds)


from plico_interferometer.client.abstract_interferometer_client import \
    AbstractInterferometerClient
from plico_interferometer.devices.WCF_interface_for_4SightFocus import \
    WCFInterfacer
from plico.utils.logger import Logger
from plico.utils.decorator import override
import numpy as np
from plico_interferometer.types.interferometer_status import \
    InterferometerStatus
from plico.utils.snapshotable import Snapshotable
from plico_interferometer.utils.timeout import Timeout

class InterferometerWCFClient(AbstractInterferometerClient):

    def __init__(self, ipaddr, port,
                 timeout=2,
                 name='PhaseCam6110',
                 **_):
        self._name = name
        self.ipaddr = ipaddr
        self.port = port
        self._i4d = WCFInterfacer(ipaddr, port)
        self.timeout = timeout
        self.logger = Logger.of('PhaseCam6110')

    @override
    def name(self):
        return self._name

    @override
    def wavefront(self, how_many=1):
        '''
        Parameters
        ----------
        how_many: int
            numbers of frame to acquire

        Returns
        -------
        masked_ima: numpy masked array
            image or mean of the images required
        '''
        if how_many == 1:
            width, height, pixel_size_in_microns, data_array = \
                self._i4d.take_single_measurement()
            masked_ima = self._fromDataArrayToMaskedArray(
                width, height, data_array * 632.8e-9)
        else:
            image_list = []
            for i in range(how_many):
                width, height, pixel_size_in_microns, data_array = \
                    self._i4d.take_single_measurement()
                masked_ima = self._fromDataArrayToMaskedArray(
                    width, height, data_array * 632.8e-9)
                image_list.append(masked_ima)
            images = np.ma.dstack(image_list)
            masked_ima = np.ma.mean(images, axis=2)

        return masked_ima

    def _fromDataArrayToMaskedArray(self, width, height, data_array):
        data = np.reshape(data_array, (width, height))
        idx, idy = np.where(np.isnan(data))
        mask = np.zeros((data.shape[0], data.shape[1]))
        mask[idx, idy] = 1
        masked_ima = np.ma.masked_array(data, mask=mask.astype(bool))
        return masked_ima

    @override
    def status(self):
        serial_number = self._i4d.get_system_info()
        return InterferometerStatus(serial_number)

    @override
    def snapshot(self,
                 prefix,
                 timeout_in_sec=Timeout.GETTER):
        self._logger.notice("Getting snapshot for %s " % prefix)
        return Snapshotable.prepend(prefix, self.status().as_dict())


import os
import glob
import torch
import torchvision
import torchvision.transforms as T
from importlib import import_module
from torch.utils.data import Dataset, Sampler, DataLoader

class CrypkoDataset(Dataset):
    def __init__(self, fnames, transform):
        self.transform = transform
        self.fnames = fnames
        self.num_samples = len(self.fnames)

    def __getitem__(self,idx):
        fname = self.fnames[idx]
        # 1. Load the image
        img = torchvision.io.read_image(fname)
        # 2. Resize and normalize the images using torchvision.
        img = self.transform(img)
        return img

    def __len__(self):
        return self.num_samples

class InfiniteSampler(Sampler):
    def __init__(self, data_source):
        super(InfiniteSampler, self).__init__(data_source)
        self.N = len(data_source)

    def __iter__(self):
        while True:
            for idx in torch.randperm(self.N):
                yield idx

def get_dataset(root, img_size = 64):
    fnames = glob.glob(os.path.join(root, '*'))
    print(len(fnames),root )
    compose = [
        T.ToPILImage(),
        T.Resize((img_size, img_size)),
        T.ToTensor(),
        # T.Normalize(mean = (0.5, 0.5, 0.5), std = (0.5, 0.5, 0.5)),
    ]
    transform = T.Compose(compose)
    dataset = CrypkoDataset(fnames, transform)
    return dataset

def get_dataloader(root, img_size = 64, batch_size = 64):
    data = get_dataset(root, img_size)

    noofbatch=0
    if len(data)%batch_size>0:
        noofbatch=int(len(data)/batch_size)+1
    else:
        noofbatch=int(len(data)/batch_size)
    data_loader = iter(
        DataLoader(
            data,
            batch_size = batch_size,
            num_workers = 1,
            sampler = InfiniteSampler(data)
        )
    )
    return data_loader,noofbatch

#--------------ssim------------
import numpy as np
import math
import cv2
from math import log10, sqrt
from skimage.metrics import structural_similarity as compare_ssim

def calc_ssim(img1, img2):
    def ssim(img1, img2):
        C1 = (0.01 * 255)**2
        C2 = (0.03 * 255)**2

        img1 = img1.astype(np.float64)
        img2 = img2.astype(np.float64)
        kernel = cv2.getGaussianKernel(11, 1.5)
        window = np.outer(kernel, kernel.transpose())

        mu1 = cv2.filter2D(img1, -1, window)[5:-5, 5:-5]  # valid
        mu2 = cv2.filter2D(img2, -1, window)[5:-5, 5:-5]
        mu1_sq = mu1**2
        mu2_sq = mu2**2
        mu1_mu2 = mu1 * mu2
        sigma1_sq = cv2.filter2D(img1**2, -1, window)[5:-5, 5:-5] - mu1_sq
        sigma2_sq = cv2.filter2D(img2**2, -1, window)[5:-5, 5:-5] - mu2_sq
        sigma12 = cv2.filter2D(img1 * img2, -1, window)[5:-5, 5:-5] - mu1_mu2

        ssim_map = ((2 * mu1_mu2 + C1) * (2 * sigma12 + C2)) / ((mu1_sq + mu2_sq + C1) *
                                                                (sigma1_sq + sigma2_sq + C2))
        return ssim_map.mean()

    ### args:
        # img1: [h, w, c], range [0, 255]
        # img2: [h, w, c], range [0, 255]
        # the same outputs as MATLAB's
    border = 0
    img1_y = np.dot(img1, [65.738,129.057,25.064])/256.0+16.0
    img2_y = np.dot(img2, [65.738,129.057,25.064])/256.0+16.0
    if not img1.shape == img2.shape:
        raise ValueError('Input images must have the same dimensions.')
    h, w = img1.shape[:2]
    img1_y = img1_y[border:h-border, border:w-border]
    img2_y = img2_y[border:h-border, border:w-border]

    if img1_y.ndim == 2:
        return ssim(img1_y, img2_y)
    elif img1.ndim == 3:
        if img1.shape[2] == 3:
            ssims = []
            for i in range(3):
                ssims.append(ssim(img1, img2))
            return np.array(ssims).mean()
        elif img1.shape[2] == 1:
            return ssim(np.squeeze(img1), np.squeeze(img2))
    else:
        raise ValueError('Wrong input image dimensions.')

def calc_psnr(sr, hr, scale=1, rgb_range=255, cal_type='y'):
    #if hr.nelement() == 1: return 0
    #print(sr.shape,hr.shape)
    diff = (sr - hr) / rgb_range

    if cal_type=='y':
        gray_coeffs = [65.738, 129.057, 25.064]
        convert = diff.new_tensor(gray_coeffs).view(1, 3, 1, 1) / 256
        diff = diff.mul(convert).sum(dim=1)

    if scale == 1:
        valid = diff
    else:
        valid = diff[..., scale:-scale, scale:-scale]
    mse = valid.pow(2).mean()

    return -10 * math.log10(mse)

def calc_psnr_and_ssim(sr, hr):
    ### prepare data
    sr = (sr+1.) * 127.5
    hr = (hr+1.) * 127.5

    if (sr.size() != hr.size()):
        h_min = min(sr.size(2), hr.size(2))
        w_min = min(sr.size(3), hr.size(3))
        sr = sr[:, :, :h_min, :w_min]
        hr = hr[:, :, :h_min, :w_min]

    img1 = np.transpose(sr.squeeze().round().cpu().numpy(), (1,2,0))
    img2 = np.transpose(hr.squeeze().round().cpu().numpy(), (1,2,0))
    psnr = calc_psnr(sr, hr)
    ssim = calc_ssim(img1, img2)
    ssim2 = compare_ssim(img1, img2,multichannel=True)
    return psnr, ssim,ssim2

#-------------------------------------------------------------------
import shutil
import logging
class Logger(object):
    def __init__(self, log_file_name, logger_name, log_level=logging.DEBUG):
        ### create a logger
        self.__logger = logging.getLogger(logger_name)

        ### set the log level
        self.__logger.setLevel(log_level)

        ### create a handler to write log file
        file_handler = logging.FileHandler(log_file_name)

        ### create a handler to print on console
        console_handler = logging.StreamHandler()

        ### define the output format of handlers
        #formatter = logging.Formatter('[%(asctime)s] - [%(filename)s file line:%(lineno)d] - %(levelname)s: %(message)s')
        formatter = logging.Formatter('%(message)s')
        file_handler.setFormatter(formatter)
        console_handler.setFormatter(formatter)

        ### add handler to logger
        self.__logger.addHandler(file_handler)
        #self.__logger.addHandler(console_handler)

    def get_log(self):
        return self.__logger

def mkExpDir(save_dir):
    if (os.path.exists(save_dir)):
        shutil.rmtree(save_dir)
    os.makedirs(save_dir)
    os.makedirs(os.path.join(save_dir, 'saliency'))
    os.makedirs(os.path.join(save_dir, 'save_results'))
    os.makedirs(os.path.join(save_dir, 'diff'))
    os.makedirs(os.path.join(save_dir, 'test_results'))
    os.makedirs(os.path.join(save_dir, 'weights'))
    args_file = open(os.path.join(save_dir, 'args.txt'), 'w')
    _logger = Logger(log_file_name=os.path.join(save_dir,'model.log'),
        logger_name='modellog').get_log()
    return _logger


from utils import writer, cursor, format_filename
from datetime import date, timedelta
import re
import logging

MAC_PATTERN = re.compile(r':')
LOGIN_PATTERN = re.compile(r'-\d{4}-\d{2}-\d{2}[_ ]\d{2}:\d{2}:\d{2}')

FORMAT = 'ABONENT_IDENT_%Y%m%d_%H%M.txt'
FIELDS = [
    'ABONENT_ID',
    'REGION_ID',
    'IDENT_TYPE',
    'PHONE',
    'INTERNAL_NUMBER',
    'IMSI',
    'IMEI',
    'ICC',
    'MIN',
    'ESN',
    'EQUIPMENT_TYPE',
    'MAC',
    'VPI',
    'VCI',
    'LOGIN',
    'E_MAIL',
    'PIN',
    'USER_DOMAIN',
    'RESERVED',
    'ORIGINATOR_NAME',
    'IP_TYPE',
    'IPV4',
    'IPV6',
    'IPV4_MASK',
    'IPV6_MASK',
    'BEGIN_TIME',
    'END_TIME',
    'LINE_OBJECT',
    'LINE_CROSS',
    'LINE_BLOCK',
    'LINE_PAIR',
    'LINE_RESERVED',
    'LOC_TYPE',
    'LOC_LAC',
    'LOC_CELL',
    'LOC_TA',
    'LOC_CELL_WIRELESS',
    'LOC_MAC',
    'LOC_LATITUDE',
    'LOC_LONGITUDE',
    'LOC_PROJECTION_TYPE',
    'RECORD_ACTION',
    'INTERNAL_ID1',
    'INTERNAL_ID2'
]

QUERY_FULL = '''
SELECT v.uid, v.login,
right(HEX(i.segment),8) ipv4, right(HEX(i.mask),8) ipv4_mask,
CONVERT_TZ(i.timefrom,'+03:00','+00:00') begin_time,
CONVERT_TZ(i.timeto,'+03:00','+00:00') end_time,
(SELECT login FROM radius.sessionsradius r WHERE r.vg_id = v.vg_id) mac
FROM staff_history i
INNER JOIN vgroups v ON (v.vg_id = i.vg_id AND v.id = 2)
INNER JOIN (SELECT DISTINCT(abonentId) uid FROM sorm WHERE record_action !=3) s ON (v.uid = s.uid)
'''

def report_daily(db):
    return report_full(db, (" WHERE i.last_mod_date >= (select COALESCE(max(batch_time),current_timestamp) from sorm_batch where batch_name='ident')"
                            " OR EXISTS (SELECT abonentId FROM sorm s WHERE s.abonentId = v.uid AND record_action = 1)"))

def report_full(db, params=''):
    with cursor(db) as cur:
        cur.execute(QUERY_FULL + params)
        logging.info("collect ident [{0}]".format(cur.rowcount))
        filename = format_filename(FORMAT)
        with writer(filename, FIELDS) as csvout:
            for row in cur:
                # map out record
                outRow = {
                    'ABONENT_ID': row.get('uid'),
                    'REGION_ID': 1,  # STSZ
                    'IDENT_TYPE': 5, # data
                    'EQUIPMENT_TYPE': 0,  # mac
                    'MAC': '' if row.get('mac') == None else MAC_PATTERN.sub('', row.get('mac')),
                    'LOGIN': LOGIN_PATTERN.sub('', row.get('login')),
                    'IP_TYPE': 0,  # ipv4
                    'IPV4': row.get('ipv4'),
                    'IPV4_MASK': row.get('ipv4_mask'),
                    'BEGIN_TIME': row.get('begin_time'),
                    'END_TIME': '' if row.get('end_time') == '0000-00-00 00:00:00' else row.get('end_time')
                }
                csvout.writerow(outRow)
        logging.info("flush filename:{0} [{1}]".format(filename, cur.rowcount))
        # store batch info
        cur.execute("INSERT INTO sorm_batch (batch_name, file_name, file_rec_count) VALUES (%s, %s, %s)", ('ident', filename, cur.rowcount))
        db.commit()
    return filename


'''
Description:  百度关键词爬取  https://www.yuque.com/huangzhongqing/spider/zpdk9z
Author: HCQ
Company(School): UCAS
Email: 1756260160@qq.com
Date: 2020-12-28 10:05:14
LastEditTime: 2020-12-30 09:59:55
FilePath: /Spider/08百度搜狗百科关键词爬取/baidu.py
'''
import urllib.request
import urllib.parse
import re
from bs4 import BeautifulSoup

def baike( word ) :
       def test_url( soup ) :      		# 检测是否收录该词条，返回 True or False
              result = soup.find( text=re.compile("百度百科未收录该词条") )
              if result :
                     return False
              else:
                     return True

       def summary( soup ) :
              # h1标签的文本（百科的主标题）
              word = soup.h1.text   # 此处word含义转变 , 不要弄混
              # h2标签的文本（百科的副标题）
              if soup.h2 :
                     word += soup.h2.text

              print( word )

               #（百科的简介）
              if soup.find( class_="lemma-summary" ) :
                     print( soup.find( class_="lemma-summary" ).text )

       def start( word ):
              keyword = urllib.parse.urlencode( {"word" : word} )  	# 解析，用于组成URL

              response = urllib.request.urlopen( "http://baike.baidu.com/search/word?%s" % keyword )
              html = response.read()
              soup = BeautifulSoup( html , "html.parser" )

              if test_url( soup ) :
                     summary( soup )

       try :
              start( word )
       except AttributeError :
              print("百度百科未收录该词条")

if(__name__ == "__main__") :
       content = str( input("请输入关键词  :  ") )
       baike(content)


from drf_yasg import openapi
from drf_yasg.utils import swagger_auto_schema
from rest_framework import views, status
from rest_framework.authentication import TokenAuthentication

from transactions.models import VaccinationStatus, VaccineType
from transactions.permissions import IsAdminAuthenticated
from transactions.services.default_employee_by_admin_service import DefaultEmployeeByAdminService

class EmployeeByAdminAPIView(views.APIView):
    authentication_classes = [TokenAuthentication]
    permission_classes = [IsAdminAuthenticated]
    employee_admin_service = DefaultEmployeeByAdminService()

    @swagger_auto_schema(
        tags=["employee by admin"],
        responses={
            status.HTTP_200_OK: employee_admin_service.serializer_id_class(many=True),
            status.HTTP_400_BAD_REQUEST: "Bad request",
            status.HTTP_403_FORBIDDEN: "You do not have permission to perform this action"
        },
        manual_parameters=[
            openapi.Parameter(
                'vaccination_status', openapi.IN_QUERY, description="Vaccination Status", type=openapi.TYPE_STRING,
                enum=VaccinationStatus.values
            ),
            openapi.Parameter(
                'vaccine_type', openapi.IN_QUERY, description="Vaccine Type", type=openapi.TYPE_STRING,
                enum=VaccineType.values
            ),
            openapi.Parameter(
                'vaccination_date_start', openapi.IN_QUERY, description="Vaccination Date Start",
                type=openapi.TYPE_STRING, format="%Y-%m-%d"
            ),
            openapi.Parameter(
                'vaccination_date_end', openapi.IN_QUERY, description="Vaccination Date End",
                type=openapi.TYPE_STRING, format="%Y-%m-%d"
            )
        ]
    )
    def get(self, request, *args, **kwargs):
        return self.employee_admin_service.get(None, request)

    @swagger_auto_schema(
        tags=["employee by admin"],
        request_body=employee_admin_service.serializer_create_by_admin_class,
        responses={
            status.HTTP_200_OK: employee_admin_service.serializer_id_class,
            status.HTTP_400_BAD_REQUEST: "Bad request",
            status.HTTP_403_FORBIDDEN: "You do not have permission to perform this action"
        }
    )
    def post(self, request, *args, **kwargs):
        return self.employee_admin_service.create(request)

class EmployeeByAdminPkAPIView(views.APIView):
    authentication_classes = [TokenAuthentication]
    permission_classes = [IsAdminAuthenticated]
    employee_admin_service = DefaultEmployeeByAdminService()

    @swagger_auto_schema(
        tags=["employee by admin"],
        responses={
            status.HTTP_200_OK: employee_admin_service.serializer_id_class(many=True),
            status.HTTP_400_BAD_REQUEST: "Bad request",
            status.HTTP_403_FORBIDDEN: "You do not have permission to perform this action"
        }
    )
    def get(self, request, pk, *args, **kwargs):
        return self.employee_admin_service.get(pk, request)

    @swagger_auto_schema(
        tags=["employee by admin"],
        request_body=employee_admin_service.serializer_update_by_admin_class,
        responses={
            status.HTTP_200_OK: employee_admin_service.serializer_id_class,
            status.HTTP_400_BAD_REQUEST: "Bad request",
            status.HTTP_403_FORBIDDEN: "You do not have permission to perform this action"
        }
    )
    def put(self, request, pk, *args, **kwargs):
        return self.employee_admin_service.update(pk, request)

    @swagger_auto_schema(
        tags=["employee by admin"],
        responses={
            status.HTTP_204_NO_CONTENT: "OK No content",
            status.HTTP_400_BAD_REQUEST: "Bad request",
            status.HTTP_403_FORBIDDEN: "You do not have permission to perform this action"
        }
    )
    def delete(self, request, pk, *args, **kwargs):
        return self.employee_admin_service.delete(pk)


import pika
from configparser import ConfigParser
import logging
import json
import sys
from kafka import KafkaProducer

from binance_f import RequestClient
from binance_f.model import *
from binance_f.constant.test import *
from binance_f.base.printobject import *

logging.basicConfig(format='%(asctime)s %(message)s')

def config(filename, section):
   parser = ConfigParser()
   parser.read(filename)
   rmq = {}
   if parser.has_section(section):
      params = parser.items(section)
      for param in params:
         rmq[param[0]] = param[1]
   else:
      raise Exception('Section {0} not found in the {1} file'.format(section, filename))
   return rmq

def main():
   if len(sys.argv) != 2 or sys.argv[1] != "-kp":
      logging.error("Incorrect args. Program: consumer.py -kp <ip:port>")

   ip_kp = sys.argv[2]
   logging.warning("IP:PORT KAFKA PRODUCER: " + ip_kp)
   producer = KafkaProducer(bootstrap_servers=ip_kp)
   params = config('config.ini', 'rabbitmq')
   connection = pika.BlockingConnection(pika.ConnectionParameters(host=params['url']))
   channel = connection.channel()
   channel.queue_declare(queue=params['queue'])

   def callback(ch, method, properties, body):
      message = json.loads(body)
      logging.warning(" [x] Received %r" % message["symbol"])
      p_binance = config('config.ini', 'apibinance')
      request_client = RequestClient(api_key=p_binance["apikey"], secret_key=p_binance["secret"])
      result = request_client.get_candlestick_data(symbol="BTCUSDT", interval=CandlestickInterval.MIN1, startTime=None, endTime=None, limit=1)

      future = producer.send(message["symbol"], b'example_message')
      result = future.get(timeout=10)
      logging.warning(result)

   channel.basic_consume(queue=params['queue'], on_message_callback=callback, auto_ack=True)
   channel.start_consuming()

if __name__ == '__main__':
   main()