#!/usr/bin/env python3

from PyQt5.QtWidgets import *
import sys, os, pandas as pd
import numpy as np
import re, time, subprocess, stat

extensions = ['xlsx', 'xls']

class ReadExcel:
    def __init__(self):
        self.app = QApplication(sys.argv)
        self.mainWindow = QMainWindow()
        self.new_file_path = ''
        self.file_path = ''
        self.excel_new_data = np.array([])
        self.excel_files = []
        self.company_data = []
        self.new_data = []
        self.frame_data = []
        self.root_path = ''

    def open_file(self):
        self.file_path = QFileDialog.getExistingDirectory(self.mainWindow)

    def read_file(self):
        for root, dirs, files in os.walk(self.file_path):
            self.root_path = root
            if files:
                for file in files:
                    file_list = file.split('.')
                    extension = file_list[-1]
                    if extension in extensions:
                        re_str1 = '^[a-z]{3}_[a-z]{3}_[a-zA-Z]+_[0-9]{3}shotInfo_v[0-9]{4}.xl'
                        # re_str2 = '^[a-z]{3}_[a-z]{3}_[a-zA-Z]+_000shotInfo_v[0-9]{4}.xl'
                        # if re.match(re_str1, file) and not re.match(re_str2, file):
                        #     self.excel_files.append(os.path.join(root, file))
                        if re.match(re_str1, file):
                            self.excel_files.append(os.path.join(root, file))

            else:
                self.message(False, '未找到文件~!')
                sys.exit()
        self.read_excel()

    # def read_excel(self):
    #     if not self.excel_files:
    #         self.message(False, '未找到Excel表格')
    #     for excel_file in self.excel_files:
    #         print(excel_file)
    #         excel_data = pd.read_excel(excel_file, header=10, dtype=object)
    #         excel_data = excel_data.dropna(axis=0, how='all')
    #         excel_data = excel_data.dropna(axis=1, how='all')
    #         excel_array = np.array(excel_data)[:-1]
    #         print(self.excel_new_data)
    #         print(excel_array)
    #         if len(self.excel_new_data) == 0:
    #             self.excel_new_data = excel_array
    #         else:
    #             self.excel_new_data = np.row_stack(self.excel_new_data, excel_array)
    #     print(self.excel_new_data)

    def read_excel(self):
        if not self.excel_files:
            self.message(False, '未找到Excel表格~!')
            sys.exit()
        self.message(True, '玩命加载中。。。。。')
        self.create_file_path()
        for excel_file in self.excel_files:
            excel_data = pd.read_excel(excel_file, header=10, dtype=object)
            excel_data = excel_data.dropna(axis=0, how='all')
            excel_array = np.array(excel_data)[:-1]
            if len(excel_array) == 0:
                continue
            # field_num = self.get_field_num(excel_array)
            field_num = self.get_field_num(excel_file)
            company_names = self.get_company(excel_array)
            for company in company_names:
                adopt_num = 0
                fail_num = 0
                for e in excel_array:
                    if e[17] == company:
                        if e[18] == 'pass':
                            adopt_num += 1
                        else:
                            fail_num += 1
                company = str(company)
                company_arr = {
                    'company_name': company,
                    'adopt_num': adopt_num,
                    'fail_num': fail_num,
                    'field': field_num
                }
                self.company_data.append(company_arr)

    def get_company(self, e_data):
        company = e_data[:, 17]
        company = list(set(company))
        new_company = []
        for c in company:
            if not pd.isna(c):
                new_company.append(c)
        return new_company

    def get_field_num(self, e_file):
        e_file_name = os.path.basename(e_file)
        result = re.findall(r"_\d{3}", e_file_name)
        result = result[0]
        field = result[1:]
        return field

    def create_data_frame(self):
        data = []
        shot_total_num = 0
        shot_adopt_num = 0
        shot_fail_num = 0
        for i in self.company_data:
            arr = []
            arr.append(i['field'])
            arr.append(i['company_name'])
            total_num = i['adopt_num'] + i['fail_num']
            shot_total_num += total_num
            shot_adopt_num += i['adopt_num']
            shot_fail_num += i['fail_num']
            arr.append(total_num)
            arr.append(i['adopt_num'])
            arr.append(i['fail_num'])
            progress = round(i['adopt_num'] / total_num, 2) * 100
            progress = int(progress)
            progress = "%s%%" % progress
            arr.append(progress)
            data.append(arr)
        arr = []
        arr.append('总')
        arr.append('')
        arr.append(shot_total_num)
        arr.append(shot_adopt_num)
        arr.append(shot_fail_num)
        shot_progress = round(shot_adopt_num / shot_total_num, 2) * 100
        shot_progress = int(shot_progress)
        shot_progress = "%s%%" % shot_progress
        arr.append(shot_progress)
        data.append(arr)
        self.new_data = data

    def run(self):
        self.open_file()
        self.read_file()
        self.create_data_frame()
        self.data_to_ex()

    def message(self, m_type, message):
        if m_type:
            QMessageBox.information(self.mainWindow, '提示', message)
        else:
            QMessageBox.critical(self.mainWindow, '错误', message)

    def data_to_ex(self):
        self.frame_data = pd.DataFrame(self.new_data, columns=['场号', '公司名称', '镜头总数', '完成数', '未完成数', '占比']).sort_values(
            by='场号')
        df = self.frame_data.style.applymap(self.set_field_color, subset=['场号']).set_properties(subset=['镜头总数'], **{
            "background-color": "#FED563"}).set_properties(subset=['完成数'], **{
            "background-color": "#66A342"}).set_properties(subset=['未完成数'], **{
            "background-color": "#FED563"}).set_properties(subset=['占比'], **{
            "background-color": "#FFFF2E"})
        df.to_excel(self.new_file_path, index=False)
        self.complete()

    def dd(self, d=''):
        print(d)
        sys.exit()

    def complete(self):
        self.message(True, '处理完成~！')
        self.open_new_file()

    def open_new_file(self):
        os.chmod(self.new_file_path, stat.S_IRWXU)
        os.system('open %s' % self.root_path)
        os.system('open %s' % self.new_file_path)
        sys.exit('结束~!')

    def get_u_field(self):
        field = list(self.frame_data.loc[:, '场号'])
        s_field = set(field)
        u_field = []
        for s in s_field:
            if field.count(s) > 1:
                u_field.append(s)
        u_field = np.unique(np.array(u_field))
        return u_field

    def set_field_color(self, col):
        u_field = self.get_u_field()
        if col in u_field:
            return "background-color: #FFFF2E"

    def create_file_path(self):
        t = time.gmtime()
        date = time.strftime("%m-%d", t)
        name = 'outsource' + str(date)
        self.new_file_path = self.root_path + '/%s.xlsx' % name

if __name__ == "__main__":
    ReadExcel().run()


import sys
import pygame
from bullet import Bullet
def check_events(ai_settings,screen,ship,bullets):
    """Respond to keypresses and mouse events."""
    for event in pygame.event.get(): # event loop
        if event.type == pygame.QUIT:
            sys.exit()
        elif event.type == pygame.KEYDOWN:
            check_keydown_events (event,ai_settings,screen,ship,bullets)
        elif event.type == pygame.KEYUP:
            check_keyup_events(event,ship)

def update_screen(ai_settings,screen,ship,alien,bullets):
    screen.fill(ai_settings.bg_color) # only takes one argument, a color
    for bullet in bullets.sprites():
        bullet.draw_bullet()
    ship.blitme() # draw the ship
    # redraw the screen during each pass through the loop
    alien.blitme()
    pygame.display.flip()

def check_keydown_events (event,ai_settings,screen,ship,bullets):
    if event.key == pygame.K_RIGHT:
        ship.moving_right = True
    elif event.key == pygame.K_LEFT:
        ship.moving_left = True
    elif event.key == pygame.K_SPACE:
        new_bullet = Bullet(ai_settings,screen,ship)
        bullets.add(new_bullet)

def check_keyup_events(event,ship):
    if event.key == pygame.K_RIGHT:
        ship.moving_right = False
    elif event.key == pygame.K_LEFT:
        ship.moving_left = False


#!/usr/bin/env python

from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from mass.render.build import build
from itertools import product

defaults = dict(
    # directory to load stimuli from
    cpo_path=None,

    # log10 mass ratio of type1:type0 blocks
    kappa=0.0,

    # amount of time in seconds to present the stimulus for
    presentation_time=6.0,

    # amount of time in seconds to show feedback for
    feedback_time=3.5,

    # labels for type 0 and type 1 blocks
    label0="0",
    label1="1",

    # colors of type 0 and type 1 blocks
    color0=None,
    color1=None,

    # seed for random number generator
    seed=1209,

    # amount to rotate the camera, in degrees
    camera_spin=180,

    # whether to render feedback
    feedback=True,

    # drop occluder after stimulus presentation
    occlude=False,

    # render the full stimulus as one video
    full_render=False
)

def make_options():
    options = []

    # stable example
    options.append({
        'condition': 'shared',
        'tag': 'stable_example',
        'cpo_path': 'mass-inference-stable-example-G',
        'seed': 10239,
        'full_render': True
    })

    # unstable example
    options.append({
        'condition': 'shared',
        'tag': 'unstable_example',
        'cpo_path': 'mass-inference-unstable-example-G',
        'seed': 10240,
        'full_render': True
    })

    # pretest
    options.append({
        'condition': 'shared',
        'tag': 'pretest',
        'cpo_path': 'mass-inference-training-G',
        'seed': 10241,
    })

    # mass example
    for r, cb in product(['0.1', '10'], [0, 1]):
        labels = ['red', 'blue']
        colors = ['#CA0020', '#0571B0']
        if (r == '10' and cb == 1) or (r == '0.1' and cb == 0):
            labels = labels[::-1]
            colors = colors[::-1]

        options.append({
            'condition': 'vfb-%s-cb%d' % (r, cb),
            'tag': 'mass_example',
            'cpo_path': 'mass-inference-example-G',
            'seed': 10242,
            'label0': labels[0],
            'label1': labels[1],
            'color0': colors[0],
            'color1': colors[1],
            'kappa': 1.0,
            'full_render': True
        })

    # experiment A
    for r, cb in product(['0.1', '10'], [0, 1]):
        labels = ['red', 'blue']
        colors = ['#CA0020', '#0571B0']
        if cb == 1:
            labels = labels[::-1]
            colors = colors[::-1]
        if r == '0.1':
            kappa = -1.0
        elif r == '10':
            kappa = 1.0

        options.append({
            'condition': 'vfb-%s-cb%d' % (r, cb),
            'tag': 'experimentA',
            'cpo_path': 'mass-inference-G-a',
            'seed': 10243,
            'label0': labels[0],
            'label1': labels[1],
            'color0': colors[0],
            'color1': colors[1],
            'kappa': kappa,
        })

    # experiment B
    for r, cb in product(['0.1', '10'], [0, 1]):
        labels = ['purple', 'green']
        colors = ['#7B3294', '#008837']
        if cb == 1:
            labels = labels[::-1]
            colors = colors[::-1]
        if r == '0.1':
            kappa = -1.0
        elif r == '10':
            kappa = 1.0

        options.append({
            'condition': 'vfb-%s-cb%d' % (r, cb),
            'tag': 'experimentB',
            'cpo_path': 'mass-inference-G-b',
            'seed': 10245,
            'label0': labels[0],
            'label1': labels[1],
            'color0': colors[0],
            'color1': colors[1],
            'kappa': kappa,
        })

    return options

def make_parser():
    parser = ArgumentParser(
        formatter_class=ArgumentDefaultsHelpFormatter)

    parser.add_argument(
        "-e", "--exp",
        required=True,
        help="Experiment version.")
    parser.add_argument(
        "-c", "--condition",
        default="*",
        help="Name of this condition.")
    parser.add_argument(
        "-t", "--tag",
        default="*",
        help="Short name to identify this stimulus set.")
    parser.add_argument(
        "-f", "--force",
        action="store_true",
        default=False,
        help="Force all tasks to be put on the queue.")

    return parser

def check(exp, val):
    if exp == "*":
        return True
    if exp == val:
        return True

if __name__ == "__main__":
    parser = make_parser()
    args = parser.parse_args()

    for opts in make_options():
        cond_ok = check(args.condition, opts['condition'])
        tag_ok = check(args.tag, opts['tag'])

        if cond_ok and tag_ok:
            params = defaults.copy()
            params.update(opts)
            params['exp'] = args.exp
            params['force'] = args.force
            build(**params)


class Node:
    def __init__(self, value=None, pointer=None):
        self.value = value
        self.pointer = pointer

class LinkedStack:
    def __init__(self):
        self.head = None

    def isEmpty(self):
        return not bool(self.head)

    def push(self, item):
        self.head = Node(item, self.head)

    def pop(self):
        if self.head:
            node = self.head
            self.head = node.pointer
            return node.value
        else:
            print('Stack is empty')

    def size(self):
        node = self.head
        count = 0
        while node:
            count += 1
            node = node.pointer

    def _printList(self):
        node = self.head
        while node:
            print(node.value)
            node = node.pointer

if __name__ == '__main__':
    stack = LinkedStack()
    for i in range(15,20):
        stack.push(i)
    print("Stack:")
    stack._printList()
    for i in range(10, 5, -1):
        stack.push(i)
    print("Stack:")
    stack._printList()
    for i in range(1, 13):
        stack.pop()
    print("Stack:")
    stack._printList()

    stack.push(25)
    stack.push(3)
    stack.push(11)
    stack.push(31)
    stack.push(29)
    print("Stack:")
    print(stack._printList())


from math import sqrt

def calculate_accuracy(predictedLabels, correctLabels):
    # accuracyCouter = 0
    # if predictedLabel == correctLabel:
    #     accuracyCouter += 1

    correctPredictions = {k: predictedLabels[k] for k in predictedLabels if
                              k in correctLabels and predictedLabels[k] == correctLabels[k]}
    print("Total Number of Correct Predictions: ", len(correctPredictions))

    correctPredictionCount = len(correctPredictions)

    accuracy = (correctPredictionCount/ (len(predictedLabels))) * 100
    return accuracy * 0.01

#save pedictions and groundtruth as dictionaries
#key = time , value = label

predictions = {55:"cutIn", 76: "cutOut", 90: "end", 108: "end"}
groundTruth = {55:"cutIn", 76: "cutIn", 90: "end", 108: "end", 200: "end"}

# accuracy = total correct predictions / total predictions made * 100
accuracy = calculate_accuracy(predictions, groundTruth)
print("This is the accuracy:", accuracy)
z = 1.64  #number of standard deviations from the gaussian distribution (90%)
n = 500 #sample size
interval = z * sqrt( (accuracy * (1 - accuracy)) / n)  # binomial confidence interval

print("Interval: ", interval)


import numpy as np
from toolbox import listOfTuples

"""
Antenna Array Model
Authors: Jacek Kibilda
"""

def positionAntennaElements(nx,ny,nz,dx,dy,dz):
    nx_vec = np.arange(nx)
    ny_vec = np.arange(ny)
    nz_vec = np.arange(nz)

    s = (nx*ny*nz,3)
    element_array = np.zeros(s)

    #xPosition
    xPos = dx * nx_vec
    element_array[:,0] = np.tile(xPos, ny * nz)

    #yPosition
    yPos = dy * ny_vec
    element_array[:,1] = np.tile(np.tile(yPos, (nx,1)).T.reshape((nx * ny,1)),(nz,1)).T

    #zPosition
    zPos = dz * nz_vec
    element_array[:,2] = np.tile(zPos, (nx * ny,1)).T.reshape((nx * ny * nz,1)).T

    return element_array

class Omnidirectional:
    def __init__(self, lamb, nx, ny, nz, dx, dy, dz):
        self.lamb = lamb
        self.nx = nx
        self.ny = ny
        self.nz = nz
        self.dx = dx
        self.dy = dy
        self.dz = dz
        return

    def calc_array_factor(self, theta, phi):
        n = self.nx * self.ny * self.nz
        return np.ones(n)

#This codebook is generated for the array in y-z plane
#The codebook generates a beam on a grid that spans vAngMin and vAngMax (vertical angles), and hAngMin and hAngMax (horizontal angles); vBeams denotes the number of vertical beams, and hBeams the number of horizontal beams
#Default values correspond to the MHU values
def codebook_generator(wavelen,nx,ny,nz,dx,dy,dz,element_array,hBeams=9,vBeams=7,hAngMin=-45,hAngMax=45,vAngMin=-35,vAngMax=35):

    k = 2 * np.pi / wavelen

    # Defining the RF beamforming codebook in the x-direction
    codebook_size_x = nx
    codebook_size_y = ny
    codebook_size_z = nz

    #NOTE: theta and phi are swapped in definition here to follow the convention used by the MHU

    # MHU - np.linspace(-45,45,9) Azimuth
    phi = np.linspace(hAngMin,hAngMax,hBeams)

    # MHU - np.linspace(-35,35,7) Elevation; boresight is (0,0) so we need to translate to our domain where boresight is (90,0)
    theta = np.linspace(vAngMin,vAngMax,vBeams) + 90

    num_beams = hBeams + vBeams

    #Prepare vectors that allocate an angle to each antenna element
    theta_tile = np.tile(theta, (hBeams,1)).T.flatten()
    thetaRad = np.deg2rad(theta_tile)
    phi_tile = np.tile(phi, vBeams)
    phiRad = np.deg2rad(phi_tile)

    beam_angles = listOfTuples(theta_tile,phi_tile)

    phase_shift = None
    #Indices in the range 1:63; the index of antenna beam at boresight is 32
    all_beams = np.arange(num_beams) + 1

    return phase_shift, all_beams, beam_angles

class AntennaArray:
    def __init__(self, wavelen, nx, ny, nz, dx, dy, dz, hbeams=0, vbeams=0,
            hangmin=0, hangmax=0, vangmin=0, vangmax=0, elem_type='isotropic'):
        self.wavelen = wavelen
        self.nx = nx
        self.ny = ny
        self.nz = nz
        self.dx = dx
        self.dy = dy
        self.dz = dz

        #Antenna Element type; default is 'isotropic'; other models are currently not supported
        self.elem_type = elem_type

        self.element_array = positionAntennaElements(nx, ny, nz, dx, dy, dz)

        if hbeams * vbeams > 0:
            self.codebook, self.beam_ids, self.beam_angles = \
                codebook_generator(wavelen, nx, ny, nz, dx, dy, dz,
                self.element_array, hbeams, vbeams, hangmin, hangmax, vangmin,
                vangmax)
            self.codebook_ids = np.arange(len(self.beam_angles))
        else:
            self.codebook = self.beam_ids = self.beam_angles = self.codebook_ids = None

        return

    def calc_array_factor(self, theta, phi):
        thetaRad = theta#np.deg2rad(theta)
        phiRad = phi#np.deg2rad(phi)

        k = 2 * np.pi / self.wavelen

        n = self.nx * self.ny * self.nz

        a = np.ones(n)/np.sqrt(n) #normalize the output

        x_component = k * self.element_array[:,0] * np.cos(phiRad) * np.sin(thetaRad)
        y_component = k * self.element_array[:,1] * np.sin(phiRad) * np.sin(thetaRad)
        z_component = k * self.element_array[:,2] * np.cos(thetaRad)

        w = np.exp(1j * x_component.T + 1j * y_component.T + 1j * z_component.T)

        aw = a * w

        return aw, phiRad

    def _calc_steering_vector_magnitude(self, wd, ptxi):
        """
        This functions returns the gain (magnitude factor) of the directional
        antenna described by the steering vector wd compared to the isotropic
        antenna that radiates ptxi watts at every direction.
        """
        m  = int(self.nx*self.ny*self.nz)
        wi = np.sqrt(ptxi)*np.ones((m, 1))
        g  = np.linalg.norm(wd.T @ wi)
        return g

    def _calc_steering_vector_angles(self, theta_rad, phi_rad):
        """
        This function returns the steering vector with the phase information
        of each element.
        """
        m = int(self.nx * self.ny * self.nz)
        k = 2*np.pi/self.wavelen

        x_component = k*self.element_array[:,0]*np.cos(phi_rad)*np.sin(theta_rad)
        y_component = k*self.element_array[:,1]*np.sin(phi_rad)*np.sin(theta_rad)
        z_component = k*self.element_array[:,2]*np.cos(theta_rad)

        w = np.exp(1j*x_component.T + 1j*y_component.T + 1j*z_component.T)
        w = w/np.sqrt(m) # Normalization by the number of elements.

        # NB.: w.shape is (m, 1) because we assume a linear antenna array.
        return w.reshape((m, 1))

    def steering_vec(self, beam_id, phi=0, ptxi=1):
        '''
        Input:
            - beam_id: The codebook id that will dictated the beam direction;
            - relative_phis: The relative angles (horizontal axis) corresponding
                to the location of receivers.
        Output:
            - aw: The radiation pattern
        Returns:
            - Codebook: Vector with the radition pattern corresponding to beam_id.
            - Beam angles: Humane beam information corresponding to the beam_id
                and thus the radiation vector within the codebook.
        '''
        # Validation of the setup.
        err_msg = "Error! You have not initialized the codebook. "
        err_msg += "Please, make sure hbeams and vbeams > 0."
        assert len(self.beam_angles) > 0, err_msg
        # Vertical and horizontal angles to where the beam is pointing.
        beam_theta, beam_phi = self.beam_angles[beam_id]
        theta_rad = np.deg2rad(beam_theta)
        phi_rad = np.deg2rad(phi-beam_phi)
        wd = self._calc_steering_vector_angles(theta_rad, phi_rad)
        g = self._calc_steering_vector_magnitude(wd, ptxi)
        aw = wd*g
        return aw, self.beam_angles[beam_id]


import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

root = 'C:/Users/Dan/Desktop/'

f = 'BookF.csv'

df = pd.read_csv(root+f)

freq = df['Freq(MHz)']
phase = df['phase(ns)']
phasef = phase/freq
amp = df['Amplitude(mV)']

plt.plot(freq, amp/np.max(amp), lw=5, c='r')


from unittest.mock import patch

from django.db import IntegrityError
from django.test import TestCase
from django.utils import timezone

from core import factories, models
from core.tasks import process_turn
from core.tests import DiplomacyTestCaseMixin
from core.models.base import DeadlineFrequency, Phase, Season

process_path = 'core.tasks._process_turn'

class TestTurnEnd(TestCase, DiplomacyTestCaseMixin):

    def setUp(self):
        self.variant = models.Variant.objects.get(id='standard')
        self.user = factories.UserFactory()
        self.game = models.Game.objects.create(
            name='Test game',
            variant=self.variant,
            num_players=7,
            created_by=self.user,
            order_deadline=DeadlineFrequency.TWENTY_FOUR_HOURS,
            retreat_deadline=DeadlineFrequency.TWELVE_HOURS,
            build_deadline=DeadlineFrequency.TWELVE_HOURS,
        )
        self.patch_process_turn_apply_async()
        self.patch_revoke_task_on_delete()

    def create_turn(self):
        return models.Turn.objects.create(
            game=self.game,
            phase=Phase.ORDER,
            season=Season.SPRING,
            year=1900,
        )

    def test_no_turn_end_on_created_turn(self):
        turn = self.create_turn()
        self.assertEqual(0, len(models.TurnEnd.objects.all()))
        self.assertIsNone(turn.turn_end)

    def test_turn_end_creation(self):
        turn = self.create_turn()
        datetime = timezone.now()

        turn_end = models.TurnEnd.objects.new(turn=turn, datetime=datetime)
        self.assertIsInstance(turn_end, models.TurnEnd)
        self.assertEqual(self.dummy_task_id, turn_end.task_id)

        self.apply_async.assert_called_with(
            kwargs={'turn_id': turn.id, 'processed_at': datetime},
            eta=datetime,
        )
        self.assertEqual(turn_end, turn.turn_end)

    def test_process_outcome(self):
        turn = self.create_turn()
        datetime = timezone.now()

        models.TurnEnd.objects.new(turn=turn, datetime=datetime)

        process_turn(turn_id=turn.id, processed_at=datetime)
        turn.refresh_from_db()

        self.assertTrue(turn.processed)
        self.assertSimilarTimestamp(turn.processed_at, datetime)
        self.assertEqual(1, len(models.TurnEnd.objects.all()))

    def test_process_standalone(self):
        turn = self.create_turn()
        datetime = timezone.now()
        process_turn(turn_id=turn.id, processed_at=datetime)
        turn.refresh_from_db()
        self.assertTrue(turn.processed)
        self.assertSimilarTimestamp(turn.processed_at, datetime)

    def test_turn_end_duplication(self):
        turn = self.create_turn()
        datetime = timezone.now()
        tomorrow = datetime + timezone.timedelta(days=1)

        models.TurnEnd.objects.new(turn=turn, datetime=datetime)

        with self.assertRaises(IntegrityError):
            models.TurnEnd.objects.new(turn=turn, datetime=tomorrow)

        # Assert that only one process_turn() task was invoked, not two.
        self.apply_async.assert_called_once_with(
            kwargs={'turn_id': turn.id, 'processed_at': datetime},
            eta=datetime,
        )

    def test_turn_end_graceful_fail(self):
        turn = self.create_turn()
        datetime = timezone.now()

        models.TurnEnd.objects.new(turn=turn, datetime=datetime)

        with patch(process_path) as process:
            process.side_effect = models.Turn.DoesNotExist
            process_turn(turn_id=turn.id, processed_at=datetime)
        self.assertEqual(0, len(models.TurnEnd.objects.all()))


infile = "C-small-attempt2.in"
outfile = "C-small.out"

lines = [l.strip() for l in open(infile,"r")]
from collections import defaultdict, Counter
from functools import partial
from itertools import combinations

num_cases = int(lines[0])
lines = lines[1:]

cases = [[int(x) for x in lines[i].split()] for i in range(num_cases)]

def transpose(matrix):
    num_rows = len(matrix)
    num_cols = len(matrix[0])
    return [[matrix[i][j] for i in range(num_rows)] for j in range(num_cols)]

def neighbors(i,j,rows,cols):
    go_up = i > 0
    go_down = i < rows - 1
    go_left = j > 0
    go_right = j < cols - 1

    if go_up:
        yield (i-1,j)
        if go_right: yield (i-1,j+1)
        if go_left: yield (i-1,j-1)
    if go_down:
        yield (i+1,j)
        if go_right: yield(i+1,j+1)
        if go_left: yield(i+1,j-1)
    if go_left:
        yield (i, j - 1)
    if go_right:
        yield (i, j+1)

def create_board(rows, cols, num_pads, mine_locs, double_pad=False):

    actual_rows = [[['.', # me
               0, # adjacent_mines
               False] # explored
              for j in range(cols)] for i in range(rows)]
    # need to add adjacency to last row if padded:
    if num_pads > 0:
        for cell in actual_rows[-1]:
            cell[1] += 1
    pad_rows = [[['*',None,True] for j in range(cols)] for i in range(num_pads)]
    board = actual_rows + pad_rows

    if double_pad:
        for i, row in enumerate(board):
            row[-1][1] = 1
            board[i] = row + [['*',None,True] for j in range(num_pads)]

    for i, j in mine_locs:
        board[i][j][0] = "*"
        for ii, jj in neighbors(i,j,rows,cols):
            board[ii][jj][1] += 1

    return board

from collections import deque

def try_solve(board, rows, cols, num_mines, click):

    i,j = click
    board[i][j][0] = 'c'
    revealed = 0
    queue = deque([click])
    while queue:
        i, j = queue.popleft()
        cell = board[i][j]

        if cell[2]: continue # already visited

        # mark as visited
        cell[2] = True
        revealed += 1

        if cell[1] > 0:
            continue

        # otherwise 0 mines
        # queue up all unvisitedneighbors
        for (ii,jj) in neighbors(i,j,rows,cols):
            if not board[ii][jj][2]:
                queue.append((ii,jj))

    success = revealed + num_mines == rows * cols
    return success, board

def solve(rows, cols, pad_cols, mines, double_pad = False):
    #print "solve"
    coordinates = [(i,j) for i in range(rows) for j in range(cols)]

    #print coordinates
    non_mines = rows * cols - mines

    if non_mines >= mines:

        mine_locations = combinations(coordinates,mines)

        if mines > 0:

            for locs in mine_locations:
                loc_set = set(locs)
                for click in coordinates:
                    if click not in loc_set:
                        board = create_board(rows, cols, pad_cols, locs, double_pad)
                        success, grid = try_solve(board, rows, cols, mines, click)

                        if success:
                            #print "success", click
                            return grid

            return None

        else: # mines = 0

            #print "mines == 0"

            for click in coordinates:
                board = create_board(rows, cols, pad_cols, [], double_pad)
                #print board
                success, grid = try_solve(board, rows, cols, mines, click)

                if success:
                    #print "success", click
                    return grid
            return None

    else: # mines > non_mines

        #print "allocating",non_mines,"non-mines instead"
        non_mine_locations = combinations(coordinates, non_mines)
        for non_mine_locs in non_mine_locations:
            non_mine_loc_set = set(non_mine_locs)
            locs = [pair for pair in coordinates if pair not in non_mine_loc_set]
            loc_set = set(locs)
            for click in non_mine_locs:
                board = create_board(rows, cols, pad_cols, locs, double_pad)
                success, grid = try_solve(board, rows, cols, mines, click)

                if success:
                    #print "success", click
                    return grid

def pretty(grid, need_to_transpose):
    if grid is None:
        return "Impossible"

    if need_to_transpose:
        return "\n".join(["".join([cell[0] for cell in row]) for row in transpose(grid)])
    else:
        return "\n".join(["".join([cell[0] for cell in row]) for row in grid])

def double_pad(case):
    rows, cols, mines = case

    num_pads = 0

    while mines >= cols + rows - 1:
        num_pads += 1
        mines -= (cols + rows - 1)
        rows -= 1
        cols -= 1

    #print rows, cols, num_pads, mines
    solution = solve(rows, cols, num_pads, mines, double_pad = True)

    return solution

def pro_case(case, need_to_transpose=False, use_padding=True):

    rows, cols, mines = case
    #print
    #print rows, cols, mines

    # want to chew up rows, i.e. want cols <= rows

    if need_to_transpose:
        rows, cols = cols, rows

    # now cols <= rows for sure

    num_pads = 0

    if use_padding:
        while mines >= cols:
            num_pads += 1
            mines -= cols
            rows -= 1

    solution = solve(rows, cols, num_pads, mines)

    return solution

def process_case(case):
    # first try non-transposed:
    rows, cols, mines = case

    result = pro_case(case,False)
    if result is not None:
        return pretty(result,False)

    transposed_result = pro_case(case,True) if rows != cols else None
    if transposed_result is not None:
        return pretty(transposed_result,True)

    double_pad_result = double_pad(case)
    if double_pad_result is not None:
        return pretty(double_pad_result,False)

    #    if result is not None:
    #        print "double_pad succeeded", case
    #        print pretty(result,False)
    #    else:
    #        #print "trying to brute", case
    #brute_force_result = pro_case(case,use_padding=False)
    #        if result is not None:
    #            print "brute force succeeded!", case
    #            print pretty(result,false)

    #normal = result is not None
    #transposed = transposed_result is not None
    #double_padded = double_pad_result is not None
    #fast_way = any([normal, transposed, double_padded])
    #brute_forced = brute_force_result is not None

    #print "(",normal, transposed,double_padded,")",brute_forced,"-->",(fast_way == brute_forced)

    #if fast_way != brute_forced:
    #    print "failure!!!!!!!"

    return pretty(result,False)

with open(outfile,"w") as g:

    for i, case in enumerate(cases):
        #print i, case
        result = process_case(case)
        g.write("Case #" + str(i + 1) + ":\n")
        #g.write(str(case) + "\n")
        g.write(result + "\n")


# -*- coding:utf-8 -*-
# 创建时间：2019年7月18日
# 功能：用前置摄像头检测黑色矩形，返回矩形中心点坐标
# 实现方法：使用黑色为依据识别
# 注：矩形框宽100cm，高70cm；边框为黑色，边框宽度不大于6cm，
#    矩形框下边框距地面110cm
# 修改：2019年7月23日：1.重新调整好参数
#                     2.输出识别物体中心坐标
#                     3.添加错误识别函数；

import cv2
import numpy as np
import time

# 初始化函数
def Init():
    # cap = cv2.VideoCapture(0)
    cap = cv2.VideoCapture('./2.mp4')
    cap.set(3,720)  # 设置摄像头输出宽
    cap.set(4,720)  # 设置摄像头输出高
    # 设置摄像头分辨率
    cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
    cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 240)
    print("start reading video...")
    time.sleep(2.0)
    print("start working")

    return cap

cap = Init()  # 初始化

while cap.isOpened():
    ret, frame = cap.read()
    frame_copy = frame.copy()
    hsv_frame = cv2.cvtColor(frame,cv2.COLOR_BGR2HSV)  # 将每一帧图片转化HSV空间颜色
    hsv_frame_after = cv2.GaussianBlur(hsv_frame,(19,19),0) # 高斯滤波进行平滑处理

    # 依据之前的脚本获取的阈值设置最高值与最低值
    lower = np.array([0, 0, 134])
    upper = np.array([82, 88, 225])

    mask = cv2.inRange(hsv_frame_after, lower, upper)  # 构建掩膜
    # thresh = cv2.threshold(hsv_frame_after, 60, 255, cv2.THRESH_BINARY)[1]  # 进行闸值化
    conts, hier = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)  # 找出边界
    cv2.drawContours(frame, conts, -1, (0, 255, 0), 3)  # 画出边框
    # 计算出中心坐标并输出
    if conts:
        cnts = conts[0]
        M = cv2.moments(cnts)
        cX = int(M["m10"] / (M["m00"] + 0.0001))
        cY = int(M["m01"] / (M["m00"] + 0.0001))
        if cX != 0 and cY != 0:
            print('中心坐标',cX,cY)
        if cX == 0 or cY == 0:
            print('识别有误！')

    dst = cv2.bitwise_and(frame, frame, mask=mask)  # 对每一帧进行位与操作，滤去除要目标物体之外的图像

    cv2.imshow('the original video',frame_copy)
    cv2.imshow('test',dst)

    # 按下esc键退出
    key = cv2.waitKey(delay=2)
    if key == ord("q") or key == 27:
        break

cap.release()
cv2.destroyAllWindows()


from flask import Flask, render_template, request, redirect, url_for, send_from_directory
from flask_wtf import FlaskForm
from wtforms import StringField, TextAreaField
from transformers import pipeline

sequence = """ The traffic began to slow down on Pioneer Boulevard in Los Angeles, making it difficult
to get out of the city. However, WBGO was playing some cool jazz, and the weather was cool, making it
rather pleasant to be making it out of the city on this Friday afternoon. Nat King Cole was singing as
Jo and Maria slowly made their way out of LA and drove toward Barstow. They planned to get to Las Vegas
early enough in the evening to have a nice dinner and go see a show. """

# sequence = "The traffic began to slow down on Pioneer Boulevard in Los Angeles, making it difficult to get out of the city. However, WBGO was playing some cool jazz, and the weather was cool, making it rather pleasant to be making it out of the city on this Friday afternoon. Nat King Cole was singing as Jo and Maria slowly made their way out of LA and drove toward Barstow. They planned to get to Las Vegas early enough in the evening to have a nice dinner and go see a show."
# question = 'Where is Pioneer Boulevard ?'
question = 'What is playing?'

class Form(FlaskForm):
    text = TextAreaField(render_kw={"rows": 8})
    question = StringField(label='What\'s your question?:', description='sdf')

def question_answering_route():
    form = Form()
    answer = {"answer": "N/A", "score": "N/A"}
    if form.validate_on_submit():
        nlp_qa = pipeline('question-answering')
        answer = nlp_qa(context=form.text.data or sequence, question=form.question.data or question)
        return redirect(url_for('question_answering', form=form, answer=answer["answer"]))

    return render_template('question-answering.html', form=form, answer=answer)

def answer_question():
    args = request.args
    nlp_qa = pipeline('question-answering')
    answer = nlp_qa(context=args["text"] or sequence, question=args["question"] or question)

    return answer


def selection_sort(arr):
    for i in range(len(arr)):
        minimum = i

        for j in range(i + 1, len(arr)):
            # Select the smallest value
            if arr[j] < arr[minimum]:
                minimum = j

        # Place it at the front of the
        # sorted end of the array
        arr[minimum], arr[i] = arr[i], arr[minimum]

    return arr

arr1 = [64, 34, 25, 12, 22, 11, 90]
arr2 = [64, 34, 25, 12, 22, 11]
arr3 = [-64, 34, 25]
arr4 = []

selection_sort(arr1)
selection_sort(arr2)
selection_sort(arr3)
selection_sort(arr4)

print ("Sorted array1 is:")
print(arr1)

print ("Sorted array2 is:")
print(arr2)

print ("Sorted array3 is:")
print(arr3)

print ("Sorted array4 is:")
print(arr4)

cards = ['5♣', '8♠', '4♠', '9♣', 'K♣', '6♣', '5♥', '3♣', '8♥', 'A♥', 'K♥', 'K♦', '10♣', 'Q♣', '7♦', 'Q♦', 'K♠', 'Q♠', 'J♣', '5♦', '9♥', '6♦', '2♣', '7♠', '10♠', '5♠', '4♣', '8♣', '9♠', '6♥', '9♦', '3♥', '3♠', '6♠', '2♥', '10♦', '10♥', 'A♠', 'A♣', 'J♥', '7♣', '4♥', '2♦', '3♦', '2♠', 'Q♥', 'A♦', '7♥', '8♦', 'J♠', 'J♦', '4♦']
selection_sort(cards)
print("Sorted cards:" )
print(cards)


infile = open('data.txt', 'r')
text_data = infile.readlines()
vals = []
for element in text_data:
  vals.append(element.split())

total = 0
for policy in vals:
  nums = policy[0].split('-')
  mi = int(nums[0])
  ma = int(nums[1])
  if mi <= policy[2].count(policy[1][0]) <= ma:
    total += 1

print(total)


# -*- coding: utf-8 -*-
""""""""""""""""""""" MODULE IMPORT """""""""""""""""""""
### Import external modules
import numpy as np
import matplotlib.pyplot as plt
import os
import sys
from scipy.signal import savgol_filter

### Import internal modules
sys.path.insert(0, os.path.realpath('../Classes'))
sys.path.insert(0, os.path.realpath('../Functions'))
sys.path.insert(0, os.path.realpath('Classes'))
sys.path.insert(0, os.path.realpath('Functions'))
from LoadData import LoadData
from peak_det import peakdet

""""""""""""""""""""" FUNCTION """""""""""""""""""""
def estimate_OU_par_from_signal(ll_signal, W, gamma_A = 0.03, gamma_B = 0.03):
    """
    Estimate mean and variance of OU processes from a given set of circadian
    traces.

    Parameters
    ----------
    W : list
        Waveform.
    gamma_A : float
        Regression parameter for the amplitude.
    gamma_b : float
        Regression parameter for the background.

    Returns
    -------
    The mean and standard deviations of the amplitude and the background.
    """

    ##################### SMOOTH SIGNAL ################
    ll_signal = [savgol_filter(l_signal, 21, 3) for l_signal in ll_signal]

    ##################### COMPUTE AMPLITUDE WAVEFORM ################
    if W is None:
        amp_waveform = 1
    else:
        amp_waveform = np.max(W)-np.min(W)

    ##################### LOOK FOR MAXIMA/MINIMA ################
    l_min_spikes = []
    l_max_spikes = []
    l_amp_spikes = []
    for i, l_signal in enumerate(ll_signal):
        l_max, l_min = peakdet(l_signal, 0.3)
        if l_max.shape[0]<2 or l_min.shape[0]<1:
            continue

        ##################### REMOVE ARTIFACT ################
        idx=0
        to_del_lmax=[]
        to_del_lmin=[]
        for t1, t2, t3 in zip(l_max[:,0],l_min[:,0], l_max[1:,0]) :
            if t3-t1<14 or abs(t1-t2)<7 or abs(t3-t2)<7:
                if l_max[idx,1]>l_max[idx+1,1]:
                    to_del_lmax.append(idx+1)
                else:
                    to_del_lmax.append(idx)
                to_del_lmin.append(idx)
            idx+=1
        l_max = [v for i,v in enumerate(list(l_max)) if i not in to_del_lmax]
        l_min = [v for i,v in enumerate(list(l_min)) if i not in to_del_lmax]

        if len(l_min)>0:
            l_min_spikes.extend( np.array(l_min)[:,1] )
        if len(l_max)>0:
            l_max_spikes.extend( np.array(l_max)[:,1] )

        if len(l_max)<2 or len(l_min)<1:
            continue

        ##################### PLOT MIN/MAX IDENTIFICATION ################
        l_max = np.array(l_max)
        l_min = np.array(l_min)
        if i<0:
            plt.scatter(l_max[:,0], l_max[:,1], color='blue')
            plt.scatter(l_min[:,0], l_min[:,1], color='red')
            plt.plot(l_signal)
            #for idx, v  in enumerate(l_peak[i]):
                #if v!=0:
                #    plt.axvline(idx)
            plt.show()
            plt.close()

        ##################### COMPUTE AMPLITUDES ################
        first_max=False
        last_max=False
        if l_max[0,0]<l_min[0,0]:
            first_max=True
        if l_max[-1,0]>l_min[-1,0]:
            last_max=True

        #create a list of spike amplitude
        l_amplitude = []
        if l_max.shape[0]>=2 and l_min.shape[0]>=2:
            if first_max:
                for min1, max1, min2 in zip(l_min[:,1], l_max[1:,1],
                                                                  l_min[1:,1]):
                    l_amplitude.append( max1-(min2+min1)/2)
            else:
                for min1, max1, min2 in zip(l_min[0:,1], l_max[0:,1],
                                                                  l_min[1:,1]):
                    l_amplitude.append( max1-(min2+min1)/2)

        elif l_max.shape[0]==1 and l_min.shape[0]==2:
            l_amplitude.append( l_max[0,1]-(l_min[1,1]+l_min[0,1])/2     )

        if i==-1:
            print(l_amplitude)
        if len(l_amplitude)>0:
            l_amp_spikes.extend(l_amplitude)

    ##################### REMOVE TOO SMALL AMPLITUDES (BUGS) ################
    l_amp_spikes = [a for a in l_amp_spikes if a>0.3]

    #compute stationnary mean and variance for amplitude
    l_log_amp_spikes = np.log(np.array(l_amp_spikes)/amp_waveform)
    var_A_stat = np.var(l_log_amp_spikes)
    std_A = (var_A_stat * 2*gamma_A)**0.5
    mu_A = np.mean(l_log_amp_spikes)

    var_B_stat = np.var(l_min_spikes)
    std_B = (var_B_stat * 2*gamma_B)**0.5
    mu_B = np.mean(l_min_spikes)

    '''
    print("mu_A =", mu_A)
    print("std_A =", std_A)
    print("mu_B =", mu_B)
    print("std_B =", std_B)
    '''

    return mu_A, std_A, mu_B, std_B

def estimate_OU_par(cell, temperature, W = None, gamma_A = 0.03, gamma_B =0.03):
    """
    Estimate mean and variance of OU processes given a set of conditions,
    according to which a set of traces is filtered.

    Parameters
    ----------
    cell : string
        Cell type.
    temperature : integer
        Temperature condition.
    W : list
        Waveform.
    gamma_A : float
        Regression parameter for the amplitude.
    gamma_b : float
        Regression parameter for the background.

    Returns
    -------
    The mean and standard deviations of the amplitude and the background.
    """
    ######### CORRECTION BECAUSE NOT ENOUGH TRACES AT 34°C AND 40°C #########
    print('CAUTION : Parameters for None temperature selected since not enough \
            traces at 34°C and 40°C')
    temperature = None

    ##################### LOAD DATA ################
    if cell == 'NIH3T3':
        path =  "Data/NIH3T3.ALL.2017-04-04/ALL_TRACES_INFORMATION.p"
        dataClass=LoadData(path, 10000000, temperature = temperature,
                            division = False)
    elif cell == 'U2OS':
        path = "Data/U2OS-2017-03-20/ALL_TRACES_INFORMATION_march_2017.p"
        dataClass=LoadData(path, 10000000, temperature = temperature,
                            division = True)

    try:
        (ll_area, ll_signal, ll_nan_circadian_factor, ll_obs_phi, ll_peak,
        ll_idx_cell_cycle_start, T_theta, T_phi) = \
                                            dataClass.load(load_annotation=True)
    except:
        dataClass.path = '../'+dataClass.path
        (ll_area, ll_signal, ll_nan_circadian_factor, ll_obs_phi, ll_peak,
        ll_idx_cell_cycle_start, T_theta, T_phi) = \
                                            dataClass.load(load_annotation=True)

    return estimate_OU_par_from_signal(ll_signal, W, gamma_A, gamma_B)

""""""""""""" TEST """""""""""""
if __name__ == '__main__':
    os.chdir('..')

    print("NIH3T3 34°C no coupling:")
    mu_A_34, std_A_34, mu_B_34, std_B_34 = estimate_OU_par(cell = "NIH3T3",
                                                            temperature = 34)
    print("NIH3T3 37°C no coupling:")
    mu_A_37, std_A_37, mu_B_37, std_B_37 = estimate_OU_par(cell = "NIH3T3",
                                                             temperature = 37)
    print("NIH3T3 40°C no coupling:")
    mu_A_40, std_A_40, mu_B_40, std_B_40 = estimate_OU_par(cell = "NIH3T3",
                                                             temperature = 40)

    ### PLOT RESULTS ###
    plt.errorbar([34,37,40], [mu_A_34, mu_A_37, mu_A_40],
                                yerr = [std_A_34, std_A_37, std_A_40], fmt='o')
    plt.xlim([33,41])
    plt.xlabel("Temperature")
    plt.ylabel("Amplitude mean and deviation")
    plt.savefig('Results/RawData/OU_A_NIH3T3.pdf')
    plt.show()
    plt.close()

    plt.errorbar([34,37,40], [mu_B_34, mu_B_37, mu_B_40],
                                yerr = [std_B_34, std_B_37, std_B_40], fmt='o')
    plt.xlim([33,41])
    plt.xlabel("Temperature")
    plt.ylabel("Background mean and deviation")
    plt.savefig('Results/RawData/OU_B_NIH3T3.pdf')
    plt.show()
    plt.close()

    print("U20S 34°C no coupling:")
    mu_A_34, std_A_34, mu_B_34, std_B_34 = estimate_OU_par(cell = "U2OS",
                                                            temperature = 34)
    print("U20S 37°C no coupling:")
    mu_A_37, std_A_37, mu_B_37, std_B_37 = estimate_OU_par(cell = "U2OS",
                                                            temperature = 37)
    print("U20S 40°C no coupling:")
    mu_A_40, std_A_40, mu_B_40, std_B_40 = estimate_OU_par(cell = "U2OS",
                                                            temperature = 40)

    ### PLOT RESULTS ###
    plt.errorbar([34,37,40], [mu_A_34, mu_A_37, mu_A_40],
                                yerr = [std_A_34, std_A_37, std_A_40], fmt='o')
    plt.xlim([33,41])
    plt.xlabel("Temperature")
    plt.ylabel("Amplitude mean and deviation")
    plt.savefig('Results/RawData/OU_A_U20S.pdf')
    plt.show()
    plt.close()

    plt.errorbar([34,37,40], [mu_B_34, mu_B_37, mu_B_40],
                                yerr = [std_B_34, std_B_37, std_B_40], fmt='o')
    plt.xlim([33,41])
    plt.xlabel("Temperature")
    plt.ylabel("Background mean and deviation")
    plt.savefig('Results/RawData/OU_B_U20S.pdf')
    plt.show()
    plt.close()


import numpy as np
from scipy import spatial

def getNNlist(filePos,fileCel,Tk,i):
    #
    T              = ['100','200','300','500','1000']
    # i+1 indexing (1,...,10). So for input i=0, convert to i=0+1
    with open(filePos+T[Tk]+"K_"+str(i+1)+".dat") as file:
        dataPos    = [[float(digit) for digit in line.split()] for line in file]
    nat            = len(dataPos)
    natUnit        = len(dataPos)/5 # = 27
    #
    dataCel  = [[0.0,0.0,0.0] for y in range(0,3)]
    map27to1 = [ 0 for y in range(0,nat*27)]
    fp = open(fileCel+T[Tk]+"K_"+str(i+1)+".cel")
    for i, line in enumerate(fp):
        if i >=2:
            dataCel[i-2] = [float(digit) for digit in line.split()]
    fp.close()
    dataCel = np.asarray(dataCel)
    #
    dataPos27 = [[0.0,0.0,0.0] for y in range(0,len(dataPos)*27)]
    dataPos27 = np.asarray(dataPos27)
    for i in range(0,len(dataPos)):
        dataPos27[i+ 1*nat]=dataPos[i] + 1.0*dataCel[0] - 1.0*dataCel[1] - 1.0*dataCel[2]
        dataPos27[i+ 2*nat]=dataPos[i] + 1.0*dataCel[0]                  - 1.0*dataCel[2]
        dataPos27[i+ 3*nat]=dataPos[i] + 1.0*dataCel[0] + 1.0*dataCel[1] - 1.0*dataCel[2]
        dataPos27[i+ 4*nat]=dataPos[i]                  - 1.0*dataCel[1] - 1.0*dataCel[2]
        dataPos27[i+ 5*nat]=dataPos[i]                                   - 1.0*dataCel[2]
        dataPos27[i+ 6*nat]=dataPos[i]                  + 1.0*dataCel[1] - 1.0*dataCel[2]
        dataPos27[i+ 7*nat]=dataPos[i] - 1.0*dataCel[0] - 1.0*dataCel[1] - 1.0*dataCel[2]
        dataPos27[i+ 8*nat]=dataPos[i] - 1.0*dataCel[0]                  - 1.0*dataCel[2]
        dataPos27[i+ 9*nat]=dataPos[i] - 1.0*dataCel[0] + 1.0*dataCel[1] - 1.0*dataCel[2]
        #
        dataPos27[i+10*nat]=dataPos[i] + 1.0*dataCel[0] - 1.0*dataCel[1]
        dataPos27[i+11*nat]=dataPos[i] + 1.0*dataCel[0]
        dataPos27[i+12*nat]=dataPos[i] + 1.0*dataCel[0] + 1.0*dataCel[1]
        dataPos27[i+13*nat]=dataPos[i]                  - 1.0*dataCel[1]
        dataPos27[i       ]=dataPos[i]
        dataPos27[i+14*nat]=dataPos[i]                  + 1.0*dataCel[1]
        dataPos27[i+15*nat]=dataPos[i] - 1.0*dataCel[0] - 1.0*dataCel[1]
        dataPos27[i+16*nat]=dataPos[i] - 1.0*dataCel[0]
        dataPos27[i+17*nat]=dataPos[i] - 1.0*dataCel[0] + 1.0*dataCel[1]
        #
        dataPos27[i+18*nat]=dataPos[i] + 1.0*dataCel[0] - 1.0*dataCel[1] + 1.0*dataCel[2]
        dataPos27[i+19*nat]=dataPos[i] + 1.0*dataCel[0]                  + 1.0*dataCel[2]
        dataPos27[i+20*nat]=dataPos[i] + 1.0*dataCel[0] + 1.0*dataCel[1] + 1.0*dataCel[2]
        dataPos27[i+21*nat]=dataPos[i]                  - 1.0*dataCel[1] + 1.0*dataCel[2]
        dataPos27[i+22*nat]=dataPos[i]                                   + 1.0*dataCel[2]
        dataPos27[i+23*nat]=dataPos[i]                  + 1.0*dataCel[1] + 1.0*dataCel[2]
        dataPos27[i+24*nat]=dataPos[i] - 1.0*dataCel[0] - 1.0*dataCel[1] + 1.0*dataCel[2]
        dataPos27[i+25*nat]=dataPos[i] - 1.0*dataCel[0]                  + 1.0*dataCel[2]
        dataPos27[i+26*nat]=dataPos[i] - 1.0*dataCel[0] + 1.0*dataCel[1] + 1.0*dataCel[2]
        ##
        for j in range(0,27):
            map27to1[i+ j*nat] = i

    myTree = spatial.cKDTree(dataPos27,leafsize=100)
    NNindBa   = np.asarray([[0 for x in range(0,12)] for y in range(0,natUnit  )])
    NNindTi   = np.asarray([[0 for x in range(0,6 )] for y in range(0,natUnit  )])
    NNindOxTi = np.asarray([[0 for x in range(0,2 )] for y in range(0,natUnit*3)])
    NNindOxBa = np.asarray([[0 for x in range(0,4 )] for y in range(0,natUnit*3)])
    #
    NNind_NotMapD_Ba   = np.asarray([[0 for x in range(0,12)] for y in range(0,natUnit  )])
    NNind_NotMapD_Ti   = np.asarray([[0 for x in range(0,6 )] for y in range(0,natUnit  )])
    NNind_NotMapD_OxTi = np.asarray([[0 for x in range(0,2 )] for y in range(0,natUnit*3)])
    NNind_NotMapD_OxBa = np.asarray([[0 for x in range(0,4 )] for y in range(0,natUnit*3)])
    i=0
    for item in dataPos:
        if    i < 27         : # Ba
            TheResult = myTree.query(item, k=13, distance_upper_bound=7.0)
            NNindBa[            i             ] = [ map27to1[TheResult[1][j]] for j in range(1,13) ] # Ba-Ox
            NNind_NotMapD_Ba[   i             ] = [          TheResult[1][j]  for j in range(1,13) ] # Ba-Ox unMapped (to calc |Rij|)
        elif 26 < i and i < 54 : # Ti
            TheResult = myTree.query(item, k=7 , distance_upper_bound=7.0)
            NNindTi[            i -   natUnit ] = [ map27to1[TheResult[1][j]] for j in range(1,7 ) ] # Ti-Ox
            NNind_NotMapD_Ti[   i -   natUnit ] = [          TheResult[1][j]  for j in range(1,7 ) ] # Ti-Ox unMapped (to calc |Rij|)
        else                 : # Ox
            TheResult = myTree.query(item, k=7 , distance_upper_bound=7.0)
            NNindOxTi[          i - 2*natUnit ] = [ map27to1[TheResult[1][j]] for j in range(1,3 ) ] # Ox-Ti < 4.2 (1,2)
            NNindOxBa[          i - 2*natUnit ] = [ map27to1[TheResult[1][j]] for j in range(3,7 ) ] # Ox-Ba < 5.4 (3,4,5,6)
            NNind_NotMapD_OxTi[ i - 2*natUnit ] = [          TheResult[1][j]  for j in range(1,3 ) ] # Ox-Ti < 4.2 (1,2)     unMapped (to calc |Rij|)
            NNind_NotMapD_OxBa[ i - 2*natUnit ] = [          TheResult[1][j]  for j in range(3,7 ) ] # Ox-Ba < 5.4 (3,4,5,6) unMapped (to calc |Rij|)
        i=i+1
    #
    return NNindBa, NNindTi, NNindOxTi, NNindOxBa, NNind_NotMapD_Ba, NNind_NotMapD_Ti, NNind_NotMapD_OxTi, NNind_NotMapD_OxBa, dataPos27


class Solution:
    def findCircleNum( isConnected):
        n = len(isConnected)
        visited = [False] * n

        def dfs(r):
            visited[r] = True
            for i in range(n):
                if isConnected[r][i] and not visited[i]:
                    dfs(i)
        numC = 0
        for i in range(n):
            if not visited[i]:
                numC += 1
                dfs(i)

        return numC
if __name__ == '__main__':
    print(Solution.findCircleNum(isConnected = [[1,0,0],[0,1,0],[0,0,1]]))


import random
import sys
import copy
import pickle
from rlglue.agent.Agent import Agent
from rlglue.agent import AgentLoader as AgentLoader
from rlglue.types import Action
from rlglue.types import Observation
import tictactoe as ttt

PLAYER = 1

class tictactoe_agent(Agent):
    lastAction = Action()  #
    lastObservation = Observation()  #
    randGenerator = random.Random()
    stepsize = 0.1
    epsilon = 0.1
    gamma = 1
    numStates = 0
    numActions = 0
    value_function = None
    player = PLAYER

    policyFrozen = False
    exploringFrozen = False

    def agent_init(self):
        self.lastAction = Action()
        self.lastObservation = Observation()

    def egreedy(self, state):
        numActions = len(state[0])
        if not self.exploringFrozen and self.randGenerator.random() < self.epsilon:
            return self.randGenerator.randint(0, numActions - 1)
        return self.value_function[state].index(max(self.value_function[state]))

    def agent_start(self, observation):
        moves = observation.intArray[2:]
        turn = observation.intArray[0]
        board = observation.intArray[1]
        assert self.player == turn, "It needs to be this agent's turn for it to make a move"
        theState = (moves, board, turn, self.player)
        thisIntAction: int = self.egreedy(theState)
        returnAction = Action()
        returnAction.intArray = [thisIntAction]

        self.lastAction = copy.deepcopy(returnAction)
        self.lastObservation = copy.deepcopy(observation)

        return returnAction

    def agent_step(self, reward, observation: Observation):
        moves = observation.intArray[2:]
        turn = observation.intArray[0]
        board = observation.intArray[1]
        newState = (tuple(moves), board, turn, self.player)
        lastState = (tuple(self.lastObservation.intArray[2:]), self.lastObservation.intArray[1],
                     self.lastObservation.intArray[0], self.player)
        lastAction = self.lastAction.intArray[0]
        newIntAction = self.egreedy(newState)
        Q_sa = self.value_function[lastState][lastAction]
        Q_sprime_aprime = self.value_function[newState][newIntAction]
        new_Q_sa = Q_sa + self.stepsize * (reward + self.gamma * Q_sprime_aprime - Q_sa)
        if not self.policyFrozen:
            self.value_function[lastState][lastAction] = new_Q_sa
        returnAction = Action()
        returnAction.intArray = [newIntAction]
        self.lastAction = copy.deepcopy(returnAction)
        self.lastObservation = copy.deepcopy(observation)

        return returnAction

    def agent_end(self, reward):
        moves = self.lastObservation.intArray[2:]
        turn = self.lastObservation.intArray[0]
        board = self.lastObservation.intArray[1]
        lastState = (moves, board,
                     turn, self.player)
        lastAction = self.lastAction.intArray[0]
        Q_sa = self.value_function[lastState][lastAction]
        new_Q_sa = Q_sa + self.stepsize * (reward - Q_sa)

        if not self.policyFrozen:
            self.value_function[lastState][lastAction] = new_Q_sa

    def agent_cleanup(self):
        pass

    def save_value_function(self, fileName):
        theFile = open(fileName, "w")
        pickle.dump(self.value_function, theFile)
        theFile.close()

    def load_value_function(self, fileName):
        theFile = open(fileName, "r")
        self.value_function = pickle.load(theFile)
        theFile.close()

    def agent_message(self, inMessage):

        #	Message Description
        # 'freeze learning'
        # Action: Set flag to stop updating policy
        #
        if inMessage.startswith("freeze learning"):
            self.policyFrozen = True
            return "message understood, policy frozen"

        #	Message Description
        # unfreeze learning
        # Action: Set flag to resume updating policy
        #
        if inMessage.startswith("unfreeze learning"):
            self.policyFrozen = False
            return "message understood, policy unfrozen"

        # Message Description
        # freeze exploring
        # Action: Set flag to stop exploring (greedy actions only)
        #
        if inMessage.startswith("freeze exploring"):
            self.exploringFrozen = True
            return "message understood, exploring frozen"

        # Message Description
        # unfreeze exploring
        # Action: Set flag to resume exploring (e-greedy actions)
        #
        if inMessage.startswith("unfreeze exploring"):
            self.exploringFrozen = False
            return "message understood, exploring frozen"

if __name__ == '__main__':
    AgentLoader.loadAgent(tictactoe_agent())


# -*- coding: utf-8 -*-
"""
Created on Mon May 20 15:52:16 2019

@author: Susan
"""
import time
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

def profit_show(profit):
    profit = np.array(profit)
    print('總損益點數: %d' % np.sum(profit))
    print('勝率: %f' % (np.sum(profit > 0) / len(profit)))
    print('賺錢平均點數: %f' % (np.mean(profit[profit > 0])))
    print('輸錢平均點數: %f' % (np.mean(profit[profit <= 0])))
    plt.title('Daily Accumulated Points')
    plt.plot(np.cumsum(profit))
    plt.show()
    plt.title('Histogram of daily profits and losses')
    plt.hist(profit, bins=100)
    plt.show()

# Data process
taiex_df = pd.read_csv('TXF20112015_original.csv')
del taiex_df['dCode']
trade_day = np.unique(taiex_df['dDateTime']//10000)

#%%
'''RSV-M-mins'''
M = 30
alpha = 1/3
taiex_df['RSV'] = 100* (( taiex_df['dClose'] - taiex_df['dLowest'].rolling(window=M).min() ) / (taiex_df['dHighest'].rolling(window=M).max() - taiex_df['dLowest'].rolling(window=M).min()))

#%%
plt.plot(taiex_df['dClose'][:300], label='Price', color = 'k')
plt.legend()
plt.show()

plt.plot(taiex_df['RSV'][:300], label='RSV', color = 'r')
plt.axhline(20)
plt.axhline(80)
plt.ylim((0,100))
plt.legend()
plt.show()
#%%
print('=====策略1 RSV=====')
start = time.time()
profit_arr1 = []
for date in trade_day:
    dayData = taiex_df[taiex_df['dDateTime']//10000 == date].reset_index(drop=True)
    profit = 0
    isBought = False
    for i in range(len(dayData)):
        if dayData['RSV'].iloc[i] <= 20 and not isBought:
            isBought = True
            buy_price = dayData['dClose'].iloc[i]
        elif dayData['RSV'].iloc[i] >= 80 and isBought:
            isBought = False
            sell_price = dayData['dClose'].iloc[i]
            profit += sell_price - buy_price
    if isBought:
        sell_price = dayData['dClose'].iloc[-1]
        profit += sell_price - buy_price
    profit_arr1.append(profit)
profit_show(profit_arr1)
end = time.time()
print('Time : %d' % (end-start))

#%%
'''RSV，K_line,D_line'''
taiex_df['K_line'] = 0
taiex_df['D_line'] = 0
k = np.zeros(len(taiex_df))
d = np.zeros(len(taiex_df))
k[:M-2] = np.nan
d[:M-2] = np.nan
rsv = taiex_df['RSV'].values
k[M-2] = 50
d[M-2] = 50
for i in range(M-1, len(taiex_df)):
    k[i] = alpha * rsv[i] + (1 - alpha) * k[i-1]
    d[i] = alpha * k[i] + (1 - alpha) * d[i-1]
taiex_df['K_line'] = k
taiex_df['D_line'] = d

#%%

plt.plot(taiex_df['K_line'][M:M+60], label='K')
plt.plot(taiex_df['D_line'][M:M+60], label='D')
plt.axhline(20)
plt.axhline(80)
plt.ylim((0,100))
plt.legend()
plt.show()

#%%
def cross(df):
    # 判斷是否在>=80 或 <=20之間
    df['above_80'] = (df['K_line'].values>=80) & (df['D_line'].values>=80)
    df['below_20'] = (df['K_line'].values<=20) & (df['D_line'].values<=20)

    # 找交叉
    diff = df['K_line'] - df['D_line'] # 計算「K線」減去「D線」之差
    asign = np.sign(diff) # asign 會在這個數大於 0 時回傳 +1，小於 0 時回傳-1
    asign[np.where(asign==0)[0]] = 1 # 換句話說，asign 裡的 +1 就是當短線高於長線時，而 -1 就是長線高於短線時

    # 而黃金交叉（短線從長線下面穿越到長線）就是在 -1 到 +1 之時
    cross_signal = (np.where(np.roll(asign, 1)<asign))[0]
    df['golden_cross'] = 0
    df['golden_cross'][cross_signal] = 1 # 把結果存到 golden_cross 裡面

    # 而死亡交叉（短線從長線上面跌落到長線）就是在 +1 到 -1 之時
    death_signal = (np.where(np.roll(asign, 1)>asign))[0]
    df['death_cross'] = 0
    df['death_cross'][death_signal] = 1 # 把結果存到 death_cross 裡面

    df['Golden_80'] = (df['above_80']) & (df['golden_cross'])
    df['Death_80'] = (df['above_80']) & (df['death_cross'])
    df['Golden_20'] = (df['below_20']) & (df['golden_cross'])
    df['Death_20'] = (df['below_20']) & (df['death_cross'])

    print("Golden Cross 黃金交叉總共有:",df['golden_cross'].sum(),"次")
    print("Death Cross 死亡交叉總共有:",df['death_cross'].sum(),"次")
    df = df.drop(columns = ['golden_cross', 'death_cross'])
    return df

taiex_df = cross(taiex_df)

#%%
plt.plot(taiex_df['K_line'][M:M+60], label='K')
plt.plot(taiex_df['D_line'][M:M+60], label='D')
golden = np.where(taiex_df['Golden_20'][:M+60])[0]
death = np.where(taiex_df['Death_80'][:M+60])[0]
for i in golden:
    plt.axvline(i, color='red')
for j in death:
    plt.axvline(j, color='g')
plt.axhline(20)
plt.axhline(80)
plt.legend()
plt.show()

#%%

print('=====策略2.1 KD，20G作多，80D平倉 =====')
profit_arr21 = []
for date in trade_day:
    dayData = taiex_df[taiex_df['dDateTime']//10000 == date].reset_index(drop=True)
    profit = 0
    isBought = False

    if np.all(dayData['Golden_20'] == 0) and np.all(dayData['Death_80'] == 0):
        #plt.show()
        profit_arr21.append(0)
        continue
    for i in range(len(dayData)):
        minute_data = dayData.iloc[i]
        if minute_data['Golden_20'] and not isBought:
            isBought = True
            buy_price = minute_data['dClose']
        elif minute_data['Death_80'] and isBought:
            isBought = False
            sell_price = minute_data['dClose']
            profit += sell_price - buy_price

    if isBought:
        sell_price = dayData['dClose'].iloc[-1]
        profit += sell_price - buy_price
    profit_arr21.append(profit)
profit_show(profit_arr21)

#%%
def L_depart(df):
    df['Price_Uptrend'] = np.clip(np.sign(np.ediff1d(df['dClose'], to_begin=0)), 0, 1)
    df['KD_Uptrend'] = np.clip(np.sign(np.ediff1d(df['K_line'], to_begin=0)), 0, 1) * \
                       np.clip(np.sign(np.ediff1d(df['D_line'], to_begin=0)), 0, 1)
    df['L_depart'] = (1 - df['Price_Uptrend']) & (df['KD_Uptrend']) & (df['below_20'])
    return df

taiex_df = L_depart(taiex_df)

#%%
def H_depart(df):
    df['Price_Uptrend'] = np.clip(np.sign(np.ediff1d(df['dClose'], to_begin=0)), 0, 1)
    df['KD_Downtrend'] = (1 - np.clip(np.sign(np.ediff1d(df['K_line'], to_begin=0)), 0, 1)) * \
                       (1 - np.clip(np.sign(np.ediff1d(df['D_line'], to_begin=0)), 0, 1))
    df['H_depart'] = (df['Price_Uptrend']) & (df['KD_Downtrend']) & (df['above_80'])
    return df

taiex_df = H_depart(taiex_df)
#%%

print('=====策略3=====')
#（G20 ＋無背離＋沒買） →作多！
#（D80 ＋無背離＋有買）→平倉！
profit_arr3 = []
for date in trade_day:
    dayData = taiex_df[taiex_df['dDateTime']//10000 == date].reset_index(drop=True)
    profit = 0
    isBought = False

    for i in range(len(dayData)) :
        if dayData['Golden_20'].iloc[i] and not dayData['L_depart'].iloc[i] and not isBought:
            isBought = True
            buy_price = dayData['dClose'].iloc[i]
        elif dayData['Death_80'].iloc[i] and not dayData['H_depart'].iloc[i] and isBought :
            isBought = False
            sell_price = dayData['dClose'].iloc[i]
            profit += sell_price - buy_price
    if isBought :
        sell_price = dayData['dClose'].iloc[-1]
        profit += sell_price - buy_price
    profit_arr3.append(profit)
profit_show(profit_arr3)

#%%
print('====HW-策略4=====')
#（D80 ＋無背離＋沒買） →作空！
#（G20 ＋無背離＋有買）→平倉！
profit_arr4 = []
for date in trade_day:
    dayData = taiex_df[taiex_df['dDateTime']//10000 == date].reset_index(drop=True)
    profit = 0
    isBought = False

    for i in range(len(dayData)) :
        if dayData['Death_80'].iloc[i] and not dayData['H_depart'].iloc[i] and not isBought:
            isBought = True
            buy_price = dayData['dClose'].iloc[i]
        elif dayData['Golden_20'].iloc[i] and not dayData['L_depart'].iloc[i] and isBought :
            isBought = False
            sell_price = dayData['dClose'].iloc[i]
            profit += buy_price - sell_price
    if isBought :
        sell_price = dayData['dClose'].iloc[-1]
        profit += buy_price - sell_price
    profit_arr4.append(profit)
profit_show(profit_arr4)


''' Тестирование модели операций '''
from django.test import TestCase
from lawe.models import Account, Transaction

class TestOperationsModel(TestCase):
	''' Тестируем модель '''
	def testAutotimeOnCreation(self):
		''' При обновлении модели дата не должна меняться '''
		# Given
		op = Transaction.objects.create(
			debit=Account.objects.create(),
			credit=Account.objects.create(),
			amount=1
		)
		date1 = op.date
		# When
		op.save()
		# Then
		self.assertEqual(date1, op.date)


import logging
import settings
import ephem
import arrow
from telegram.ext import Updater, CommandHandler, MessageHandler, Filters

logging.basicConfig(
    format='%(name)s - %(levelname)s - %(message)s',
    level=logging.INFO
)

logger = logging.getLogger(__name__)

def greet_user(update, context):
    text = 'Вызван /start'
    logger.info(text)
    update.message.reply_text(text)

def ask_planet_name(update, context):
    user_text = update.message.text.split()
    current_date = update.message.date
    logger.info(user_text)
    for word in user_text[1:]:
        word = word.capitalize()
        if hasattr(ephem, word):
            planet = getattr(ephem, word)
            planet_info = planet(current_date)
            constellation = ephem.constellation(planet_info)
            update.message.reply_text('{} in constellation {}'.format(word, constellation))
        else:
            update.message.reply_text('{} did not found'.format(word))
            logger.info('Planet %s not found', word)

def humanize_ephem_date(ephem_date):
    date = arrow.get(str(ephem_date))
    output_date = date.format('YYYY-MM-DD')
    future = date.humanize(locale='ru')
    return f'{future} ({output_date})'

def next_full_moon(update, context):
    logger.info(update.message.text)
    next_full_moon_date = ephem.next_full_moon(update.message.date)
    msg = humanize_ephem_date(next_full_moon_date)
    update.message.reply_text(msg)

def word_count(update, context):
    user_text = update.message.text.split()[1:]
    logger.info(user_text)
    if not user_text:
        update.message.reply_text('Nothing to count')
        return
    filtered_text = [word for word in user_text if word.isalpha()]
    update.message.reply_text('Words quantity: {} '.format(len(filtered_text)))

def talk_to_me(update, context):
    user_text = update.message.text
    logger.info(user_text)
    update.message.reply_text(user_text)

def main():
    mybot = Updater(settings.API_KEY, use_context=True)

    dp = mybot.dispatcher
    dp.add_handler(CommandHandler("start", greet_user))
    dp.add_handler(CommandHandler("planet", ask_planet_name))
    dp.add_handler(CommandHandler("next_full_moon", next_full_moon))
    dp.add_handler(CommandHandler("wordcount", word_count))
    dp.add_handler(MessageHandler(Filters.text, talk_to_me))

    mybot.start_polling()
    mybot.idle()

if __name__ == "__main__":
    main()


import os
import torch

def weight_transfrom_for_groupconv(weight_path, groups=2):

    state_dict = torch.load(weight_path)
    for key in state_dict.keys():
        print(key, " weights before shape: ", state_dict[key].shape)
        if 'stage' in key:
            if "stage1" in key and "rebnconvin" in key:
                continue
            elif 'rebnconv' in key and 'conv_s1' in key and 'weight' in key:
                weights = state_dict[key]
                new_weight = []
                for channel in torch.chunk(weights, weights.shape[1] // groups, dim=1):
                    new_weight.append(torch.mean(channel, dim=1, keepdim=True))
                new_weight = torch.cat(new_weight, dim=1)
                state_dict.update({key: new_weight})

        print(key, " weights after shape: ", state_dict[key].shape)
    model_path_with_name, ext = os.path.splitext(weight_path)
    new_model_path = f"{model_path_with_name}_groups{groups}{ext}"
    torch.save(state_dict, new_model_path)
    print(f"modify model file save to: {new_model_path}")

def weight_transfrom_for_dsconv(weight_path, groups=2):

    state_dict = torch.load(weight_path)
    for key in list(state_dict.keys()):
        print(key, "old weights shape: ", state_dict[key].shape)
        if 'stage' in key:
            if 'rebnconv' in key and 'conv_s1' in key:
                if "bias" in key:
                    # 1.bias: only need rename
                    new_key = key.replace('bias', '1.bias')
                    state_dict.update({new_key: state_dict[key]})
                    print(new_key, " weights shape: ", state_dict[new_key].shape)
                    del state_dict[key]

                if 'weight' in key:
                    weights = state_dict[key]

                    # 0.weight： [out_channels, in_channels, 3, 3] -> [in_channels, 1, 3, 3]
                    new_weight = torch.mean(state_dict[key], dim=0, keepdim=True).transpose(0, 1)
                    new_key = key.replace('weight', '0.weight')
                    state_dict.update({new_key: new_weight})
                    print(new_key, " weights shape: ", state_dict[new_key].shape)

                    # 0.bias： [in_channels]
                    new_weight = torch.zeros(weights.shape[1])
                    new_key = key.replace('weight', '0.bias')
                    state_dict.update({new_key: new_weight})
                    print(new_key, " weights shape: ", state_dict[new_key].shape)

                    # 1.weight: [out_channels, in_channels, 3, 3] -> [out_channels, in_channels, 1, 1]
                    new_weight = torch.mean(torch.mean(state_dict[key], dim=2, keepdim=True), dim=3, keepdim=True)
                    new_key = key.replace('weight', '1.weight')
                    state_dict.update({new_key: new_weight})
                    print(new_key, " weights shape: ", state_dict[new_key].shape)

                    # remove old weight
                    del state_dict[key]

    # print(key, " weights after shape: ", state_dict[key].shape)
    model_path_with_name, ext = os.path.splitext(weight_path)
    new_model_path = f"{model_path_with_name}_dsconv{ext}"
    torch.save(state_dict, new_model_path)
    print(f"modify model file save to: {new_model_path}")

def mutil_gpu_model_to_singal_gpu_model(model_path):

    state_dict = torch.load(model_path, map_location="cpu")
    new_state_dict = {}
    for key in state_dict.keys():
        new_state_dict.update({key[7:]: state_dict[key]}) # remove "module." in key
    state_dict.clear()
    state_dict = new_state_dict

    name, ext = os.path.splitext(model_path)
    new_model_path = name + "_singal" + ext
    torch.save(new_state_dict, new_model_path)
    print(f"new model file save to: {new_model_path}")

# weight_path = "../models/pretrain_models/u2net.pth"
# weight_transfrom_for_groupconv(weight_path, groups=2)
# weight_transfrom_for_dsconv(weight_path)

model_path = '../models/backup/u2netdsconv_bce_itr_54000_train_1.149171_tar_0.148460.pth'
mutil_gpu_model_to_singal_gpu_model(model_path)

# nohup python -u u2net_train.py -n u2net_dsconv -b 16 -d=2,3 -m ./models/backup/u2netdsconv_bce_itr_54000_train_1.149171_tar_0.148460singal.pth >deconv_54000.out 2>&1 &


import timeit
import grequests
from lxml import etree

start = timeit.default_timer()

urls = [
    'https://nomes.club/nomes-populares-brasil/',
    'https://www.passwordrandom.com/most-popular-passwords',
]

rs = (grequests.get(u) for u in urls)

mapa = grequests.map(rs)

x = mapa[0]

y = mapa[1]

tree = etree.HTML(x.text)

lis = tree.xpath('//ol/li//a')

nomes = [li.text for li in lis]

print(nomes)

tree2 = etree.HTML(y.text)

lis2 = tree2.xpath('//tr/td[2]')

senhas = [li.text for li in lis2]

print(senhas)

stop = timeit.default_timer()

print('Time: ', stop - start)


"""
4. Dados dois números pelo usuário, exibir o de maior valor.
Entrada: 5 45 Saída: 45
Entrada: 10 8 Saída: 10
"""

num1 = float(input(f"Digite um número: "))
num2 = float(input(f"Digite outro número: "))
if num1 < num2:
     print(f"O maior número digitado pelo usuário é: {num2}")
else:
    print(f"O maior número digitado pelo usuário é: {num1}")


import hmac
import shlex
import subprocess
import time
from datetime import datetime
from pathlib import Path
import json
from urllib.parse import urljoin

import requests
from hashlib import sha256
from tempfile import TemporaryDirectory

from fastapi import FastAPI, HTTPException, Request
from github import Github

from .config import load_config
from .constants import (
    GITHUB_WEBHOOK_SECRET,
    GITHUB_ACCESS_TOKEN,
    MAX_DOWNLOAD_ARTIFACT_RETRIES,
)
from .data import load_build_data, Target as DataTarget, save_build_data

app = FastAPI()
config = load_config()
data = load_build_data()
github = Github(GITHUB_ACCESS_TOKEN)

async def verify_github_webhook(request: Request):
    # Take sha256 hmac of the request body, using GITHUB_WEBHOOK_TOKEN as the key
    if not GITHUB_WEBHOOK_SECRET:
        raise HTTPException(status_code=500, detail="GITHUB_WEBHOOK_TOKEN not set")
    hmac_sha256 = hmac.new(
        GITHUB_WEBHOOK_SECRET.encode(), await request.body(), sha256
    ).hexdigest()
    if not request.headers.get("X-Hub-Signature-256") == f"sha256={hmac_sha256}":
        raise HTTPException(status_code=401, detail="Unauthorized")

def github_get_request(url):
    response = requests.get(
        url,
        headers={
            "Authorization": f"token {GITHUB_ACCESS_TOKEN}",
            "Accept": "application/vnd.github.v3+json",
        },
    )
    response.raise_for_status()
    return response.json()

def reload_controller(target):
    # Update the controller with the /reload endpoint
    reload_response = requests.get(urljoin(target.controller_api_url, "reload"))
    reload_response.raise_for_status()

def check_controller_connection(target):
    # This is a very simple stopgap to prevent a corrupted state when a target
    # machine is not running; all this is meant to do is raise an error when
    # things don't work
    current_experience_response = requests.get(
        urljoin(target.controller_api_url, "current")
    )
    current_experience_response.raise_for_status()

def set_commit_status(repository_name, sha, state, context, description=None):
    github.get_repo(repository_name).get_commit(sha).create_status(
        state, context=f"footron-ci/{context}", description=description
    )

def ssh_production_option():
    return f'-o "SetEnv=FT_PRODUCTION={datetime.now().strftime("%d%H")}"'

def rsync_ssh_production_command():
    return f"ssh {ssh_production_option()}"

def handle_workflow_run_completed(event):
    workflow_run = event["workflow_run"]

    if workflow_run["status"] != "completed" or workflow_run["event"] != "push":
        return

    branch = workflow_run["head_branch"]
    sha = workflow_run["head_sha"]

    try:
        target = config.targets[branch]
    except KeyError:
        return

    event_name = workflow_run["name"]
    artifacts_url = workflow_run["artifacts_url"]

    repository_name = event["repository"]["full_name"]

    if event_name not in ["build-controls", "build-experiences"]:
        raise HTTPException(status_code=400, detail="Unknown event name")

    try:
        check_controller_connection(target)
    except requests.RequestException:
        set_commit_status(
            repository_name, sha, "failure", event_name, "Failed to connect"
        )
        raise

    # Start timer
    start_time = time.time()

    if event_name == "build-controls":
        # Request the artifacts endpoint using access token
        artifacts_response = github_get_request(artifacts_url)

        artifacts = artifacts_response["artifacts"]

        experiences_build_url = None
        for artifact in artifacts:
            if artifact["name"] == "web-build":
                experiences_build_url = artifact["archive_download_url"]
                break
        else:
            raise RuntimeError("Missing web build")

        # Download the artifact
        with TemporaryDirectory() as temp_dir:
            temp_path = Path(temp_dir)
            web_build_zip_path = temp_path / "web-build.zip"
            set_commit_status(
                repository_name, sha, "pending", event_name, "Downloading artifacts"
            )
            download_artifact_retries = 0
            while True:
                try:
                    subprocess.run(
                        [
                            "curl",
                            "-H",
                            f"Authorization: token {GITHUB_ACCESS_TOKEN}",
                            "-L",
                            experiences_build_url,
                            "-o",
                            str(web_build_zip_path),
                        ]
                    )
                except subprocess.CalledProcessError:
                    if download_artifact_retries < MAX_DOWNLOAD_ARTIFACT_RETRIES:
                        download_artifact_retries += 1
                        print(
                            f"Failed to download artifact; retrying in 1 second... ({download_artifact_retries}/{MAX_DOWNLOAD_ARTIFACT_RETRIES})"
                        )
                        time.sleep(1)
                        continue
                    set_commit_status(
                        repository_name,
                        sha,
                        "failure",
                        event_name,
                        "Failed to download controls artifact",
                    )
                    raise
                break

            set_commit_status(
                repository_name, sha, "pending", event_name, "Extracting artifacts"
            )
            # Extract the artifacts
            subprocess.run(
                ["unzip", str(web_build_zip_path), "-d", str(temp_path / "build")]
            )

            # rsync the build directory to the target directory
            set_commit_status(
                repository_name, sha, "pending", event_name, "Copying web build"
            )
            print(f'rsync {temp_path / "build"}/ {target.web_path}')
            subprocess.run(
                [
                    "rsync",
                    "-e",
                    rsync_ssh_production_command(),
                    "-a",
                    "--delete",
                    f'{str(temp_path / "build")}/',
                    str(target.web_path),
                ]
            )

            set_commit_status(
                repository_name, sha, "pending", event_name, "Reloading controller"
            )
            reload_controller(target)
    elif event_name == "build-experiences":
        artifacts_response = github_get_request(artifacts_url)

        artifacts = artifacts_response["artifacts"]

        for artifact in artifacts:
            if artifact["name"] == "experiences":
                experiences_build_url = artifact["archive_download_url"]
                break
        else:
            set_commit_status(
                repository_name,
                sha,
                "failure",
                event_name,
                "Missing experiences build",
            )
            raise RuntimeError("Missing experiences build")

        # Download the artifact using the access token in a curl header
        with TemporaryDirectory() as temp_dir:
            temp_path = Path(temp_dir)
            experience_build_zip_path = temp_path / "experiences.zip"

            set_commit_status(
                repository_name, sha, "pending", event_name, "Downloading artifacts"
            )
            download_artifact_retries = 0
            while True:
                try:
                    subprocess.run(
                        [
                            "curl",
                            "-H",
                            f"Authorization: token {GITHUB_ACCESS_TOKEN}",
                            "-L",
                            experiences_build_url,
                            "-o",
                            str(experience_build_zip_path),
                        ]
                    ).check_returncode()
                except subprocess.CalledProcessError:
                    if download_artifact_retries < MAX_DOWNLOAD_ARTIFACT_RETRIES:
                        download_artifact_retries += 1
                        print(
                            f"Failed to download artifact; retrying in 1 second... ({download_artifact_retries}/{MAX_DOWNLOAD_ARTIFACT_RETRIES})"
                        )
                        time.sleep(1)
                        continue
                    set_commit_status(
                        repository_name,
                        sha,
                        "failure",
                        event_name,
                        "Failed to download expereinces artifact",
                    )
                    raise
                break

            # Extract the artifacts
            set_commit_status(
                repository_name, sha, "pending", event_name, "Extracting artifacts"
            )

            try:
                subprocess.run(
                    ["unzip", str(experience_build_zip_path), "-d", temp_path]
                ).check_returncode()
            except subprocess.CalledProcessError:
                set_commit_status(
                    repository_name,
                    sha,
                    "failure",
                    event_name,
                    "Failed to unzip web build",
                )
                raise

            set_commit_status(
                repository_name, sha, "pending", event_name, "Comparing hashes"
            )
            hashes_path = temp_path / "hashes.json"
            with open(hashes_path, "r") as hashes_file:
                hashes = json.load(hashes_file)
            if branch not in data.targets:
                data.targets[branch] = DataTarget()

            existing_hashes = data.targets[branch].hashes
            hashes_keys = set(hashes.keys())
            existing_hashes_keys = set(existing_hashes.keys())
            new_hashes = hashes_keys - existing_hashes_keys
            deleted_hashes = existing_hashes_keys - hashes_keys
            overlapping_hashes = hashes_keys & existing_hashes_keys
            changed_hashes = set()

            for hash_key in new_hashes:
                data.targets[branch].hashes[hash_key] = hashes[hash_key]
            for hash_key in overlapping_hashes:
                if hashes[hash_key] != existing_hashes[hash_key]:
                    changed_hashes.add(hash_key)

            controller_host = target.controller_host
            controller_fs_path = target.controller_fs_path

            print("Added experiences:", list(new_hashes))
            print("Deleted experiences:", list(deleted_hashes))
            print("Changed experiences:", list(changed_hashes))

            set_commit_status(
                repository_name,
                sha,
                "pending",
                event_name,
                "Deleting deleted experiences",
            )
            for hash_key in deleted_hashes:
                del data.targets[branch].hashes[hash_key]
                # rm -rf the hash directory on the target host, or the current host
                #  if the host isn't specified
                if controller_host:
                    try:
                        subprocess.run(
                            [
                                "ssh",
                                *shlex.split(ssh_production_option()),
                                controller_host,
                                f"rm -rf {controller_fs_path}/experiences/{hash_key}",
                            ]
                        ).check_returncode()
                    except subprocess.CalledProcessError:
                        set_commit_status(
                            repository_name,
                            sha,
                            "failure",
                            event_name,
                            "Failed to delete removed experiences on remote host",
                        )
                        raise
                else:
                    try:
                        subprocess.run(
                            [
                                "rm",
                                "-rf",
                                f"{controller_fs_path}/experiences/{hash_key}",
                            ]
                        ).check_returncode()
                    except subprocess.CalledProcessError:
                        set_commit_status(
                            repository_name,
                            sha,
                            "failure",
                            event_name,
                            "Failed to delete removed experiences on local host",
                        )
                        raise

            set_commit_status(
                repository_name,
                sha,
                "pending",
                event_name,
                "Syncing experiences",
            )
            for hash_key in changed_hashes | new_hashes:
                # rsync the changed directory to the directory on the target host
                try:
                    subprocess.run(
                        [
                            "rsync",
                            "-avP",
                            "--delete",
                            "-e",
                            rsync_ssh_production_command(),
                            # Hack to avoid dealing with video files for now
                            "--exclude=*.mp4",
                            "--exclude=*.webm",
                            f'{str(temp_path / "experiences" / hash_key)}/',
                            f"{target.controller_path}/experiences/{hash_key}",
                        ]
                    ).check_returncode()
                except subprocess.CalledProcessError:
                    set_commit_status(
                        repository_name,
                        sha,
                        "failure",
                        event_name,
                        f"Failed to sync experience '{hash_key}' to target",
                    )
                    raise

            data.targets[branch].hashes = hashes
            save_build_data(data)

            set_commit_status(
                repository_name, sha, "pending", event_name, "Syncing editorial configs"
            )
            try:
                subprocess.run(
                    [
                        "rsync",
                        "-avP",
                        "--delete",
                        "-e",
                        rsync_ssh_production_command(),
                        *(
                            f"{str(temp_path)}/{file}.toml"
                            for file in ["folders", "tags", "collections"]
                        ),
                        str(target.controller_path),
                    ]
                ).check_returncode()
            except subprocess.CalledProcessError:
                set_commit_status(
                    repository_name,
                    sha,
                    "failure",
                    event_name,
                    "Failed to sync editorial configs",
                )
                raise

            set_commit_status(
                repository_name, sha, "pending", event_name, "Reloading controller"
            )
            try:
                reload_controller(target)
            except requests.RequestException:
                set_commit_status(
                    repository_name,
                    sha,
                    "failure",
                    event_name,
                    "Failed to reload controller",
                )
                raise

    # "Successful in {}s"
    set_commit_status(
        repository_name,
        sha,
        "success",
        event_name,
        f"Successful in {int(round(time.time() - start_time))}s",
    )

def handle_workflow_job_queued(event):
    workflow_job = event["workflow_job"]

    if workflow_job["name"] != "build":
        return

    run_response = github_get_request(workflow_job["run_url"])
    event_name = run_response["name"]
    if (
        run_response["event"] != "push"
        or event_name
        not in [
            "build-experiences",
            "build-controls",
        ]
        or run_response["head_branch"] not in config.targets
    ):
        return

    sha = workflow_job["head_sha"]

    repository_name = event["repository"]["full_name"]
    # Use Python GitHub API to set a status on the commit
    github.get_repo(repository_name).get_commit(sha).create_status(
        "pending",
        context=f"footron-ci/{event_name}",
        description="Waiting for build to finish",
    )

@app.post("/build/webhook")
async def handle_webhook(request: Request):
    # Verify the payload signature X-Hub-Signature
    await verify_github_webhook(request)

    payload = await request.json()
    event_type = request.headers.get("X-GitHub-Event")

    if event_type == "workflow_run" and payload["action"] == "completed":
        handle_workflow_run_completed(payload)
    elif event_type == "workflow_job" and payload["action"] == "queued":
        handle_workflow_job_queued(payload)

    # TODO: Handle adding status for pushing to machine
    # https://docs.github.com/en/rest/commits#commit-statuses

    return {"status": "ok"}


import json
import math
import os
import re
from pathlib import Path

import numpy as np
import pytorch_lightning as pl
import torch
from datasets import load_dataset
from loguru import logger
from nltk import edit_distance
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint, LearningRateMonitor
from pytorch_lightning.loggers import TensorBoardLogger, CSVLogger
from pytorch_lightning.utilities import rank_zero_only
from torch.nn.utils.rnn import pad_sequence
from torch.optim.lr_scheduler import LambdaLR
from torch.utils.data import DataLoader, ConcatDataset
from tqdm.auto import tqdm
from transformers import (
    DonutProcessor,
    VisionEncoderDecoderConfig,
    VisionEncoderDecoderModel,
)

from config import settings
from data import DonutDataset, SynthdogDataset, CORDOCRDataset
from donut import DonutModel
from pl_modules import DonutModelPLModule
from utils import JSONParseEvaluator
from validate import validate

import torch.multiprocessing
torch.multiprocessing.set_sharing_strategy('file_system')

os.makedirs(f"logs/{settings.log_name}/", exist_ok=True)
logger.add(f"logs/{settings.log_name}/train.log")

torch.set_float32_matmul_precision("medium")
def get_data(model: VisionEncoderDecoderModel, processor: DonutProcessor):
    if settings.pre_training:
        train_dataset = SynthdogDataset(
            model=model,
            processor=processor,
            dataset_name_or_path=settings.dataset_name,
            max_length=settings.max_length,
            split="train",
            task_start_token=settings.task_start_token,
            prompt_end_token=settings.prompt_end_token,
            sample_size=1000
        )
        if settings.cord_ocr_path:
            train_cord_dataset = CORDOCRDataset(
                model=model,
                processor=processor,
                dataset_name_or_path=settings.cord_ocr_path,
                max_length=settings.max_length,
                split="train",
                task_start_token=settings.task_start_token,
                prompt_end_token=settings.prompt_end_token,
            )
            train_dataset = ConcatDataset([train_dataset, train_cord_dataset])

        val_dataset = SynthdogDataset(
            model=model,
            processor=processor,
            dataset_name_or_path=settings.dataset_name,
            max_length=settings.max_length,
            split="validation",
            task_start_token=settings.task_start_token,
            prompt_end_token=settings.prompt_end_token,
            sample_size=500
        )
    else:
        train_dataset = DonutDataset(
            model=model,
            processor=processor,
            dataset_name_or_path=settings.dataset_name,
            max_length=settings.max_length,
            split="train",
            task_start_token=settings.task_start_token,
            prompt_end_token=settings.prompt_end_token,
            sort_json_key=False
        )

        val_dataset = DonutDataset(
            model=model,
            processor=processor,
            dataset_name_or_path=settings.dataset_name,
            max_length=settings.max_length,
            split="validation",
            task_start_token=settings.task_start_token,
            prompt_end_token=settings.prompt_end_token,
            sort_json_key=False
        )

    train_dataloader = DataLoader(
        train_dataset, batch_size=settings.train_batch_size, shuffle=True, num_workers=24
    )
    val_dataloader = DataLoader(
        val_dataset, batch_size=settings.val_batch_size, shuffle=False, num_workers=24
    )

    return train_dataloader, val_dataloader

def main():
    # update image_size of the encoder
    # during pre-training, a larger image size was used
    config = VisionEncoderDecoderConfig.from_pretrained(settings.base_model)
    config.encoder.image_size = settings.image_size  # (height, width)
    # update max_length of the decoder (for generation)
    config.decoder.max_length = settings.max_length
    # TODO we should actually update max_position_embeddings and interpolate the pre-trained ones:
    # https://github.com/clovaai/donut/blob/0acc65a85d140852b8d9928565f0f6b2d98dc088/donut/model.py#L602
    if False:
        model = DonutModel.from_encoder_decoder_pretrained(
            encoder_pretrained_model_name_or_path="microsoft/swin-base-patch4-window12-384",
            decoder_pretrained_model_name_or_path="hyunwoongko/asian-bart-ecjk",
            base_config=config,
        )
    else:
        model = VisionEncoderDecoderModel.from_pretrained(
            settings.base_model, config=config
        )

    processor = DonutProcessor.from_pretrained(settings.base_model)
    processor.image_processor.size = settings.image_size[
        ::-1
    ]  # should be (width, height)
    processor.image_processor.do_align_long_axis = False
    train_dataloader, val_dataloader = get_data(model=model, processor=processor)

    batch = next(iter(train_dataloader))
    pixel_values, labels, target_sequences = batch

    logger.info(
        f"""
        Verify batch
        Shape of pixel_values: {pixel_values.shape}
        First 100 labels: {labels[:100]}
        Target Sequences: {target_sequences[:100]}
        """
    )

    model.config.pad_token_id = processor.tokenizer.pad_token_id
    model.config.decoder_start_token_id = processor.tokenizer.convert_tokens_to_ids(
        [settings.task_start_token]
    )[0]
    # sanity check
    logger.info(f"Pad token ID: {processor.decode([model.config.pad_token_id])}")
    logger.info(
        f"Decoder start token ID: {processor.decode([model.config.decoder_start_token_id])}"
    )

    tensorboard_logger = TensorBoardLogger(save_dir="logs", name=settings.log_name)
    if settings.pre_training:
        early_stop_callback = EarlyStopping(
            monitor="train_loss", patience=10, verbose=True, mode="min"
        )
        checkpoint_callback = ModelCheckpoint(
            monitor='character_error_rate'
        )
    else:
        early_stop_callback = EarlyStopping(
            monitor="val_edit_distance", patience=10, verbose=True, mode="min"
        )
        checkpoint_callback = ModelCheckpoint(
            monitor='val_edit_distance'
        )

    lr_monitor = LearningRateMonitor(logging_interval='step')

    trainer = pl.Trainer(
        accelerator="gpu",
        devices=settings.gpu_devices,
        max_epochs=settings.max_epochs,
        max_steps=settings.max_steps,
        val_check_interval=settings.val_check_interval,
        check_val_every_n_epoch=settings.check_val_every_n_epoch,
        gradient_clip_val=settings.gradient_clip_val,
        precision="bf16-mixed",  # we'll use mixed precision
        num_sanity_val_steps=settings.num_sanity_val_steps,
        logger=tensorboard_logger,
        log_every_n_steps=settings.log_every_n_steps,
        callbacks=[lr_monitor, checkpoint_callback],
    )

    model_module = DonutModelPLModule(
        processor, model, train_dataloader, val_dataloader
    )
    trainer.fit(model_module)

    if not settings.pre_training:
        scores = validate(model=model, processor=processor, dataset_subset="validation")
        logger.info(f"Mean accuracy: {scores['mean_accuracy']}")

    model_module.processor.save_pretrained(f"logs/{settings.log_name}")
    model_module.model.save_pretrained(f"logs/{settings.log_name}")

if __name__ == "__main__":
    main()