from .my_a_star_v_2_heapdict import *
from .single_agent_planner import compute_heuristics

class path_map:
    paths=[]
    position_table = {}
    edge_table={}
    no_path_indexes=[]
    def __init__(self,paths):
        self.paths=[]
        self.position_table={}
        self.edge_table={}
        self.no_path_indexes=[]
        self.paths=paths
        for agent in range(len(paths)):
            path=paths[agent]
            if path != None:
                for time in range(len(path)):
                    if time == 0:
                        continue
                    if self.position_table.get((path[time],time)) == None:
                        self.position_table[(path[time],time)] = [agent]
                    else:
                        self.position_table[(path[time],time)].append(agent)
                    if self.edge_table.get((min(path[time],path[time-1]), \
                        max(path[time],path[time-1]),time)) == None:
                        self.edge_table[(min(path[time],path[time-1]), \
                            max(path[time],path[time-1]),time)] = [agent]
                    else:
                        self.edge_table[(min(path[time],path[time-1]), \
                            max(path[time],path[time-1]),time)].append(agent)
    def get_node_collisions(self):
        node_collisions=[[]]*len(self.paths)
        for key,value in self.position_table.items():
            if value != None and len(value) > 1:
                for agent in value:
                    node_collisions[agent].append(key)
        return node_collisions
    def get_edge_collisions(self):
        edge_collisions=[[]]*len(self.paths)
        for key,value in self.edge_table.items():
            if value != None and len(value) > 1:
                for agent in value:
                    edge_collisions[agent].append(key)
        return edge_collisions
    def get_collisions(self):
        collisions=[]
        for i in range(len(self.paths)):
            collisions.append([])
        for key,value  in self.position_table.items():
            if value != None and len(value) > 1:
                for i in range(len(value)):
                    agent = value[i]
                    l=value[:i]+value[i+1:]
                    collisions[agent].append(('n',key,l))
        for key,value in self.edge_table.items():
            if value != None and len(value)>1:
                for i in range(len(value)):
                    agent = value[i]
                    l=value[:i]+value[i+1:]
                    collisions[agent].append(('e',key,l))
        return collisions
    def remove_agent(self,agent):
        path=self.paths[agent]
        #print(self.position_table)
        #print(self.paths)
        for time in range(len(path)):
            if time == 0:
                continue
            self.position_table[(path[time],time)].remove(agent)
            if self.position_table[(path[time],time)] == []:
                self.position_table[(path[time],time)]=None
            if time != 0:
                self.edge_table[(min(path[time],path[time-1]), \
                    max(path[time],path[time-1]),time)].remove(agent)
                if self.edge_table[(min(path[time],path[time-1]), \
                    max(path[time],path[time-1]),time)] == []:
                    self.edge_table[(min(path[time],path[time-1]), \
                        max(path[time],path[time-1]),time)] = None
        self.paths[agent]=[]
    def get_one_collision(self):
        node_collisions=[[]]*len(self.paths)
        for key,value in self.position_table.items():
            if len(value) > 1:
                return (key,value)
        return None
    def get_one_edge_collision(self):
        for key,value in self.edge_table.items():
            if value != None and len(value) > 1:
                return (key,value)
        return None
    def change_path(self,path,agent):
        self.remove_agent(agent)
        if path != None:
            for time in range(len(path)):
                if time == 0:
                    continue
                if self.position_table.get((path[time],time)) == None:
                    self.position_table[(path[time],time)] = [agent]
                else:
                    self.position_table[(path[time],time)].append(agent)
                if self.edge_table.get((min(path[time],path[time-1]), \
                    max(path[time],path[time-1]),time)) == None:
                    self.edge_table[(min(path[time],path[time-1]), \
                        max(path[time],path[time-1]),time)] = [agent]
                else:
                    self.edge_table[(min(path[time],path[time-1]), \
                        max(path[time],path[time-1]),time)].append(agent)
            self.paths[agent]=path
def get_max_conflict(collisions):
    if not collisions:
        return None
    mc=0
    for i in range(len(collisions)):
        if len(collisions[i]) > len(collisions[mc]):
            mc = i
    return mc,len(collisions[mc])

def prioritized_search(map,starts,goals):
    paths=[]
    constraints=[]
    edge_constraints=[]
    heuristics=[]
    for agent in range(len(starts)):
        h=compute_heuristics(map,goals[agent])
        heuristics.append(h)
        path = my_a_star(map, starts[agent], goals[agent], h, agent, constraints,edge_constraints)
        paths.append(path)
    pm = path_map(paths)
    collisions = pm.get_collisions()
    agent,cc=get_max_conflict(collisions)
    while cc != 0:
       # print(paths,'\n\n\n',pm.position_table,'\n\n\n',collisions)
        conflicts=collisions[agent]
        for conflict in conflicts:
            t,key,a = conflict
            if t == 'n':
                l,t=key
                constraints.append((agent,l,t))
                #for ca in a:
                #    constraints.append((ca,l,t))
            if t == 'e':
                l1,l2,t=key
                edge_constraints.append((agent, (l1, l2), t))
                #for ca in a:
                #    edge_constraints.append((ca,(l1,l2),t))
        path= my_a_star(map, starts[agent], goals[agent], heuristics[agent], agent, constraints,edge_constraints)
        if path == None:
            print(f'No path found for agent{agent}')
            pm.change_path([],agent)
        else:
            pm.change_path(path,agent)
        collisions = pm.get_collisions()
        #print(collisions)
        agent,cc=get_max_conflict(collisions)
    return pm.paths


import converter
from flask import Flask, request, render_template, send_file

app = Flask(__name__)

@app.route("/")
def index():
    return render_template('first.html')

@app.route('/keyNames', methods=['POST'])
def key_names():
    count = int(request.form['column'])
    column_count = [i+1 for i in range(count)]

    return render_template('second.html', column_count=column_count)

@app.route('/itemNames', methods=['POSt'])
def item_names():
    names = request.form.getlist('key-names[]')

    return render_template('third.html', names=names)

@app.route('/download', methods=['POST'])
def download():
    file_name = request.form['title']
    text = request.form['text']
    base_dir = '/tmp/'

    converter.convert.remove_files(base_dir)
    converter.convert.create_csv(file_name, text, base_dir)
    converter.convert.create_tsv(file_name, text, base_dir)
    converter.convert.create_json(file_name, base_dir)
    converter.convert.create_yaml(file_name, base_dir)
    converter.convert.create_xml(file_name, base_dir)
    converter.convert.create_zip(file_name, base_dir)

    download_file_name = base_dir + file_name + '.zip'
    download_file = file_name + '.zip'

    return send_file(download_file_name, as_attachment=True, attachment_filename=download_file, mimetype='application/zip')

if __name__ == '__main__':
    app.run()


# import libraries
import osmnx as ox
import networkx as nx
import plotly.graph_objects as go
import numpy as np
import matplotlib.pyplot as plt
import osmium as osm
import pandas as pd

# import data
# define class function
class OSMHandler(osm.SimpleHandler):
    def __init__(self):
        osm.SimpleHandler.__init__(self)
        self.osm_data = []

    def tag_inventory(self, elem, elem_type):
        for tag in elem.tags:
            self.osm_data.append([elem_type,
                                   elem.id,
                                   elem.version,
                                   elem.visible,
                                   pd.Timestamp(elem.timestamp),
                                   elem.uid,
                                   elem.user,
                                   elem.changeset,
                                #    elem.location,
                                   len(elem.tags),
                                   tag.k,
                                   tag.v])

    def node(self, n):
        self.tag_inventory(n, "node")

    def way(self, w):
        self.tag_inventory(w, "way")

    def relation(self, r):
        self.tag_inventory(r, "relation")

osmhandler = OSMHandler()
# scan the input file and fills the handler list accordingly
osmhandler.apply_file("munich_center.osm")

# transform the list into a pandas DataFrame
data_colnames = ['type', 'id', 'version', 'visible', 'ts', 'uid',
                 'user', 'chgset','ntags', 'tagkey', 'tagvalue']
df_osm = pd.DataFrame(osmhandler.osm_data, columns=data_colnames)

# find target
target = df_osm[df_osm['tagkey'] == 'target']
# find source
source = df_osm[df_osm['tagkey'] == 'source']

# get footway data using bbox [11.5430,48.1249,11.6104,48.1510]
footways = ox.graph_from_bbox(48.1391,48.1319, 11.5698,11.5604, network_type='walk')
# footways = ox.graph.graph_from_xml('munich_center.osm')
# plot the graph
# ox.plot_graph(footways)

# define origin and desination locations
# sample point based on target and source
origin_point = ( 48.135710, 11.565572)
destination_point = (48.138244, 11.561798)

# plot graph and origin - destination
fig, ax = ox.plot_graph(footways, show=False)
ax.scatter(origin_point[1], origin_point[0], c='red', s=100, label='orig', marker='o')
ax.scatter(destination_point[1], destination_point[0], c='lime', s=100, label='dest', marker='o')
ax.annotate('origin-point', (origin_point[1], origin_point[0]), color = 'white')
ax.annotate('destination-point', (destination_point[1], destination_point[0]),color = 'white')
plt.show()

# get the nearest nodes to the locations
origin_node = ox.get_nearest_node(footways, origin_point)
destination_node = ox.get_nearest_node(footways, destination_point)

# printing the closest node id to origin and destination points
origin_node, destination_node

points_list = list(footways.edges())

nodesNumber = footways.number_of_nodes()
footways.number_of_nodes(),footways.number_of_edges()

nodes_list= []
for point in points_list:
    v,u = point[0],point[1]
    nodes_list.append(v)
    nodes_list.append(u)

nodes_set = set(nodes_list)

reverse_nodes_dict = dict(enumerate(nodes_set))
nodes_dict = dict(zip(nodes_set,range(0,nodesNumber)))

new_points_list = []
for point in points_list:
    v,u = point[0],point[1]
    newV, newU = nodes_dict[v],nodes_dict[u]
    new_node = (newV, newU)
    new_points_list.append(new_node)

nodes_dict[origin_node],nodes_dict[destination_node]

G = nx.Graph()
G.add_edges_from(new_points_list)
R = np.matrix(np.ones(shape = (nodesNumber, nodesNumber)))
R *=-1
gamma = 0.8
initial_state = 1

def available_actions(state):
    startingNode_row = R[state, ]
    av_act = np.where(startingNode_row >= 0)[1]
    return av_act

def sample_next_action(available_act):
    next_action = int(np.random.choice(available_act, 1))
    return next_action

def update(Q,startingNode, action, gamma):

    max_index = np.where(Q[action,]==np.max(Q[action,]))[1]

    if max_index.shape[0] > 1:
        max_index = int(np.random.choice(max_index, size = 1))
    else:
        max_index = int(max_index)

    max_value = Q[action, max_index]

    Q[startingNode, action] = R[startingNode, action] + gamma *max_value

    if np.max(Q > 0):
        return(np.sum(Q/np.max(Q)*100))
    else:
        return (0)

def shortest_Path(origin,destination):

	for point in new_points_list:
	    if point[1] ==destination:
	        R[point] = 150
	    else:
	        R[point] = 0

	    if point[0] == destination:
	        R[point[::-1]] = 150
	    else:
	        R[point[::-1]] = 0

def shortest_Path(origin,destination):

	for point in new_points_list:
	    if point[1] ==destination:
	        R[point] = 150
	    else:
	        R[point] = 0

	    if point[0] == destination:
	        R[point[::-1]] = 150
	    else:
	        R[point[::-1]] = 0

	R[destination, destination] = 150

	Q = np.matrix(np.zeros([nodesNumber, nodesNumber]))

	available_act = available_actions(initial_state)
	action = sample_next_action(available_act)
	update(Q,initial_state, action, gamma)

	scores=[]
	for i in  range(nodesNumber*100):
	    startingNode = np.random.randint(0, int(Q.shape[0]))
	    available_act = available_actions(startingNode)
	    action = sample_next_action(available_act)
	    score = update(Q,startingNode, action, gamma)
	    scores.append(score)

	steps = [origin]
	current = origin
	while current != destination:
	    next_step_index = np.where(Q[current,]== np.max(Q[current,]))[1]

	    if next_step_index.shape[0] > 1:
	        next_step_index = int(np.random.choice(next_step_index, size = 1))
	    else:
	        next_step_index = int(next_step_index)

	    steps.append(next_step_index)
	    current = next_step_index

	return scores, steps

scores,steps = shortest_Path(nodes_dict[origin_node],nodes_dict[destination_node])
plt.plot(scores)
plt.show()
print('Best Route Based on Q Matrix: ',steps)

route = [reverse_nodes_dict[r] for r in steps]

# def new function
def two_q(graph = None, source = None, target = None):
    # graph is osm network loaded
    # source, target are variables of tuple X-Y

    # get the nearest nodes to the locations
    origin_node = ox.get_nearest_node(graph, source)
    destination_node = ox.get_nearest_node(graph, target)

    # printing the closest node id to origin and destination points
    # origin_node, destination_node

    points_list = list(graph.edges())

    # get all nodes from graph
    nodesNumber = graph.number_of_nodes()
    graph.number_of_nodes(),graph.number_of_edges()

    nodes_list= []
    for point in points_list:
        v,u = point[0],point[1]
        nodes_list.append(v)
        nodes_list.append(u)

    nodes_set = set(nodes_list)

    reverse_nodes_dict = dict(enumerate(nodes_set))
    nodes_dict = dict(zip(nodes_set,range(0,nodesNumber)))

    new_points_list = []
    for point in points_list:
        v,u = point[0],point[1]
        newV, newU = nodes_dict[v],nodes_dict[u]
        new_node = (newV, newU)
        new_points_list.append(new_node)

    nodes_dict[origin_node],nodes_dict[destination_node]

    # set initial condition algorithm
    G = nx.Graph()
    G.add_edges_from(new_points_list)
    R = np.matrix(np.ones(shape = (nodesNumber, nodesNumber)))
    R *=-1
    gamma = 0.8
    initial_state = 1

    scores,steps = shortest_Path(nodes_dict[origin_node],nodes_dict[destination_node])
    # plt.plot(scores)
    # save image
    # plot.savefig('Scores.png')
    # plt.show()
    # print('Best Route Based on Q Matrix: ',steps)

    route = [reverse_nodes_dict[r] for r in steps]
    return route

# get route from two_q function
route = two_q(graph = footways, source = origin_point, target = destination_point)

# visualization
# getting coordinates of the nodes
# we will store the longitudes and latitudes in following list
long = []
lat = []
for i in route:
     point = footways.nodes[i]
     long.append(point['x'])
     lat.append(point['y'])

def plot_path(lat, long, origin_point, destination_point):

    # adding the lines joining the nodes
    fig = go.Figure(go.Scattermapbox(
        name = "Path",
        mode = "lines",
        lon = long,
        lat = lat,
        marker = {'size': 10},
        line = dict(width = 4.5, color = 'blue')))
    # adding source marker
    fig.add_trace(go.Scattermapbox(
        name = "Source",
        mode = "markers",
        lon = [origin_point[1]],
        lat = [origin_point[0]],
        marker = {'size': 12, 'color':"red"}))

    # adding destination marker
    fig.add_trace(go.Scattermapbox(
        name = "Destination",
        mode = "markers",
        lon = [destination_point[1]],
        lat = [destination_point[0]],
        marker = {'size': 12, 'color':'green'}))

    # getting center for plots:
    lat_center = np.mean(lat)
    long_center = np.mean(long)
    # defining the layout using mapbox_style
    fig.update_layout(mapbox_style="stamen-terrain",
        mapbox_center_lat = 30, mapbox_center_lon=-80)
    fig.update_layout(margin={"r":0,"t":0,"l":0,"b":0},
                      mapbox = {
                          'center': {'lat': lat_center,
                          'lon': long_center},
                          'zoom': 13})
    fig.show()

plot_path(lat, long, origin_point, destination_point)


import streamlit as st
import copy

from datafunctions import inspeccion_callsample
from currency import currencyoptions,getcurrency
from _dashboard_comite import dashboard_comite

st.set_page_config(layout="wide")

data = inspeccion_callsample()

col1, col2, col3, col4,col5 = st.columns([1,1,2,2,1])
with col1:
    currency    = st.selectbox('Moneda', options=currencyoptions())
    currencycal = getcurrency(currency)

with col2:
    id_inmueble = st.selectbox('ID', options=data['id_inmueble'],key='id_inmueble')
with col5:
    st.markdown('<div>&nbsp</div>', unsafe_allow_html=True)
    if st.button('Actualizar información'):
        st.experimental_memo.clear()
        st.experimental_rerun()

# Dashboard Comite
dashboard_comite(id_inmueble)


"""
Verification of the AFT implementation with the Duffing (Cubic stiffness)
nonlinearity.
"""

import sys
import numpy as np
import unittest

# Python Utilities
sys.path.append('..')
import verification_utils as vutils

sys.path.append('../..')
from tmdsimpy.nlforces.cubic_stiffness import CubicForce

class TestDuffingAFT(unittest.TestCase):

    def __init__(self, *args, **kwargs):
        """
        Define tolerances here for all the tests

        Returns
        -------
        None.

        """
        super(TestDuffingAFT, self).__init__(*args, **kwargs)

        self.analytical_sol_tol = 1e-13 # Tolerance comparing to analytical solution

        self.rtol_grad = 1e-11 # Relative gradient tolerance

    def test_force_fun(self):
        """
        Test the basic force function for cubic stiffness

        Returns
        -------
        None.

        """

        ####################
        # Simple Mapping to spring displacements
        Q = np.array([[1.0, 0], \
                      [-1.0, 1.0], \
                      [0, 1.0]])

        # Weighted / integrated mapping back for testing purposes
        T = np.array([[1.0, 0.25, 0.0], \
                      [0.0, 0.25, 1.0]])

        kalpha = np.array([3, 1, 7])

        duff_force = CubicForce(Q, T, kalpha)

        X = np.array([2, 3])

        gold_F = np.array([24.25, 189.25])

        F,dFdX = duff_force.force(X)

        self.assertLess(np.linalg.norm(F - gold_F), self.analytical_sol_tol,
                        'Simple nalytical force is not as expected.')

        # Check gradient of analytical force
        fun = lambda X: duff_force.force(X)
        grad_failed = vutils.check_grad(fun, X, verbose=False, rtol=3*self.rtol_grad)

        self.assertFalse(grad_failed, 'Incorrect gradient for simple force function.')

    def test_simple_analytical(self):
        """
        First set of test cases. Move the first harmonic and check against
        analytical expansions for AFT.

        Also Verify Gradients for this system

        Returns
        -------
        None.

        """

        ####################
        # Simple Mapping to spring displacements
        Q = np.array([[1.0, 0], \
                      [-1.0, 1.0], \
                      [0, 1.0]])

        # Weighted / integrated mapping back for testing purposes
        T = np.array([[1.0, 0.25, 0.0], \
                      [0.0, 0.25, 1.0]])

        kalpha = np.array([3, 0, 7])

        duff_force = CubicForce(Q, T, kalpha)

        """
        Case 1:
            -Fix Center DOF
            -Only move the first Harmonic
            -Compare to analytical expansion of cos^3/sin^3
        """
        # h = np.array([0, 1, 2, 3]) # Manual Checking expansion / debugging
        h = np.array([0, 1, 2, 3, 4, 5, 6, 7]) # Automate Checking with this
        Nhc = 2*(h !=0).sum() + (h==0).sum() # Number of Harmonic Components

        Nd = Q.shape[1]

        U = np.zeros((Nd*Nhc, 1))

        # First DOF, Cosine Term, Fundamental
        U[Nd+0, 0] = 4

        # Second DOF, Sine Term, Fundamental
        U[2*Nd+1, 0] = 3

        w = 1.5 # Test for various w

        # Calculation with simple first harmonic motion.
        Fnl, dFnldU, dFnldw = duff_force.aft(U, w, h)

        # # Analytically Verify Force expansion:
        # # X^3*cos^3(x) = X^3*( 3/4*cos(x) + 1/4*cos(3x) )
        # # X^3*sin^3(x) = X^3*(3/4*sin(x) - 1/4*sin(3x)
        Fnl_analytical = np.zeros_like(Fnl)
        Fnl_analytical[Nd+0] = T[0,0]*( 0.75*kalpha[0]*(Q[0,0]*U[Nd+0, 0])**3 )
        Fnl_analytical[5*Nd+0] = T[0,0]*( 0.25*kalpha[0]*(Q[0,0]*U[Nd+0, 0])**3 )

        Fnl_analytical[2*Nd+1] = T[1,2]*( 0.75*kalpha[2]*(Q[2,1]*U[2*Nd+1, 0])**3 )
        Fnl_analytical[6*Nd+1] = T[1,2]*( -0.25*kalpha[2]*(Q[2,1]*U[2*Nd+1, 0])**3 )

        analytical_sol_error = np.linalg.norm(Fnl - Fnl_analytical)

        self.assertLess(analytical_sol_error, self.analytical_sol_tol,
                        'Numerical AFT does not match analytical Duffing AFT solution.')

        # Numerically Verify Gradient
        fun = lambda U: duff_force.aft(U, w, h)[0:2]
        grad_failed = vutils.check_grad(fun, U, verbose=False, rtol=self.rtol_grad)

        self.assertFalse(grad_failed, 'Incorrect AFT gradient w.r.t. displacements')

        # Numerically Verify Frequency Gradient
        fun = lambda w: duff_force.aft(U, w[0], h)[0::2]
        grad_failed = vutils.check_grad(fun, np.array([w]), verbose=False, rtol=self.rtol_grad)

        self.assertFalse(grad_failed, 'Incorrect AFT gradient w.r.t. frequency')

    def test_mixed_harmonics(self):
        """
        Test Gradients for cases where only some harmonics are included instead
        of all of the harmonics from 0 to hmax

        Returns
        -------
        None.

        """

        np.random.seed(1023)

        w = 1.5

        Q = np.array([[1.0, 0], \
                      [-1.0, 1.0], \
                      [0, 1.0]])

        # Skip the 4th harmonic
        h = np.array([0, 1, 2, 3, 5, 6, 7])
        kalpha = np.array([3, 5, 7])

        # Weighted / integrated mapping back for testing purposes
        T = np.array([[1.0, 0.25, 0.0], \
                      [0.0, 0.25, 0.5]])

        Nhc = 2*(h !=0).sum() + (h==0).sum() # Number of Harmonic Components
        Nd = Q.shape[1]

        U = np.random.rand(Nd*Nhc, 1)

        duff_force = CubicForce(Q, T, kalpha)

        fun = lambda U: duff_force.aft(U, w, h)[0:2]
        grad_failed = vutils.check_grad(fun, U, verbose=False, rtol=self.rtol_grad)

        self.assertFalse(grad_failed, 'Incorrect AFT gradient w.r.t. displacements')

        # Numerically Verify Frequency Gradient
        fun = lambda w: duff_force.aft(U, w[0], h)[0::2]
        grad_failed = vutils.check_grad(fun, np.array([w]), verbose=False, rtol=self.rtol_grad)

        self.assertFalse(grad_failed, 'Incorrect AFT gradient w.r.t. frequency')

        ######################
        # Test without zeroth harmonic + skipping the 4th harmonic

        h = np.array([1, 2, 3, 5, 6, 7])

        Nhc = 2*(h !=0).sum() + (h==0).sum() # Number of Harmonic Components
        Nd = Q.shape[1]

        U = np.random.rand(Nd*Nhc, 1)

        fun = lambda U: duff_force.aft(U, w, h)[0:2]
        grad_failed = vutils.check_grad(fun, U, verbose=False, rtol=self.rtol_grad)

        self.assertFalse(grad_failed, 'Incorrect AFT gradient w.r.t. displacements')

        # Numerically Verify Frequency Gradient
        fun = lambda w: duff_force.aft(U, w[0], h)[0::2]
        grad_failed = vutils.check_grad(fun, np.array([w]), verbose=False, rtol=self.rtol_grad)

        self.assertFalse(grad_failed, 'Incorrect AFT gradient w.r.t. frequency')

if __name__ == '__main__':
    unittest.main()


import numpy as np
import tensorflow as tf
import tensorflow_io as tfio

class NormalizeSpectrograms(tf.keras.layers.Layer):
    def __init__(self, learn_log, do_augmentation, **kwargs):
        super().__init__(**kwargs)
        self.learn_log = learn_log
        self.do_augmentation = do_augmentation

    @tf.function(
        experimental_relax_shapes=True
    )  # (input_signature=(tf.TensorSpec(shape=[32, 20], dtype=tf.float32)))
    def normalize(self, x, training=None):
        print("Tracing with:", x.shape)
        one = x
        if self.learn_log:
            spec = tf.stack([one, one, one, one])
        else:
            row_mean = tf.expand_dims(tf.math.reduce_mean(x, axis=1), 1)
            row_std = tf.expand_dims(tf.add(tf.math.reduce_std(x, axis=1), 0.001), 1)
            two = (one - row_mean) / row_std

            three = tf.math.divide(
                tf.math.subtract(x, tf.math.reduce_mean(x)),
                tf.math.reduce_std(x),
            )
            four = tf.math.divide_no_nan(x, tf.math.reduce_max(x))
            spec = tf.stack([one, two, three, four])
        if training and self.do_augmentation:

            if self.learn_log:
                mult = 1.0 + np.random.randn(1, 1, 1) * 0.1
                mult = np.clip(mult, 0.1, 200)
                spec *= mult
            else:
                spec = tf.math.multiply(spec, 1.0 + np.random.randn(1, 1, 1) * 0.1)
                spec = tf.add(spec, np.random.randn(1, 1, 1) * 0.1)
                if np.random.rand() > 0.9:
                    print("in random")
                    spec = tf.add(
                        spec, tf.multiply(tf.roll(spec, 1, axis=0), np.random.randn())
                    )
        spec = tf.transpose(spec, perm=[1, 2, 0])
        return spec

    @tf.function
    def vectorize(self, spec, training=None):
        return tf.vectorized_map(self.normalize, spec, training)

    def call(self, spec, training=None):
        res = self.vectorize(spec, training)
        return res

    def get_config(self):
        config = {
            "do_augmentation": self.do_augmentation,
            "learn_log": self.learn_log,
        }
        config.update(super().get_config())

        return config

class MaskSpectrograms(tf.keras.layers.Layer):
    """
    .. csv-table::
        :header: "Option name", "Description", "Default", "Type"

        "time_mask", "Options for the time masking augmentation", , "dict"
        "prop", "proportion of the time the masking will occur", 90, "int"
        "value", "", shape[0] / 2, "int"
        "freq_mask", "Options for the frequency asking augmentation", , "dict"
        "prop", "proportion of the time the masking will occur", 90, "int"
        "value", "", shape[1] / 2 , "int"

    """

    def __init__(self, time_mask, freq_mask, **kwargs):
        super().__init__(**kwargs)
        self.time_mask = time_mask
        self.time_opts = time_mask if isinstance(time_mask, dict) else {}
        self.freq_mask = freq_mask
        self.freq_opts = freq_mask if isinstance(freq_mask, dict) else {}

    @tf.function(experimental_relax_shapes=True)
    def mask(self, spec):
        # * Use freq_mask for time masking  and vice-versa as our spectrograms shapes
        # * are inverted compared to tf ones (freq in the first dim, time in the seconf)
        if self.time_mask:
            # * Allow mask to be a boolean instead
            if tf.random.uniform(  # pylint: disable=unexpected-keyword-arg
                shape=(), minval=1, maxval=100, dtype=tf.int32
            ) <= self.time_opts.get("prop", 75):

                spec = tfio.audio.freq_mask(
                    spec, param=self.time_opts.get("value", int(spec.shape[1] / 2))
                )
        if self.freq_mask:
            if tf.random.uniform(  # pylint: disable=unexpected-keyword-arg
                shape=(), minval=1, maxval=100, dtype=tf.int32
            ) <= self.freq_opts.get("prop", 75):
                spec = tfio.audio.time_mask(
                    spec, param=self.freq_opts.get("value", int(spec.shape[0] / 2))
                )
        return spec

    @tf.function
    def vectorize(self, spec):
        return tf.map_fn(self.mask, spec)

    def call(self, spec, training=None):
        if training:
            spec = self.vectorize(spec)
        return spec

    def get_config(self):
        config = {
            "time_mask": self.time_mask,
            "freq_mask": self.freq_mask,
        }
        config.update(super().get_config())

        return config


from BMI500HW7.main import aki_model

aki = aki_model()
aki.load_data()
aki.get_umap()
aki.cluster()
aki.visualization(colorby="cluster")
# aki.visualization("tsne", colorby="cluster", save = "./figs/fig_cluster_tsne.png")

vars = ['GENDER', "AKI_STAGE", 'BICAR_AVG', 'CHLO_AVG', 'UN_AVG', 'HEM_AVG', 'PC_AVG', 'uo_rt_6hr_avg', 'uo_rt_6hr_max', 'uo_rt_6hr_min', 'uo_rt_12hr_avg', 'uo_rt_12hr_max', 'uo_rt_12hr_min', 'uo_rt_24hr_avg', 'uo_rt_24hr_max', 'uo_rt_24hr_min', 'creat_diff', 'creat_avg', 'creat_baseline']
for i in vars:
    aki.visualization(colorby=i)

aki.train_on_data()
aki.predict([["F",24.000000000000007,107.49999999999994,18.5,27.850000000000012,55.499999999999993,1.621664706,1.9795,1.2941,1.497776471,1.6923,1.3012,1.441717647,1.5242,1.2687,0.099999999999999978,0.45,0.4]])

vals = ['GENDER', 'BICAR_AVG', 'CHLO_AVG', 'UN_AVG', 'HEM_AVG', 'PC_AVG', 'uo_rt_6hr_avg', 'uo_rt_6hr_max', 'uo_rt_6hr_min', 'uo_rt_12hr_avg', 'uo_rt_12hr_max', 'uo_rt_12hr_min', 'uo_rt_24hr_avg', 'uo_rt_24hr_max', 'uo_rt_24hr_min', 'creat_diff', 'creat_avg', 'creat_baseline']

# y_pred = aki.predict(aki.data[vals])
# from sklearn.metrics import accuracy_score
# accuracy_score(aki.data["AKI_STAGE"], y_pred)
# # 0.7322385413052697


import csv
import tqdm
import click
import logging
import numpy as np
import matplotlib.pyplot as plt
from torch.utils.data import Dataset

from string import digits

import torch
import torch.nn as nn
import torch.optim as optim
import torchvision
import torch.nn.functional as F

seed = 42
epochs = 20
batch_size = 32
learning_rate = 1e-3
context_frames = 6
sequence_length = 16

context_epochs = 30
context_batch_size = 1
context_learning_rate = 1e-3

test_train_split = 0.9  # precentage of train data from total

torch.manual_seed(seed)
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")#  use gpu if available

class ContextAutoEncoder(nn.Module):
	def __init__(self, **kwargs):
		super().__init__()
		logger = kwargs["logger"]
		logger.info("Auto Encoder Initialised")

		self.encoder_hidden1_layer = nn.Linear(in_features=kwargs["input_shape"], out_features=128)
		self.encoder_hidden2_layer = nn.Linear(in_features=128, out_features=74)
		self.encoder_output_layer = nn.Linear(in_features=74, out_features=48)

		self.decoder_hidden1_layer = nn.Linear(in_features=48, out_features=74)
		self.decoder_hidden2_layer = nn.Linear(in_features=74, out_features=128)
		self.decoder_output_layer = nn.Linear(in_features=128, out_features=kwargs["input_shape"])

	def forward(self, features):
		activation1 = self.encoder_hidden1_layer(features)
		activation1 = torch.relu(activation1)

		activation2 = self.encoder_hidden2_layer(activation1)
		activation2 = torch.relu(activation2)

		code = self.encoder_output_layer(activation2)
		code = torch.sigmoid(code)

		activation1 = self.decoder_hidden1_layer(code)
		activation1 = torch.relu(activation1)

		activation2 = self.decoder_hidden2_layer(activation1)
		activation2 = torch.relu(activation2)

		activation3 = self.decoder_output_layer(activation2)
		reconstructed = torch.sigmoid(activation3)

		return reconstructed

	def encoder(self, features):
		activation1 = self.encoder_hidden1_layer(features)
		activation1 = torch.relu(activation1)

		activation2 = self.encoder_hidden2_layer(activation1)
		activation2 = torch.relu(activation2)

		code = self.encoder_output_layer(activation2)
		code = torch.sigmoid(code)

		return code

class Feedforward(torch.nn.Module):
	def __init__(self, input_size, hidden_size, output_size):
		super(Feedforward, self).__init__()
		self.input_size  = input_size
		self.hidden_size = hidden_size
		self.output_size = output_size
		self.fc1 = torch.nn.Linear(self.input_size, self.hidden_size)
		self.relu1 = torch.nn.ReLU()
		self.fc2 = torch.nn.Linear(self.hidden_size, self.output_size)
		self.relu2 = torch.nn.ReLU()

	def forward(self, x):
		hidden = self.fc1(x)
		relu = self.relu1(hidden)
		output = self.fc2(relu)
		output = self.relu2(output)
		return output

class FullModelTest(nn.Module):
	def __init__(self):
		super(FullModelTest, self).__init__()
		self.lstm1 = nn.LSTM(48, 48).to(device)
		self.lstm2 = nn.LSTM(48*2, 48*2).to(device)  # Context + Tactle
		self.ff1 = Feedforward(input_size=6, hidden_size=48, output_size=48).to(device) # actions
		self.lstm3 = nn.LSTM(48*3, 48*3).to(device)
		self.ff2 = Feedforward(input_size=144, hidden_size=144, output_size=96).to(device)
		self.ff3 = Feedforward(input_size=96, hidden_size=96, output_size=48).to(device)
		self.lstm4 = nn.LSTM(48, 48).to(device)

	def forward(self, tactiles, actions, context):
		state = actions[0]
		state.to(device)
		batch_size__ = tactiles.shape[1]
		outputs = []
		hidden1 = (torch.rand(1,batch_size__,48).to(device), torch.rand(1,batch_size__,48).to(device))
		hidden2 = (torch.rand(1,batch_size__,48*2).to(device), torch.rand(1,batch_size__,48*2).to(device))
		hidden3 = (torch.rand(1,batch_size__,48*3).to(device), torch.rand(1,batch_size__,48*3).to(device))
		hidden4 = (torch.rand(1,batch_size__,48).to(device), torch.rand(1,batch_size__,48).to(device))
		for index, (sample_tactile, sample_action, sample_context) in enumerate(zip(tactiles.squeeze(), actions.squeeze(), context)):
			sample_tactile.to(device)
			sample_action.to(device)
			sample_context.to(device)
			# 2. Run through lstm:
			if index > context_frames-1:
				out1, hidden1 = self.lstm1(out6, hidden1)
				context_and_tactile = torch.cat((sample_context, out1.squeeze()), 1)
				out2, hidden2 = self.lstm2(context_and_tactile.unsqueeze(0), hidden2)
				# 1. Run Actions through FFN:
				state_action = torch.cat((state, sample_action.squeeze()), 1)
				state_action_ouput = self.ff1.forward(state_action.unsqueeze(0))
				action_tactile = torch.cat((state_action_ouput.squeeze(), out2.squeeze()), 1)
				out3, hidden3 = self.lstm3(action_tactile.unsqueeze(0), hidden3)
				# scale back down to 48 length vector for output
				out4 = self.ff2.forward(out3)
				out5 = self.ff3.forward(out4)
				# final LSTM
				out6, hidden4 = self.lstm4(out5, hidden4)
			else:
				out1, hidden1 = self.lstm1(sample_tactile.unsqueeze(0), hidden1)
				context_and_tactile = torch.cat((sample_context, out1.squeeze()), 1)
				out2, hidden2 = self.lstm2(context_and_tactile.unsqueeze(0), hidden2)
				# 1. Run Actions through FFN:
				state_action = torch.cat((state, sample_action.squeeze()), 1)
				state_action_ouput = self.ff1.forward(state_action.unsqueeze(0))
				action_tactile = torch.cat((state_action_ouput.squeeze(), out2.squeeze()), 1)
				out3, hidden3 = self.lstm3(action_tactile.unsqueeze(0), hidden3)
				# scale back down to 48 length vector for output
				out4 = self.ff2.forward(out3)
				out5 = self.ff3.forward(out4)
				# final LSTM
				out6, hidden4 = self.lstm4(out5, hidden4)
			outputs.append(out6.squeeze())
		return torch.stack(outputs)

class ModelTrainer:
	def __init__(self, BG, logger, data_dir):
		self.data_dir = data_dir
		self.BG = BG
		## Train the context model:
		self.context_model = ContextAutoEncoder(input_shape=context_epochs*48, logger=logger).to(device)
		self.context_optimizer = optim.Adam(self.context_model.parameters(), lr=learning_rate)  # create an optimizer object || Adam optimizer with learning rate 1e-3
		self.context_criterion = nn.L1Loss()  # mean-squared error loss
		self.train_context_loader, self.test_context_loader = self.BG.load_context_data()

		### Train the LSTM chain:
		self.train_full_loader, self.test_full_loader = self.BG.load_full_data()
		self.full_model = FullModelTest()
		self.criterion = nn.L1Loss()
		self.optimizer = optim.Adam(self.full_model.parameters(), lr=learning_rate)

	def train_full_model(self):
		previous_test_mean_loss = 1.0
		progress_bar = tqdm.tqdm(range(0, epochs), total=(epochs*len(self.train_full_loader)))
		mean_test = 0
		for epoch in progress_bar:
			loss = 0
			losses = 0.0
			for index, batch_features in enumerate(self.train_full_loader):
				# 1. Calculate context model:
				context_data_list = []
				for context_data in batch_features[0]:
					context = context_data.view(-1, context_epochs*48).to(device)
					context = self.context_model.encoder(context)  # [0]
					context_list = []
					for sequence in range(sequence_length):
						context_list.append(context.cpu().detach().numpy())
					context_data_list.append(context_list)
				context_data_list = np.asarray(context_data_list).squeeze()
				context_data_list = torch.FloatTensor(context_data_list)

				# 2. Reshape data and send to device:
				context = context_data_list.permute(1,0,2).to(device)
				tactile = batch_features[1].permute(1,0,2).to(device)
				action = batch_features[2].permute(1,0,2).to(device)
				state = batch_features[3].permute(1,0,2).to(device)

				tactile_predictions = self.full_model.forward(tactiles=tactile, actions=action, context=context)  # Step 3. Run our forward pass.
				self.optimizer.zero_grad()
				loss = self.criterion(tactile_predictions.unsqueeze(0).to(device), tactile)
				loss.backward()
				self.optimizer.step()

				losses += loss.item()
				if index:
					mean = losses / index
				else:
					mean = 0
				progress_bar.set_description("epoch: {}, ".format(epoch) + "loss: {:.4f}, ".format(float(loss.item())) + "mean loss: {:.4f}, ".format(mean))
				progress_bar.update()
			print("Training mean loss: {:.4f}, ".format(losses / index))

			test_losses = 0.0
			test_loss = 0.0
			with torch.no_grad():
				for index__, batch_features in enumerate(self.test_full_loader):
					# 1. Calculate context model:
					context_data_list = []
					for context_data in batch_features[0]:
						context = context_data.view(-1, context_epochs*48).to(device)
						context = self.context_model.encoder(context)  # [0]
						context_list = []
						for sequence in range(sequence_length):
							context_list.append(context.cpu().detach().numpy())
						context_data_list.append(context_list)
					context_data_list = np.asarray(context_data_list).squeeze()
					context_data_list = torch.FloatTensor(context_data_list)

					# 2. Reshape data and send to device:
					context = context_data_list.permute(1,0,2).to(device)
					tactile = batch_features[1].permute(1,0,2).to(device)
					action = batch_features[2].permute(1,0,2).to(device)
					state = batch_features[3].permute(1,0,2).to(device)

					tactile_predictions = self.full_model.forward(tactiles=tactile, actions=action, context=context)  # Step 3. Run our forward pass.
					self.optimizer.zero_grad()
					test_loss = self.criterion(tactile_predictions.unsqueeze(0).to(device), tactile)
					test_losses += test_loss.item()

			print("Test mean loss: {:.4f}, ".format(test_losses / index__))

			if previous_test_mean_loss < test_losses / index__:
				print("Early stopping")
				break
			else:
				previous_test_mean_loss = test_losses / index__

	def train_context_model(self):
		for context_epoch in range(context_epochs):
			loss = 0
			for batch_features in self.train_context_loader:
				batch_features = batch_features.view(-1, context_epochs*48).to(device)  # reshape mini-batch data to [N, 784] matrix load it to the active device
				self.context_optimizer.zero_grad()  # reset the gradients back to zero PyTorch accumulates gradients on subsequent backward passes
				outputs = self.context_model(batch_features)  # compute reconstructions
				train_loss = self.context_criterion(outputs, batch_features)  # compute training reconstruction loss
				train_loss.backward()  # compute accumulated gradients
				self.context_optimizer.step()  # perform parameter update based on current gradients
				loss += train_loss.item()  # add the mini-batch training loss to epoch loss

			loss = loss / len(self.train_context_loader)  # compute the epoch training loss
			print("epoch : {}/{}, recon loss = {:.8f}".format(context_epoch + 1, context_epochs, loss)) # display the epoch training loss

	def test_context_model(self):
		loss = 0
		with torch.no_grad():
			for batch_features in self.test_context_loader:
				test_examples = batch_features.view(-1, context_epochs*48).to(device)
				reconstruction = self.context_model(test_examples)
				train_loss = self.context_criterion(reconstruction, test_examples)
				loss += train_loss.item()

		loss = loss / len(self.test_context_loader)  # compute the epoch training loss
		print("Test loss = {:.8f}".format(loss)) # display the epoch training loss

class TestModel():
	def __init__(self, context_model, model, data_dir):
		data_map = []
		with open(data_dir + 'map.csv', 'r') as f:  # rb
			reader = csv.reader(f)
			for row in reader:
				data_map.append(row)

		if len(data_map) <= 1: # empty or only header
			logger.error("No file map found")
			exit()

		test_sample = []
		order = []
		data_map = data_map[int(len(data_map)*test_train_split):-1]
		sample_to_test = data_map[0][8][0]
		for sample in data_map:
			if sample_to_test in sample[8]:
				test_sample.append(sample)
				order.append(int(''.join(c for c in sample[9] if c in digits)))

		sorted_data = [x for _, x in sorted(zip(order, test_sample))]

		dataset_sample = []
		context_file_names = []
		context_data = []
		for value in sorted_data:  # ignore header
			state = np.float32(np.load(data_dir + '/' + value[4]))
			dataset_sample.append([np.load(data_dir + value[8]),
								 np.float32(np.load(data_dir + '/' + value[2])),
								 np.float32(np.load(data_dir + '/' + value[3])),
								 np.asarray([state[0] for i in range(0, len(state))])])

		# test model on the full test sample:
		transform = torchvision.transforms.Compose([torchvision.transforms.ToTensor()])
		train_loader = torch.utils.data.DataLoader(dataset_sample, batch_size=10, shuffle=False)
		tactile_predictions = []
		with torch.no_grad():
			for index__, batch_features in enumerate(train_loader):
				# 1. Calculate context model:
				context_data_list = []
				for context_data in batch_features[0]:
					context = context_data.view(-1, context_epochs*48).to(device)
					context = context_model.encoder(context)  # [0]
					context_list = []
					for sequence in range(sequence_length):
						context_list.append(context.cpu().detach().numpy())
					context_data_list.append(context_list)
				context_data_list = np.asarray(context_data_list).squeeze()
				context_data_list = torch.FloatTensor(context_data_list)

				# 2. Reshape data and send to device:
				context = context_data_list.permute(1,0,2).to(device)
				tactile = batch_features[1].permute(1,0,2).to(device)
				action = batch_features[2].permute(1,0,2).to(device)
				state = batch_features[3].permute(1,0,2).to(device)

				tactile_predictions.append(model.forward(tactiles=tactile, actions=action, context=context))  # Step 3. Run our forward pass.

		# calculate tactile values for full sample:
		time_step_to_test = 0    # [batch_set, 16, 10, 48]
		predicted_data = []
		for batch_set in tactile_predictions:
			for batch in range(0, len(batch_set[0])):
				values = batch_set[time_step_to_test][batch]
				predicted_data.append(values)

		# plot taxles for each time_step and each force dimension:
		# fig, axs = plt.subplots(1,3, figsize=(15, 6), facecolor='w', edgecolor='k')
		# fig.subplots_adjust(hspace = .5, wspace=.001)
		# axs = axs.ravel()
		index = 0
		titles = ["sheerx", "sheery", "normal"]
		for j in range(3):
			for i in range(16):
				groundtruth_taxle = []
				predicted_taxel = []
				for k, time_step in enumerate(dataset_sample):
					groundtruth_taxle.append(time_step[1][0][index]) # tactile_image, at time[0], taxel [0]
					data = predicted_data[k].cpu().detach().numpy()
					data = data.reshape(int(len(data)/3), 3)
					predicted_taxel.append(data.T.flatten()[index])
				index += 1
				plt.plot(predicted_taxel, alpha=0.3)
				plt.plot(groundtruth_taxle)
				plt.ylim([0, 1])
				# axs[j].plot(groundtruth_taxle)
				# axs[j].plot(predicted_taxel, alpha=0.1)
				# axs[j].set_title(titles[j])
				plt.show()

class BatchGenerator:
	def __init__(self, data_dir, logger):
		self.data_dir = data_dir
		data_map = []
		with open(data_dir + 'map.csv', 'r') as f:  # rb
			reader = csv.reader(f)
			for row in reader:
				data_map.append(row)

		if len(data_map) <= 1: # empty or only header
			logger.error("No file map found")
			exit()

		self.data_map = data_map

	def load_context_data(self):
		dataset_train = ContextDataSet(self.data_dir, self.data_map, train=True)
		dataset_test = ContextDataSet(self.data_dir, self.data_map, train=False)
		transform = torchvision.transforms.Compose([torchvision.transforms.ToTensor()])
		train_loader = torch.utils.data.DataLoader(dataset_train, batch_size=context_batch_size, shuffle=True)
		test_loader = torch.utils.data.DataLoader(dataset_test, batch_size=context_batch_size, shuffle=True)
		return train_loader, test_loader

	def load_full_data(self):
		dataset_train = FullDataSet(self.data_dir, self.data_map, train=True)
		dataset_test = FullDataSet(self.data_dir, self.data_map, train=False)
		transform = torchvision.transforms.Compose([torchvision.transforms.ToTensor()])
		train_loader = torch.utils.data.DataLoader(dataset_train, batch_size=batch_size, shuffle=True)
		test_loader = torch.utils.data.DataLoader(dataset_test, batch_size=batch_size, shuffle=True)
		return train_loader, test_loader

class ContextDataSet():
	def __init__(self, data_dir, data_map, train=True):
		context_file_names = []
		context_data = []
		for value in data_map[1:]:  # ignore header
			if value[8] not in context_file_names:
				context_file_names.append(value[8])
				context_data.append(np.load(data_dir + value[8]))
		if train:
			self.samples = context_data[0:int(len(context_data)*test_train_split)]
		else:
			self.samples = context_data[int(len(context_data)*test_train_split):-1]
		data_map = None

	def __len__(self):
		return len(self.samples)

	def __getitem__(self,idx):
		return(self.samples[idx])

class FullDataSet():
	def __init__(self, data_dir, data_map, train=True):
		dataset_full = []
		for value in data_map[1:]:  # ignore header
			state = np.float32(np.load(data_dir + '/' + value[4]))
			dataset_full.append([np.load(data_dir + value[8]),
								 np.float32(np.load(data_dir + '/' + value[2])),
								 np.float32(np.load(data_dir + '/' + value[3])),
								 np.asarray([state[0] for i in range(0, len(state))])])
		if train:
			self.samples = dataset_full[0:int(len(dataset_full)*test_train_split)]
		else:
			self.samples = dataset_full[int(len(dataset_full)*test_train_split):-1]
		data_map = None

	def __len__(self):
		return len(self.samples)

	def __getitem__(self,idx):
		return(self.samples[idx])

@click.command()
@click.option('--data_dir', type=click.Path(exists=True), default='/home/user/Robotics/Data_sets/slip_detection/vector_normalised_001/', help='Directory containing data.')
def main(data_dir):
	logger = logging.getLogger(__name__)

	BG = BatchGenerator(data_dir, logger)
	MT = ModelTrainer(BG, logger, data_dir)

	MT.train_context_model()
	MT.test_context_model()
	MT.train_full_model()
	TestModel(MT.context_model, MT.full_model, data_dir)

if __name__ == '__main__':
	main()


from django import template
from django.conf import settings

register = template.Library()

@register.inclusion_tag("uicore/login/login.html", takes_context=True)
def login_form(context, next, form):
    """
    Provides a login form, or link to a 3rd party OIDC authentication provider based on settings.
    """

    return {
        "user": context.request.user,
        "next": next,
        "form": form,
        "use_oidc": settings.USE_OIDC,
        "maintenance_mode": settings.MAINTENANCE_MODE
    }


# importing important libraries
from sklearn.linear_model import LinearRegression
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split

# reading csv file
data = pd.read_csv("abalone.csv")
# creating traing and testing data
y = data["Rings"]
X = data.iloc[:, :-1]
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.33, random_state=42)

train = pd.concat([X_train, y_train], axis=1, join='inner')
test = pd.concat([X_test, y_test], axis=1, join='inner')

# saving the training and testing datasets as csv files
train.to_csv('abalone-train.csv', index=False)
test.to_csv('abalone-test.csv', index=False)

# finding correlation
print(data.corr())

# since shell weight ihas maximum correlation
x_train = X_train["Shell weight"]
x_test = X_test["Shell weight"]

# model building
reg = LinearRegression()
x_train = np.array(x_train)
y_train = np.array(y_train)
x_train = x_train.reshape(-1, 1)
y_train = y_train.reshape(-1, 1)

x_test = np.array(x_test)
x_test = x_test.reshape(-1, 1)
y_test = np.array(y_test)
y_test = y_test.reshape(-1, 1)
reg.fit(x_train, y_train)

# predicting x_test and using it plot line
y = reg.predict(x_test)
# scattter plot b/w selected attribute and target attribute
plt.scatter(x_train, y_train)
plt.plot(x_test, y, color="r")
plt.xlabel("Shell weight")
plt.ylabel("Rings")
plt.show()

# predicting using obtained model
pred_1 = reg.predict(x_train)
pred_2 = reg.predict(x_test)

# calculating rmse value

def rmse(predictions, targets):
    predictions = np.array(predictions)
    targets = np.array(targets)
    return np.sqrt(((predictions - targets) ** 2).mean())

print("prediction accuracy on the training data using root mean squared error",
      round(rmse(pred_1, y_train), 3))
print("prediction accuracy on the test data using root mean squared error",
      round(rmse(pred_2, y_test), 3))

# scatter plot of actual Rings (x-axis) vs predicted Rings (y-axis) on the test data.
plt.scatter(y_test, pred_2, color="red", alpha=0.5)
plt.xlabel("actual")
plt.ylabel("predicted")
plt.show()

Data classification using Bayes Classifier with Gaussian Mixture Model (GMM);/code partB/Question1.py


# -*- coding: utf-8 -*-
"""
Created on Thu Nov 11 10:42:16 2021

@author: Notebook
"""

while True:
    try:
        x = int(input("Masukkan angka: "))
        break
    except ValueError:
        print("Oops!  Inputan tidak valid.  silakan coba lagi...")

"""
Kode diatas akan berhenti jika inputan benar yaitu nomr bukan string
"""


# coding=utf-8
import datetime
import logging

from django.http import JsonResponse

from apps.Platform.models import ApiLog

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

def getClientIp(request):
    x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
    if x_forwarded_for:
        ip = x_forwarded_for.split(',')[0]
    else:
        ip = request.META.get('REMOTE_ADDR')
    return ip

def record(typeName):
    def _record(func):
        def wrapper(request):
            logger.debug("func===%s", func)
            timenow = datetime.datetime.now()
            log = ApiLog()
            log.apiType = typeName
            log.requestIp = getClientIp(request)
            log.requestTime = timenow
            log.request = request.POST.dict()

            try:
                responseText = func(request)
            except Exception as e:
                logger.exception("API %s 执行出现错误: %s", request.path, e)
                responseText = {
                    'code': 500,
                    'error': str(e),
                    'message': None,
                }

            log.response = responseText

            try:
                jsonResponse = JsonResponse(responseText)
            except Exception as e:
                logger.exception("%s生成API结果出现错误: %s", request.path, e)
                responseText = {
                    'code': 500,
                    'error': str(e),
                    'message': None,
                }
                log.response = responseText
                jsonResponse = JsonResponse(responseText)

            log.save()
            logger.debug("!~~~~~~~~~~~~~~~~~~~~~~~~ jsonResponse: %s", jsonResponse)
            return jsonResponse
        return wrapper
    return _record


import unittest
import sys
import os

# Trick to avoid adding this module to the path forever or via the shell.
dirname = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')
sys.path.append(dirname)

from exercises.classify_functions_code import reverse_compliment, count_occurance, unique_kmers, count_kmers
from solutions.optimize_functions_code import unique_kmers as optimized_unique_kmers, count_kmers as optimized_count_kmers
from solutions.find_kmer_clumps_code import clumping_kmers

'''
These tests are not exhaustive/comprehensive, just be aware of that.
'''

class TestGenomeFunctions(unittest.TestCase):

    def test_reverse_compliment(self):
        '''
        This function takes a string representing a dna sequence
        as input, and returns its reverse complement.
        '''
        cases = [
            ('', ''),
            ('A', 'T'),
            ('T', 'A'),
            ('C', 'G'),
            ('G', 'C'),
            ('ATCG', 'CGAT')
        ]

        for sequence, expected_result in cases:
            result = reverse_compliment(sequence)
            failure_message = f'Failed on input sequence: {sequence}, expected result: {expected_result}, actual_result: {result}'
            self.assertEqual(result, expected_result, failure_message)

    def test_count_occurance(self):
        '''
        This function accepts two strings, one representing the genome
        of an organism (or a part of the genome) and another representing
        some smaller DNA/RNA pattern. It returns the number of times this
        pattern occurs within the provided genome.
        '''
        cases = [
            ('G', 'G', 1),
            ('AAA', 'A', 3),
            ('AAA', 'AA', 2),
            ('TAGTAGTAG', 'TAG', 3),
            ('AATAATAT', 'AT', 3),
            ('ATCGATCG', 'ATCG', 2),
        ]

        for genome, pattern, expected_result in cases:
            result = count_occurance(genome, pattern)
            failure_message = f'Failed on input genome: {genome}, pattern: {pattern}, expected result: {expected_result}, actual_result: {result}'
            self.assertEqual(result, expected_result, failure_message)

    def test_unique_kmers(self):
        '''
        This function takes in a string representing the genome of an
        organism (or section of a genome) and an integer value k. It
        returns a list of all the unique substrings of length k
        (often called k-mers) contained in that genome.
        '''
        cases = [
            ('A', 1, ['A']),
            ('T', 1, ['T']),
            ('C', 1, ['C']),
            ('G', 1, ['G']),
            ('ATCG', 1, ['A', 'T', 'C', 'G']),
            ('ATCG', 2, ['AT', 'TC', 'CG']),
            ('ATCG', 3, ['ATC', 'TCG']),
            ('ATCG', 4, ['ATCG']),
            ('AAATTTCCCGGG', 2, ['AA', 'AT', 'TT', 'TC', 'CC', 'CG', 'GG'])
        ]

        for genome, k, expected_kmers in cases:
            kmers = unique_kmers(genome, k)
            failure_message = f'Failed on input genome: {genome}, k: {k}. Expected: {expected_kmers} Actual: {kmers}.'
            self.assertEqual(kmers, expected_kmers, failure_message)

    def test_optimized_unique_kmers(self):
        '''
        This function takes in a string representing the genome of an
        organism (or section of a genome) and an integer value k. It
        returns a list of all the unique substrings of length k
        (often called k-mers) contained in that genome.
        '''
        cases = [
            ('A', 1, ['A']),
            ('T', 1, ['T']),
            ('C', 1, ['C']),
            ('G', 1, ['G']),
            ('ATCG', 1, ['A', 'T', 'C', 'G']),
            ('ATCG', 2, ['AT', 'TC', 'CG']),
            ('ATCG', 3, ['ATC', 'TCG']),
            ('ATCG', 4, ['ATCG']),
            ('AAATTTCCCGGG', 2, ['AA', 'AT', 'TT', 'TC', 'CC', 'CG', 'GG'])
        ]

        for genome, k, expected_kmers in cases:
            kmers = optimized_unique_kmers(genome, k)
            failure_message = f'Failed on input genome: {genome}, k: {k}. Expected: {expected_kmers} Actual: {kmers}.'
            self.assertEqual(kmers, expected_kmers, failure_message)

    def test_count_kmers(self):
        '''
        This function accepts a string representing a genome
        and an integer k. It returns a dictionary which maps
        each occurring k-mer to the number of times that k-mer
        occurs within the genome.
        '''
        cases = [
            ('A', 1, {'A': 1}),
            ('T', 1, {'T': 1}),
            ('C', 1, {'C': 1}),
            ('G', 1, {'G': 1}),
            ('ATCG', 1, {'A': 1, 'T': 1, 'C': 1, 'G': 1}),
            ('ATCG', 2, {'AT': 1, 'TC': 1, 'CG': 1}),
            ('ATCG', 3, {'ATC': 1, 'TCG': 1}),
            ('ATCG', 4, {'ATCG': 1}),
            ('AAATTTCCCGGG', 2, {'AA': 2, 'AT': 1, 'TT': 2, 'TC': 1, 'CC': 2, 'CG': 1, 'GG': 2})
        ]

        for genome, k, expected_kmer_counts in cases:
            kmer_counts = count_kmers(genome, k)
            failure_message = f'Failed on input genome: {genome}, k: {k}. Expected: {expected_kmer_counts} Actual: {kmer_counts}.'
            self.assertEqual(kmer_counts, expected_kmer_counts, failure_message)

    def test_optimized_count_kmers(self):
        '''
        This function accepts a string representing a genome
        and an integer k. It returns a dictionary which maps
        each occurring k-mer to the number of times that k-mer
        occurs within the genome.
        '''
        cases = [
            ('A', 1, {'A': 1}),
            ('T', 1, {'T': 1}),
            ('C', 1, {'C': 1}),
            ('G', 1, {'G': 1}),
            ('ATCG', 1, {'A': 1, 'T': 1, 'C': 1, 'G': 1}),
            ('ATCG', 2, {'AT': 1, 'TC': 1, 'CG': 1}),
            ('ATCG', 3, {'ATC': 1, 'TCG': 1}),
            ('ATCG', 4, {'ATCG': 1}),
            ('AAATTTCCCGGG', 2, {'AA': 2, 'AT': 1, 'TT': 2, 'TC': 1, 'CC': 2, 'CG': 1, 'GG': 2})
        ]

        for genome, k, expected_kmer_counts in cases:
            kmer_counts = optimized_count_kmers(genome, k)
            failure_message = f'Failed on input genome: {genome}, k: {k}. Expected: {expected_kmer_counts} Actual: {kmer_counts}.'
            self.assertEqual(kmer_counts, expected_kmer_counts, failure_message)

    def test_clumping_kmers(self):
        '''
        NOTE: These test cases come from http://bioinformaticsalgorithms.com/data/debugdatasets/replication/ClumpFindingProblem.pdf
              and there are helpful descriptions in that document of what each case tests for.

        This function finds clusters of k-mers in a genome.

        * Accepts as input:
            * `genome` — a string representing the genome of an organism.
            * `kmer_length` - an integer representing the length of the k-mers to be considered for clumping.
            * `window_size` — an integer representing the size of the "region of interest".
            * `min_kmer_count` – an integer representing the minimum number of duplicate k-mers found within `window_size` of each other to be considered a cluster.

        * Returns as output:
            * A dictionary mapping all the k-mers that satisfy the clustering criteria to a list of all the start locations within `genome` for those clusters.
            * Each position in the list should be a start location of the k-mer satisfying the clustering criteria.
                * For example, consider the genome `'ATATGATGATAT'` the 3-mer `'ATG'` a `window_length` of 10, and a `min_kmer_count`of 2.
                * In reality, we could map this window to all the positions `[0, 1, 2]` — but all of these positions contain the SAME cluster, and the most relevant of those positions is `2` because it is the start of the first copy of our k-mer `'ATG'`.
                    * Including positions 0 and 1 in our solution would be both redundant and less useful in subsequent analysis.
        '''
        cases = [
            ('CGGACTCGACAGATGTGAAGAACGACAATGTGAAGACTCGACACGACAGAGTGAAGAGAAGAGGAAACATTGTAA', 5, 50, 4, {'CGACA': [6], 'GAAGA': [16]}),
            ('AAAACGTCGAAAAA', 2, 4, 2, {'AA': [0, 1, 9, 10, 11]}),
            ('ACGTACGT', 1, 5, 2, {'A': [0], 'C': [1], 'G': [2], 'T': [3]}),
            ('CCACGCGGTGTACGCTGCAAAAAGCCTTGCTGAATCAAATAAGGTTCCAGCACATCCTCAATGGTTTCACGTTCTTCGCCAATGGCTGCCGCCAGGTTATCCAGACCTACAGGTCCACCAAAGAACTTATCGATTACCGCCAGCAACAATTTGCGGTCCATATAATCGAAACCTTCAGCATCGACATTCAACATATCCAGCG', 3, 25, 3, {'CCA': [78, 100], 'AAA': [18, 19], 'GCC': [77], 'TTC': [65], 'CAG': [92], 'CAT': [178]}),
        ]

        for genome, k, window_length, min_count, expected_clumps in cases:
            clumps = clumping_kmers(genome, k, window_length, min_count)
            failure_message = f'Failed on input G: {genome}, k: {k}, window_length: {window_length}, min_count: {min_count}. Expected: {expected_clumps}, actual: '
            self.assertEqual(clumps, expected_clumps)

if __name__ == "__main__":
    unittest.main()


from fastapi.encoders import jsonable_encoder
from sqlalchemy.orm import Session

from app.crud.base import CRUDBase
from app.models.template import Template
from app.schemas.template import TemplateCreate, TemplateUpdate

from typing import List

from app import models

class CRUDTemplate(CRUDBase[Template, TemplateCreate, TemplateUpdate]):
    def create(  # type: ignore
        self, db: Session, *, obj_in: TemplateCreate, run_id: int
    ) -> Template:
        obj_in_data = jsonable_encoder(obj_in)
        db_obj = self.model(**obj_in_data, run_id=run_id)
        db.add(db_obj)
        db.commit()
        db.refresh(db_obj)
        return db_obj

    def get_all_by_user(
        self, db: Session, *, user_id: int
    ) -> List[Template]:
        return (
            db.query(models.Template)
            .join(models.Sample)
            .join(models.User)
            .filter(models.User.id == user_id)
            .all()
        )

    def get_all_by_user_and_run(
        self, db: Session, *, user_id: int, run_id: int
    ) -> List[Template]:
        return (
            db.query(models.Template)
            .join(models.Run)
            .join(models.Sample)
            .join(models.User)
            .filter(models.User.id == user_id)
            .filter(models.Run.id == run_id)
            .all()
        )

    def count_samples(self, db: Session, *, template_id: int) -> int:
        return (
            db.query(models.Sample.id)
            .join(models.Template)
            .filter(models.Template.id == template_id)
            .count()
        )

template = CRUDTemplate(Template)


from django.urls import path
from managepertandingan.views import *

app_name = 'managepertandingan'

urlpatterns = [
    path('notexist/', notExist, name='notExist'),
    path('listpertandingan/', listPertandingan, name='listPertandingan'),
    path('listpertandinganconf/', listPertandinganConf, name='listPertandinganConf'),
    path('listPeristiwa/<uuid:id_pertandingan>/<str:nama_tim>/', listPeristiwa, name='listPeristiwa'),

]


#!/usr/bin/env python
# -*- coding: utf-8 -*- #

import re
import json

import requests
from bs4 import BeautifulSoup

from slugify import slugify

# from src.msg import simple_msg
from .utils import get_configs, normalize_name

class FindAnime():

    def __init__(self, anime_name):
        config = get_configs()
        self.anime_name = anime_name
        host = config['host']['anbient'] + '/'
        uri = 'search?search_api_views_fulltext=' + anime_name
        url = host + uri
        # simple_msg(url, 'white', tab=True)
        self.tree = self.beautifulSoup_page(url)

    def getPage(self, url):
        request_get = requests.get(url)
        requests_response = request_get.content

        return requests_response

    def beautifulSoup_page(self, url):

        return BeautifulSoup(self.getPage(url), 'html.parser')

    def parse_search(self):
        founds = {}
        for node in self.tree.findAll('a', {"rel": 'bookmark'}):
            name = slugify(node.string)
            founds[name] = node.attrs['href']

        return founds

class PunchLib():

    def __init__(self, list_type):
        self.config = get_configs()
        self.host = self.config['host']['punchsub'] + '/'
        position = {"animes": 0, "movies": 2, "ovas": 1}
        self.full_list = self.format_full_list(position[list_type])

    def get_full_list(self, position):
        url = 'https://punchsubs.net/buscar-projeto/anime'
        request_get = requests.get(url)
        content = request_get.content
        bs = BeautifulSoup(content, 'html.parser')
        pattern = re.compile("var values = \[.*?\]\;")
        values = pattern.findall(bs.text)[0]
        value = values.split('var values = ')[1].rsplit(';', 1)
        the_list = json.loads(json.loads(json.dumps(value))[0])
        return the_list

    def format_full_list(self, position):
        full_list = self.get_full_list(position)
        new_full_list = {}
        for anime in full_list:
            new_full_list.setdefault(anime['titulo'].upper(), []).append(self._list_2_dict(anime))

        return new_full_list

    def _list_2_dict(self, value):
        name = normalize_name(value['titulo'])
        return {
            "name": name,
            # "id": value[0],
            # "quality": value[5],
            "season": "Primeira temporada",
            "totalEpisodes": value['sinopse'],
            "genres": [value['genero']],
            "link": value['link'],
            "imagem": value['imagem']
        }


import numpy as np
import matplotlib.pyplot as plt
import matplotlib
matplotlib.font_manager._rebuild()

# import seaborn as sns
# sns.set()

plt.style.use("ggplot")

with open("mode") as f:
	mode = f.read().strip()

if mode == "thesis":
	plt.rcParams['font.family'] = 'Palatino'
elif mode == "presentation":
	plt.rcParams['font.family'] = 'Fira Sans Light'
elif mode == "paper":
	plt.rcParams['font.family'] = 'CMU Serif'
else:
	import sys
	print("Invalid mode", file=sys.stderr)
	sys.exit(1)

print("  - plotmode: {}".format(mode))

plt.rcParams['axes.unicode_minus'] = False


'图顶点着色的贪婪盒覆盖算法'
import numpy as np
import random

class Greed:
    def __init__(self,B):
        self.B = B   # 网络的距离矩阵
        self.max_box = int(np.max(self.B))+1   # 覆盖网络的最大盒子直径

    def greed(self):
        n = len(self.B)
        L_boxs = list(range(1,self.max_box+1))  # 所有盒子的直径
        nodes = [i for i in range(n)]
        random.shuffle(nodes)
        random_node_id = nodes
        node_id_index = np.argsort(random_node_id)
        C = np.zeros([n,self.max_box])
        for i in range(self.max_box):
            C[:,i] = [j for j in range(1,n+1)]
        C[node_id_index[0],:] = 0    # 把节点id为0的顶点着色为颜色0
        colors = list(C[:,0]).copy()
        for L_box in L_boxs:
            for i in range(1,n):
                index1 = node_id_index[i]
                set_ = []
                for j in range(i):
                    index2 = node_id_index[j]
                    if self.B[index1,index2] >= L_box:
                        set_.append(C[index2,L_box-1])
                if set_:
                    C[index1, L_box - 1] = min(set(colors).difference(set(set_)))
                else:
                    C[index1, L_box - 1] = min(C[:,L_box - 1])
        return C

    def counter_box(self,C):
        n = C.shape[1]
        res = []
        for i in range(n):
            res.append(len(set(C[:,i])))
        return res


#!/usr/bin/env python2.7

from bottle import *

import transmissionrpc

@get('/')
def index():
	return r''''sup.

<form method="post">Magnet: <input name="magnet"> <input type="submit" value="go get it"></form> '''

@post('/')
def posted():
	magnet = request.forms.get('magnet')
	if magnet[:7] != 'magnet:': abort(400, "Whatcha talkin' bout willis?! That's no magnet link")

	# Okay, it looks a bit like a magnet link
	tc = transmissionrpc.Client()
	try:
		torrent = tc.add_torrent(magnet,peer_limit=1024)
		# Globally limited already hopefully
		torrent.download_limit = 5000 # 5MBytes/s
		torrent.upload_limit = 2000 # 2MBytes/s
	except:
		# The problem *might* be that it's already in there.
		try:
			# this will fail  if it's not realy in there
			return redirect('/'+gettorrentfrommagnet(magnet).hashString)
		except: pass
		abort(400, "That's a broken looking magnet you got there. Come back when you picked up a real one: %s" % repr(magnet))

	# Okay, hash string. Yay.
	redirect('/'+torrent.hashString)

@get('/all')
def list_all():
	tc = transmissionrpc.Client()
	torrents = tc.get_torrents()
	out = '
'
	for v in torrents:
		out += '
%s
(%s) (%s) %s\n

' % (v.hashString,v,v.status,v.progress,"(eta "+v.format_eta()+")" if v.eta > 0 else "")
	return out + '
'

@get('/favicon.ico')
def favico():
	return ""

def gettorrentfromhash(hashthingy):
	tc = transmissionrpc.Client()
	torrents = tc.get_torrents()
	for v in torrents:
		if v.hashString == hashthingy: return v

def gettorrentfrommagnet(magnet):
	'''try to match the hash of an existing torrent to the hash'''
	tc = transmissionrpc.Client()
	torrents = tc.get_torrents()
	for v in torrents:
		if len(v.hashString) > 10 and v.hashString in str(magnet):
			return v

@get('/<hashstring>/<fileid>')
def getcompletefile(hashstring,fileid):
	tc = transmissionrpc.Client()
	try:
		torrent = gettorrentfromhash(hashstring)
		files = torrent.files()
		filedetails = files[int(fileid)]
	except Exception as e:
		abort(400, "Go home torrent. You're drunk. %s" % repr(e))
	completed,size = filedetails['completed'],filedetails['size']
	if completed < size:
		return "%d%%\n\n\n%s" % ((completed *100)  / size,str(torrent))
	# Okay, everything seems legit here.
	from os import system
	# Take care of the uploading to trove
	system('s3cmd -P sync ~/Downloads s3://fetchme')
	redirect('http://fetchme.beta.anchortrove.com/Downloads/%s' % (filedetails['name']))

def show_files_in_torrent(torrent):
	hashthingy = torrent.hashString
	out = '
'
	for k,v in torrent.files().items():
		out+= '
%s' % (hashthingy,k,v['name'])
	out+= ''
	return out

@get('/<hashstring>')
def getcomplete(hashstring):
	tc = transmissionrpc.Client()
	try:
		torrent = gettorrentfromhash(hashstring)
	except Exception as e:
		abort(400, "Go home hash string. You're drunk. %s"%repr(e))

	if torrent.status == 'seeding':
		return show_files_in_torrent(torrent)

	return '
%s

Torrent is %s and %s%% done. ETA is %s.
' %  (torrent,torrent.status,torrent.progress,torrent.format_eta())

if __name__ == "__main__":
	run(host='0.0.0.0')
else:
	application = default_app()


import os
import sys
import threading
import time
import queue

def func(id , name , count):
    for i in range(count):
        print('id:',id)
        print('name:',name)

class Mythread(threading.Thread):
    def __init__(self,threadid ,threadname ,count):
        threading.Thread.__init__(self)
        self.tid   = threadid
        self.tname  =threadname
        self.count = count
    def run(self):
        func(self.tid,self.tname,self.count)

def main():
    t1 = Mythread(1,'thread :1',5)
    t2 = Mythread(2 , "thread :2",7)
    threads = []
    threads.append(t1)
    threads.append(t2)
    for t in threads:
        t.start()
    for t in threads:
        t.join()

#___测试上锁操作 ____________________

exitflg =0

class MyQueuethread(threading.Thread):
    def __init__(self,threadid ,threadname ,q):
        threading.Thread.__init__(self)
        self.tid   = threadid
        self.tname  =threadname
        self.q = q
    def run(self):
        print('Start thread:',self.tname)
        process_data(self.tname,self.q)

def process_data(threadname,q):
    while not exitflg:
        threading.Lock().acquire()
        if not q.empty():
            data =q.get()
            threading.Lock().release()
            print('thread name:',threadname)
        else:
            threading.Lock().release()
        time.sleep(1)
threadlist = ['thread 1','thread 2','thread 3']
namelist  =['one','two','three','four','five']
workQue= queue.Queue(10)
threads=[]
QLock =threading.Lock()

def Test_Q():
    for id  , tname in enumerate(threadlist):
        thread  =MyQueuethread(id,tname,workQue)
        thread.start()
        threads.append(thread)

    QLock.acquire()
    for word in namelist:
        workQue.put(word)
    QLock.release()
    for t in threads:
        t.join()
    print('exit main ')

if __name__=="__main__":
    Test_Q()


"""
Showcase of filling a voxelization.
"""
import volmdlr
from volmdlr.core import VolumeModel
from volmdlr.discrete_representation import MatrixBasedVoxelization
from volmdlr.primitives3d import Cylinder

VOXEL_SIZE = 0.005

# Create a volume model
cylinder = Cylinder(volmdlr.OXYZ, 0.1, 0.2, name="Cylinder")
volume_model = VolumeModel([cylinder])

# Voxelize the volume model (it uses the triangulated model to create the voxelization)
voxelization = MatrixBasedVoxelization.from_volume_model(volume_model, VOXEL_SIZE, name="Voxelization")

# Fill the voxelization
filled_voxelization = voxelization.fill_enclosed_voxels()  # filling the enclosed voxels of the voxelization
filled_voxelization.name = "Filled voxelization"

# Print the volumes
# The filled voxelization volume will be always over-estimated,
# but become more precise as we reduce the voxelization size

print(f"Cylinder volume: {cylinder.volume()}")
print(f"Voxelization volume: {voxelization.volume}")
print(f"Filled voxelization volume: {filled_voxelization.volume}")

# Display the result
voxelization_cs = voxelization.to_closed_triangle_shell()
voxelization_cs.color = (1, 0, 0)
voxelization_cs.alpha = 0.5

filled_voxelization_cs = filled_voxelization.to_closed_triangle_shell()
filled_voxelization_cs.color = (0, 0, 1)
filled_voxelization_cs.alpha = 0.5

volume_model = VolumeModel([voxelization_cs, filled_voxelization_cs])
volume_model.babylonjs()


# read envi file and create a h5 file
import envi
import scipy.misc
import h5py
import numpy as np

def load_data(envifile, stainfile, trainmask="", normalize = True):
    if trainmask == "":
        E = envi.envi(filename=envifile)
    else:
        mask = scipy.misc.imread(trainmask, flatten=True)
        E = envi.envi(envifile, mask=mask)
    F= E.loadall()    # return a matrix from envi file #bands * w * h
    F = np.rollaxis(F, 0, 3)

    # read the stained file
    stain = scipy.misc.imread(stainfile)
    if normalize:
        F, stain = norm(F, stain, mode='standard')

    else:

        stain = stain/255.

    return F , stain

def main():
    envifile_path = 'D:\\python-projects\\cnn-codes\\data\\DAPI\\1\\new\\'#"D:\\python-projects\\cnn-codes\\data\\davar\\a_train_c_test\\new\\a_train_b_valid_c_test\\"
    stainfile_path ='D:\\python-projects\\cnn-codes\\data\\DAPI\\1\\new\\' #"D:\\python-projects\\cnn-codes\\data\\davar\\a_train_c_test\\new\\a_train_b_valid_c_test\\"
    save_path= 'D:\\python-projects\\cnn-codes\\data\\DAPI\\1\\new\\'#'D:\\python-projects\\cnn-codes\\data\\davar\\a_train_c_test\\new\\a_train_b_valid_c_test\\'
    h5f = h5py.File(save_path + 'DAPI_kid_crop.h5', 'w')
    #train data
    envi_name = 'mosaic_base_bip_pca_mask2'#'a_mosaic_base_bip_pca60' #'ab_mosaic_base_bip_norm_pca60'
    target_name= 'target2.bmp'
    envifile = envifile_path + envi_name
    stainfile= stainfile_path + target_name
    E, stain = load_data(envifile, stainfile)
    x_train = E[np.newaxis]
    y_train = stain[np.newaxis]
    h5f.create_dataset('x_train', data=x_train)
    h5f.create_dataset('y_train', data=y_train)

    # validation data
    envi_name = 'mosaic_base_bip_pca_mask2_crop_test'
    target_name = 'cnn_re_crop.bmp'
    envifile = envifile_path + envi_name
    stainfile = stainfile_path + target_name
    E, stain = load_data(envifile, stainfile)
    h5f.create_dataset('x_valid', data=E)
    h5f.create_dataset('y_valid', data=stain)

    # #test data
    # envi_name = 'mosaic_base_bip_mask_pca_part'
    # target_name = 'm_brain_acu_neun_washed_reg1.bmp'
    # envifile = envifile_path + envi_name
    # stainfile = stainfile_path + target_name
    # E, stain = load_data(envifile, stainfile)
    # h5f.create_dataset('x_test', data=E)
    # h5f.create_dataset('y_test', data=stain)

    h5f.close()

    #read h5
    h5f = h5py.File(save_path + 'DAPI_kid_crop.h5', 'r')
    a= h5f['x_valid'][:]
    b = h5f['y_valid'][:]
    print(a.shape)
    print(b.shape)
    h5f.close()

def norm(x, y, mode):
    if mode == 'gaussian':
        pass
    elif mode == 'standard':
        x_max = np.array([np.max(x[:, :, i]) for i in range(x.shape[-1])])
        x_min = np.array([np.min(x[:, :, i]) for i in range(x.shape[-1])])
        x_norm = (x-x_min)/(x_max-x_min)
        y_norm = y/255.
        return x_norm, y_norm
if __name__ == '__main__':
    main()


import cv2
#Read pictures and zoom for easy display
img=cv2.imread("random_image.jpg")
height, width=img.shape[:2]
size=(int(width * 0.2), int(height * 0.2))
#Zoom
img=cv2.resize(img, size, interpolation=cv2.inter_area)
#bgr to hsv
hsv=cv2.cvtcolor(img, cv2.color_bgr2hsv)
#Mouse click response event
def getposhsv(event, x, y, flags, param):
    if event==cv2.event_lbuttondown:
        print("hsv is", hsv[y, x])
def getposbgr(event, x, y, flags, param):
    if event==cv2.event_lbuttondown:
        print("bgr is", img[y, x])
cv2.imshow("imagehsv", hsv)
cv2.imshow("image", img)
cv2.setmousecallback("imagehsv", getposhsv)
cv2.setmousecallback("image", getposbgr)
cv2.waitkey(0)


""" 06 - Utilizando listas faça um programa que faça 5 perguntas para uma pessoa sobre um crime. As perguntas são:
   "Telefonou para a vítima?"
   "Esteve no local do crime?"
   "Mora perto da vítima?"
   "Devia para a vítima?"
   "Já trabalhou com a vítima?"
O programa deve no final emitir uma classificação sobre a participação da pessoa no crime.
   Se a pessoa responder positivamente a 2 questões ela deve ser classificada como "Suspeita",
   Entre 3 e 4 como "Cúmplice" e 5 como "Assassino".
   Caso contrário, ele será classificado como "Inocente". """

print('RESPONDA APENAS COM "SIM" OU "NAO":')
print('-=-'*15)
p1 = input('Telefonou para a vítima?  ').lower()
p2 = input('Esteve no local do crime?  ').lower()
p3 = input('Mora perto da vítima?  ').lower()
p4 = input('Devia para a vítima?  ').lower()
p5 = input('Já trabalhou com a vítima?  ').lower()
respostas = [p1,p2,p3,p4,p5]
contar = respostas.count('sim')
print('-=-'*15)
if contar == 2:
    print('Você é um SUSPEITO!')
else:
    if contar == 3 or contar == 4:
        print('Você é um CÚMPLICE!')
    else:
        if contar == 5:
            print('Você é um ASSASSINO!')
        else:
            print('Você é INOCENTE!')
print('-=-'*15)


from HAO.链表 import LinkedList as node

def reverseKGroup(head , k):
    # write code here
    count = 0
    # cur = node.ListNode(None)
    while count < k:
        # cur.next = head
        head = head.next
        count+=1

    head.next = None
    cur = head

    return cur

l = [1,2,3,4]
head = node.listToListNode(l)
k = 2

cur = head

head = head.next
cur.next = head

head = head.next
cur.next = head

cur.next = None
# tmp = head.next.next
# head.next.next = None
# cur = tmp

tList = node.ListNodeTolist(cur)

print(tList)

# cur = node.ListNode(None)
# cur.next = head
# cur.next = head.next
# cur.next = None

# print(node.ListNodeTolist(head))
# print(node.ListNodeTolist(cur))


import pandas as pd
import json
import sys

def get_connection(index):
    connect_index = int(index) - 1
    connect = {
        "index": connect_index,
        "elecID": electrode_objects[connect_index]["elecID"],
    }
    return connect

def get_coords(x, y, z):
    coords = {
        "x": float(x),
        "y": float(y),
        "z": float(z),
    }
    return coords

def isComposite(type):
  return len(type.split(',')) > 1

subject = input("Enter Subject ID: ")

coord_file = open(sys.argv[1])
attributes_df = pd.read_excel(sys.argv[2])
functions_df = pd.read_excel(sys.argv[3])
coord_contents = coord_file.read()

seizure_types = [col for col in attributes_df.columns if col.lower().startswith('seizure')]
electrode_objects = []

for line in coord_contents.split('\n'):
  try:
    coordinate_tuple = tuple(line.rstrip().split(' '))
    (ID, x, y, z, elecType) = coordinate_tuple
    electrode_data = {
      "elecID": ID,
      "coordinates": get_coords(x, y, z),
      "elecType": elecType,
    }
    electrode_objects.append(electrode_data)
  except ValueError:
    continue

for index, row in attributes_df.iterrows():
  if pd.isnull(row['Interictal Population']):
    electrode_objects[index]['intPopulation'] = 0
  else:
    int_population = str(int(row['Interictal Population']))
    electrode_objects[index]['intPopulation'] = int_population if not isComposite(int_population) else "Variable"

  for type in seizure_types:
    if pd.isnull(row[type]):
      electrode_objects[index][type] = ""
    else:
      electrode_objects[index][type] = row[type] if not isComposite(str(row[type])) else "Variable"

functional_maps = []
for index, row in functions_df.iterrows():
  fmap_dict = {}
  for fmap in row.keys():
    if pd.isnull(row[fmap]):
      fmap_dict[f'fmap{fmap}'] = ''
    else:
      if fmap == "G1" or fmap == "G2":
        connect_index = int(row[fmap] - 1)
        connected_object = electrode_objects[connect_index]['elecID']
        fmap_dict[f'fmap{fmap}'] = {
          "index": connect_index,
          "elecID": connected_object
        }
      elif fmap == "After Discharges":
        fmap_dict["fmapAfterDischarge"] = row[fmap]
      else:
        fmap_dict[f'fmap{fmap}'] = row[fmap]
  functional_maps.append(fmap_dict)

final_json = {
  "subjID": subject,
  "totalSeizType": len(seizure_types),
  "SeizDisplay": seizure_types + ["intPopulation", "funMapping"],
  "electrodes": electrode_objects,
  "functionalMaps": functional_maps,
}

with open(f'sub-{subject}_ntoolsbrowser.json', 'w') as output_file:
  json.dump(final_json, output_file, ensure_ascii=False, indent=4)


import openpyxl
from selenium import webdriver
from lxml import etree
from openpyxl import Workbook
'''
wb = Workbook()
sheet = wb.get_active_sheet()
ws = wb.active
'''

def save_to_Excel(name):
    wb = openpyxl.Workbook()
    ws =wb.active
    ws.cell(row=1, column=1).value = '书名'
    '''
    ws.cell(row=1, column=2).value = '评分'
    '''
    for i in range(25):
        result = [name[i]]
        ws.append(result)
    wb.save("douban.xlsx")

def get_web(url):
    driver = webdriver.Chrome()
    driver.get(url)

    print('waitting for .......')
    #time.sleep(10)
    print('waitting done .......')
    driver.set_page_load_timeout(5)

    driver.save_screenshot('douban_reader.png')

    fn = 'douban_reader.html'
    with open(fn, 'w', encoding='utf-8') as f:
        f.write(driver.page_source)

    content_parse(fn)
    driver.quit()

def content_parse(fn):

    name = []

    with open(fn, 'r', encoding='utf-8') as f:
        html = f.read()

    # 生成xml树
    tree = etree.HTML(html)

    #查找book
    books = tree.xpath('//div[@class="item-root"]')
    num0 = 2
    book_name = tree.xpath(".//div[@class='title']/a")
    for i in range(len(book_name)):
        name.append(book_name[i])

    save_to_Excel(name)

    for book in books:
        book_name = book.xpath(".//div[@class='title']/a")
        book_hrefs = book.xpath(".//div[@class='title']/a/@href")

        book_list.append(book_name)

        book_name = book_name[0].text

        '''
        bookname = []
        for i in book_name:
            bookname.append(i)
        for i in bookname:
            ws.cell(row=num0, column=1).value = i

            num0 = num0 + 1
            print(num0)
        '''

        print(book_name)
        print(book_hrefs[0])

        for book_href in book_hrefs:
            driver = webdriver.Chrome()
            driver.get(book_href)
            print('2nd web waitting for .......')
            driver.set_page_load_timeout(5)
            #time.sleep(10)
            print('2nd web waitting done .......')
            driver.save_screenshot('book_href.png')

            book = 'books.html'
            with open(book, 'w', encoding='utf-8') as b:
                b.write(driver.page_source)

            parse_detail(book)

            driver.quit()

def parse_detail(book):
    rating = []

    with open(book, 'r', encoding='utf-8') as b:
        bookhtml = b.read()
    tree = etree.HTML(bookhtml)

    rating_1 = tree.xpath('//div[@class="rating_wrap clearbox"]/div[@class="rating_self clearfix"]/strong')
    '''
    for i in range(len(rating_1)):
        rating.append(rating_1[i])
    '''

    if (len(rating_1)):
            '''
            rat = []

            for i in rating:
                rat.append(i)
            for i in rat:
                ws.cell(row=num1, column=2).value = i

                num1 = num1 + 1
                print(num1)
            '''
            rating_1 = rating_1[0].text

            print(rating_1)

if __name__ == '__main__':
    urls = ['https://book.douban.com/subject_search?search_text=python&cat=1001&start={}'.format(str(i)) for i in range(0, 15, 15)]
    for url in urls:
        print(url)
        get_web(url)


# Definition for singly-linked list.
class ListNode(object):
    def __init__(self, val=0, next=None):
        self.val = val
        self.next = next

# Solution

class Solution(object):
    def addTwoNumbers(self, l1, l2):
        sumList = ListNode(0)
        sumListTail = sumList
        carry = 0
        while l1 or l2 or carry:
            sum = 0
            if l1:
                sum += l1.val
                l1 = l1.next
            if l2:
                sum += l2.val
                l2 = l2.next

            carry, val = divmod(sum + carry, 10)
            sumListTail.next = ListNode(val)
            sumListTail = sumListTail.next
        return sumList.next

# Main
def main():
    l1 = ListNode(2)
    l1.next = ListNode(4)
    l1.next.next = ListNode(3)

    l2 = ListNode(5)
    l2.next = ListNode(6)
    l2.next.next = ListNode(4)

    solution = Solution()
    sumList = solution.addTwoNumbers(l1, l2)
    while sumList:
        print(sumList.val)
        sumList = sumList.next

if __name__ == '__main__':
    main()


# N과 M (3) // 중복된 수도 함께 반복하므로 이전의 if 조건문을 빼고 그대로 넣음.

def tracking():
    if len(numbers) == M:
        print(*numbers)
    else:
        for num in range(1, N+1):
            numbers.append(num)
            tracking()
            numbers.pop()

N, M = map(int, input().split())
numbers = []
tracking()


#para gravação de texto na Tag
import RPi.GPIO as GPIO
from mfrc522 import SimpleMFRC522
#desabilita os avisos
GPIO.setwarnings(False)
#cria o objeto "leitor" para a instância "SimpleMFRC522" da biblioteca
leitor = SimpleMFRC522()
#criacao da variavel que armazena o texto que será gravado na tag
texto = "Dupla Vinicius e David 1193 tag" #altere para o texto que deseja gravar
#escreve a tag assim que ela for aproximada do leitor, e informa a conclusão
print("Aproxime a tag do leitor para gravar.")
leitor.write(texto) #função que realiza a gravação do texto configurado
print("Concluído!")


# -*- coding: utf-8 -*-
"""                             TP 4
                            Boucle WHILE
         @author: Sébastien RÉAU - sebastien.reau@fenelon.lyc.fr
"""

# Le juste prix (à exécuter en console IPython)

from random import randint as tirage # à exécuter une fois pour toutes
N = tirage(1, 1000)
message = "J'ai choisi un entier compris entre 1 et 1000. Devine-le : "
user = int(input(message))
k = 1
while user != N :
    if user < N :
        message = "Plus grand... "
    else :
        message = "Plus petit... "
    k = k + 1
    user = int(input(message + "Essaie encore : "))
print("\nGagné en " + str(k) + " coup(s) !")


""" Keen mqtt relay class """

import paho.mqtt.client as mqtt
import keen
import json
from datetime import datetime
import logging

logger = logging.getLogger('keenmqtt')

class KeenMQTT:

	def __init__(self):
		self.ready = False
		self.running = False
		self.collection_mapping = {}

	def setup(self, mqtt_client=None, keen_client=None, settings=None):
		"""Setup the clients for this instance.

		Normally called with a settings object containing `keen` and `mqtt` keys
		with dictionary values of settings.

		Args:
			mqtt_client Optional[class]: An instance of an Paho MQTT client class.
			keen_client Optional[class]: An instance of a KeenClient.
			settings Optional[dict]: A settings dict, normally loaded from a config.yaml file.
		Return:
			None
		"""
		if mqtt_client:
			self.mqtt_client = mqtt_client
			self.register_subscriptions()
		else:
			self.connect_mqtt_client(settings)

		if keen_client:
			self.keen_client = keen_client
		else:
			self.connect_keen(settings)

		if 'collection_mappings' in settings:
			for subscription in settings['collection_mappings']:
				collection = settings['collection_mappings'][subscription]
				self.add_collection_mapping(subscription, collection)

		self.ready = True

	def connect_mqtt_client(self, settings):
		"""Setup MQTT client.

		Please note that the MQTT client will not actually connect until either ``step`` or ``start``
		has been called.

		Args:
			settings Optional[dict]: The settings object, such as one read from config.yaml
		Return:
			None
		"""
		mqtt_settings = settings['mqtt']
		if 'client_id' not in mqtt_settings:
			import uuid
			mqtt_settings['client_id'] = str(uuid.uuid4())

		self.mqtt_client = mqtt.Client(mqtt_settings['client_id'])
		self.mqtt_client.on_message = self.on_mqtt_message
		self.mqtt_client.on_connect = self.on_mqtt_connect
		if 'user' in mqtt_settings and len(mqtt_settings['user']):
			self.mqtt_client.username_pw_set(mqtt_settings['user'], mqtt_settings['pass'])
		self.mqtt_client.connect(mqtt_settings['host'], mqtt_settings['port'])

	def connect_keen(self, settings):
		"""Setup the Keen IO client.

		Args:
			settings Optional[dict]: The settings object, such as one read from config.yaml
		Return:
			None
		"""
		if 'keen' in settings:
			self.keen_client = keen.KeenClient(**settings['keen'])
		else:
			self.keen_client = keen

	def on_mqtt_connect(self, c, client, userdata, rc):
		"""Called when an MQTT connection is made.

		See the Paha MQTT client documentation ``on_connect`` documentation for arguments.
		"""
		logger.info("MQTT Client connected")
		self.register_subscriptions()
		self.ready = True

	def register_subscriptions(self):
		"""This should always be called since re-subscribes after any
		unexpected disconnects.
		"""
		for subscription in self.collection_mapping:
			self.mqtt_client.subscribe(subscription)

	def on_mqtt_message(self, mosq, obj, mqtt_message):
		"""Called when an MQTT message is recieved.

		See the Paha MQTT client documentation ``on_message`` documentation for arguments.
		"""
		topic = mqtt_message.topic
		payload = mqtt_message.payload
		messages = self.decode_payload(topic, payload)

		if len(messages):
			for message in messages:
				event = {}
				collection = self.process_collection(topic, message)
				if collection:
					if self.process_topic(event, topic):
						if self.process_payload(event, topic, message):
							if self.process_time(event, topic, message):
								self.push_event(collection, event)

	def start(self):
		"""Automatically loop in a background thread."""
		self.running = True
		self.mqtt_client.loop_start()

	def stop(self):
		"""Disconnect and stop. """
		self.mqtt_client.loop_stop()
		self.running = False

	def step(self):
		"""Do a single MQTT step.

		Use this if you're not running keenmqtt in a background thread with ``start``/``stop``

		"""
		if self.running:
			raise BackgroundRunningException("Cannot perform a step whilst background loop is running.")
		self.mqtt_client.loop()

	def process_topic(self, event, topic):
		"""Process an incoming MQTT message's topic string.

		If the topic contains pertinant information, such as the device ID or location,
		this method can be overriden to perform any translation. By default, a key called
		``mqtt_topic`` containing the topic string will be added to the event dictionary.

		Args:
			event (dict): The event dictionary for this mqtt message.
			topic (str): The topic string.

		Return:
			bool: A Boolean indicating if this message should continue through the pipeline. Return
			``False`` to cancel the processing of this event and stop it from being saved in keen.
		"""
		event['mqtt_topic'] = topic
		return True

	def process_collection(self, topic, message):
		"""Assign a collection to the MQTT message.

		By default will find a matching topic in the collection_mapping dictionary and return
		the associated string. Could also be based on event contents.

		Args:
			event (dict): The event dictionary for this mqtt message.
			topic (str): The topic string.

		Return:
			str: A string indicating the Keen IO collection which this event should be pushed to, or
			false if a matching event collection could not be found.
		"""
		for subscription in self.collection_mapping:
			if mqtt.topic_matches_sub(subscription, topic):
				return self.collection_mapping[subscription]
		return False

	def add_collection_mapping(self,sub,collection):
		"""Add a subcription to event collection mapping.

		This will overide existing subscriptions if present.

		Args:
			sub (str): The string subscription pattern.
			collection (str): The sting event collection.

		Return:
			None
		"""
		self.collection_mapping[sub] = collection

	def decode_payload(self, topic, payload):
		"""Decode the payload of an incoming MQTT payload.

		By default a JSON object is expected, however this method can be overriden to provide
		alternative means to extract a MQTT payload. For example, a binary format could be
		extracted here.

		Args:
			topic (str): The topic string.
			payload (str): Raw MQTT payload.

		Returns:
			An array of dictionaries containing the decoded MQTT payload.

		Raises:
			ValueError: Whent the JSON payload cannot be parsed.
		"""
		return [json.loads(payload)]

	def process_payload(self, event, topic, message):
		"""Process an incoming MQTT message's payload.

		Perform any required translations to the payload of the MQTT message, such as removing

		Args:
			event (dict): The event dictionary for this mqtt message.
			topic (str): The topic string.
			message (dict): the decoded MQTT payload

		Returns:
			bool: A Boolean indicating if this message should continue through the pipeline. Return
			``False`` to cancel the processing of this event and stop it from being saved in Keen IO.
		"""
		event.update(message)
		return True

	def process_time(self, event, topic, message):
		"""Process the timestamp which will be sent to Keen IO.

		If the MQTT message contains time information which should be used instead of the event being
		timestamped by Keen IO, set it here.

		Args:
			event (dict): The event dictionary for this mqtt message.
			topic (str): The topic string.
			message (dict): The message dictionary.

		Returns:
			bool: A Boolean indicating if this message should continue through the pipeline. Return
			``False`` to cancel the processing of this event and stop it from being saved in Keen IO.
		"""
		iso_datetime = self.get_time(topic, message)
		if iso_datetime is not None:
			event['keen'] = {
				"timestamp": iso_datetime
			}
		return True

	def get_time(self, topic, message):
		"""Get the timestamp to send to Keen IO.

		This method is used to extract the timestamp from the MQTT message if required,
		or to generate a timestamp. By default, the current time will be fetched.

		Args:
			topic (str): The topic string.
			message (dict): The message dictionary.

		Returns:
			str: A string containing ISO-8601 string.
		"""
		return datetime.now().isoformat()

	def push_event(self, collection, event):
		"""Thin wrapper around Keen IO API object.

		Args:
			collection (str): The collection string to push to
			event (dict): The complete event to push
		Returns:
			None
		"""
		assert self.ready == True
		logger.debug("Saving event to collection {collection}: '{event}'".format(collection=collection, event=event))
		self.keen_client.add_event(collection, event)

class BackgroundRunningException(Exception):
	""" Used when the user tries to run in the foreground whilst
	a background loop is already running."""
	pass


# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html

# useful for handling different item types with a single interface
from itemadapter import ItemAdapter

from scrapy.exceptions import DropItem
from scrapy.utils import log

import pymongo
import kafka

import json
import logging

# class MongoDBPipeline(object):
#     def __init__(self, mongo_server, mongo_port, mongo_db):
#         connection = pymongo.MongoClient(
#             mongo_server,
#             mongo_port,
#         )
#         self.db = connection[mongo_db]

#     @classmethod
#     def from_crawler(cls, crawler):
#         settings = crawler.settings
#         return cls(
#             mongo_server = settings.get('MONGO_SERVER'),
#             mongo_port = settings.get('MONGO_PORT'),
#             mongo_db = settings.get('MONGO_DB'),
#         )

#     def process_item(self, item, spider):
#         collection = self.db[spider.name]
#         collection.insert_one(dict(item))
#         # log.msg("News added to MongoDB",
#         #         level=log.DEBUG, spider=spider)

#         return item

class KafkaPipeline(object):
    def __init__(self, kafka_server, kafka_topic):
        self.kafka_producer = kafka.KafkaProducer(
            bootstrap_servers = kafka_server,
            value_serializer = lambda x: json.dumps(x).encode('utf-8')
        )

        self.topic = kafka_topic

    @classmethod
    def from_crawler(cls, crawler):
        settings = crawler.settings
        return cls(
            kafka_server = settings.get('KAFKA_SERVER'),
            kafka_topic = settings.get('KAFKA_TOPIC'),
        )

    def process_item(self, item, spider):
        data = dict(item)

        try:
            self.kafka_producer.send(self.topic, data).get(timeout=10)
            logging.info(f"Sent data to topic {self.topic}: {data}")
        except Exception as e:
            logging.error(f"Error while sending data to topic {self.topic}: {data}")
            logging.error(e)

        return item


n = int(input())

for _ in range(n):
    lst = list(input())
    count1 = count2 = 0
    result = 'YES'
    for parenthesis in lst:
        if parenthesis == '(':
            count1 += 1
        elif parenthesis == ')':
            count2 += 1

        if count2 > count1:
            result = 'NO'
            break
    if count1 != count2:
        result = 'NO'
    print(result)