from __future__ import unicode_literals, division, print_function, absolute_import
from builtins import range
import numpy as np
from scipy.sparse.linalg import eigsh
from scipy.spatial.distance import pdist, squareform

def tsne_sim(S, no_dims=2, earlystop=True, init="random", verbose=True, max_iter=1500):
    """
    TSNE_Sim Performs symmetric t-SNE on similarity matrix S
       mappedX = tsne_sim(S, no_dims)

     The function performs symmetric t-SNE on pairwise similarity matrix S
     to create a low-dimensional map of no_dims dimensions (default = 2).
     The matrix S is assumed to be symmetric, sum up to 1, and have zeros
     on the diagonal.
     The low-dimensional data representation is returned in mappedX.
    """
    # Initialize some variables
    n = S.shape[0]                                     # number of instances
    momentum = 0.5                                     # initial momentum
    final_momentum = 0.8                               # value to which momentum is changed
    mom_switch_iter = 250                              # iteration at which momentum is changed
    exageration = 8.                                   # by how much we lie about the P-values
    stop_lying_iter = 100                              # iteration at which lying about P-values is stopped
    epsilon = 500.                                     # initial learning rate
    min_gain = .01                                     # minimum gain for delta-bar-delta
    # initialize the solution
    if init == "kpca" and no_dims == 2:
        # kPCA initialization
        x, y = proj2d(S, use_tsne=False)
        mappedX = np.zeros((n, no_dims))
        mappedX[:, 0] = x
        mappedX[:, 1] = y
    else:
        mappedX = .0001 * np.random.randn(n, no_dims)
    y_incs = np.zeros(mappedX.shape)
    gains = np.ones(mappedX.shape)
    last_cost = np.inf
    # Make sure S-vals are set properly
    np.fill_diagonal(S, 0.)                            # set diagonal to zero
    S = 0.5 * (S + S.T)                                # symmetrize S-values
    S /= np.sum(S)                                     # make sure S-values sum to one
    S = np.maximum(S, 1e-12)
    const = np.sum(S * np.log(S))                      # constant in KL divergence
    S *= exageration                                   # lie about the S-vals to find better local minima
    # Run the iterations
    for itr in range(max_iter):
        # Compute joint probability that point i and j are neighbors
        # sum_mappedX = np.sum(np.square(mappedX), 1)
        # Student-t distribution
        # num = 1 / (1 + np.add(np.add(-2 * np.dot(mappedX, mappedX.T), sum_mappedX).T, sum_mappedX))
        num = 1 / (1 + squareform(pdist(mappedX, "sqeuclidean")))
        np.fill_diagonal(num, 0.)                       # set diagonal to zero
        Q = num / np.sum(num)                             # normalize to get probabilities
        Q = np.maximum(Q, 1e-12)
        # Compute the gradients (faster implementation)
        L = (S - Q) * num
        # free some memory
        del num
        y_grads = 4. * np.dot((np.diag(np.sum(L, 0)) - L), mappedX)
        # Update the solution (note that the y_grads are actually -y_grads)
        gains = (gains + .2) * np.invert(np.sign(y_grads) == np.sign(y_incs)) + \
            (gains * .8) * (np.sign(y_grads) == np.sign(y_incs))
        gains[gains < min_gain] = min_gain
        y_incs = momentum * y_incs - epsilon * (gains * y_grads)
        mappedX += y_incs
        mappedX -= np.tile(np.mean(mappedX, 0), (n, 1))
        # Update the momentum if necessary
        if itr == mom_switch_iter:
            momentum = final_momentum
        if itr == stop_lying_iter:
            S /= exageration
        # Print out progress
        if not (itr + 1) % 25:
            if itr < stop_lying_iter:
                cost = const - np.sum(S / exageration * np.log(Q))
            else:
                cost = const - np.sum(S * np.log(Q))
                if earlystop and itr > mom_switch_iter and cost >= last_cost - 0.000001:
                    break
                else:
                    last_cost = cost
            if verbose:
                print("Iteration %i: error is %.5f" % (itr + 1, cost))
    return mappedX

def classical_scaling(K, nev=2, evcrit="LM"):
    """
    compute 2 eigenvalues and -vectors of a similarity matrix
    Input:
        - K: a symmetric nxn similarity matrix (for dissimilarity matrices, first multiply by -1/2)
        - nev: how many eigenvalues should be computed
        - evcrit: how eigenvalues should be selected ('LM' for largest real part, 'SM' for smallest real part)
    Return:
        - mappedX: n x 2 matrix of mapped data
    """
    n, m = K.shape
    H = np.eye(n) - np.tile(1. / n, (n, n))
    B = np.dot(np.dot(H, K), H)
    D, V = eigsh((B + B.T) / 2., k=nev, which=evcrit)  # guard against spurious complex e-vals from roundoff
    return np.dot(V.real, np.diag(np.sqrt(np.abs(D.real))))

def proj2d(K, use_tsne=True, evcrit="LM", max_iter=1500, verbose=True):
    """
    wrapper function to project data to 2D
    """
    if use_tsne:
        if verbose:
            print("performing tSNE: %i datapoints" % K.shape[0])
        X = tsne_sim(K, max_iter=max_iter, verbose=verbose)
        x = X[:, 0]
        y = X[:, 1]
    else:
        if verbose:
            print("performing classical scaling: %i datapoints" % K.shape[0])
        if evcrit == "LM" or evcrit == "SM":
            X = classical_scaling(K, evcrit=evcrit)
            x = X[:, 0]
            y = X[:, 1]
        elif evcrit == "LS":
            x = classical_scaling(K, nev=1, evcrit="LM")
            y = classical_scaling(K, nev=1, evcrit="SM")
        else:
            print("ERROR: evcrit not known!")
            return None
    return x, y


from typing import List

""" Runtime: 128 ms
Memory Usage: 14.4 MB """

class Solution:
    def setZeroes(self, matrix: List[List[int]]) -> None:
        zeroLocations = []
        rSeen = set()
        cSeen = set()

        for i in range(len(matrix)):
            for j in range(len(matrix[0])):
                if matrix[i][j] == 0:
                    zeroLocations.append([i, j])

        for r, c in zeroLocations:
            if r not in rSeen:
                rSeen.add(r)
                for col in range(len(matrix[0])):
                    matrix[r][col] = 0
            if c not in cSeen:
                cSeen.add(c)
                for row in range(len(matrix)):
                    matrix[row][c] = 0

        # print(matrix)
        print('\n'.join(['\t'.join([str(cell) for cell in row]) for row in matrix]))

myClassObj = Solution()

# inpt = [[1, 1, 1], [1, 0, 1], [1, 1, 1]]
inpt = [
    [0, 1, 2, 0],
    [3, 4, 5, 2],
    [1, 3, 1, 5]
]

myClassObj.setZeroes(inpt)


#Binomial coefficients. Compose a program that builds and writes a two-dimensional ragged array a such that
#a[n][k] contains the probability that you get exactly k heads when you toss a fair coin n times. Take a command-line argument to specify the maximum value of n.
#These numbers are known as the binomial distribution: if you multiply each element in row k by 2n,
#you get the binomial coefficients (the coefficients of xk in (x+1)n) arranged in Pascal's triangle.
#To compute them, start with a[n][0] = 0.0 for all n and a[1][1] = 1.0, then compute values in successive rows, left to right, with a[n][k] = (a[n-1][k] + a[n-1][k-1])/2.0.
import stdarray
import math
import stdio

n = int(input("max val of n: "))
a = stdarray.create2D(n,n, 0)

a[1][1] = 1
for i in range(1,n):
    for j in range(1,i):
        a[i][j] = ((math.factorial(i)/math.factorial(i-j))/math.factorial(j))/(2**i)

for i in range(n):
    for j in range(n):
        stdio.write(a[i][j])
        stdio.write(" ")
    stdio.writeln()

#i dont know what they wanted to achieve with the extra information for the question.

Chapter 1 - Elements of Programming/1.4 Arrays/1-4-39.py


"""File format classes for binary patches.

Sources:
IPS
https://zerosoft.zophar.net/ips.php

UPS
http://individual.utoronto.ca/dmeunier/ups-spec.pdf
"""

from __future__ import annotations

from mrcrowbar import models as mrc
from mrcrowbar import utils

class IPSRecord( mrc.Block ):
    offset_maj = mrc.UInt8( 0x00 )
    offset_min = mrc.UInt16_BE( 0x01 )
    size = mrc.UInt16_BE( 0x03 )
    data = mrc.Bytes( 0x05, length=mrc.Ref( "size" ) )

    @property
    def offset( self ):
        return (self.offset_maj << 16) + self.offset_min

    @offset.setter
    def offset( self, value ):
        self.offset_maj = (value & 0xff0000) >> 16
        self.offset_min = value & 0x00ffff

    @property
    def repr( self ):
        return f"offset: 0x{self.offset:06x}, size: 0x{self.size:04x}"

class IPS( mrc.Block ):
    magic = mrc.Const( mrc.Bytes( 0x00, length=5 ), b"PATCH" )
    records = mrc.BlockField( IPSRecord, 0x05, stream=True, stream_end=b"EOF" )

    @property
    def repr( self ):
        return f"records: {len( self.records )}"

    def create( self, source, target ):
        pass

    def patch( self, source ):
        return source

class UIntVLV( mrc.Field ):
    def __init__( self, offset, default=0, **kwargs ):
        super().__init__( default=default, **kwargs )
        self.offset = offset

    def get_from_buffer( self, buffer, parent=None ):
        assert utils.is_bytes( buffer )
        offset = mrc.property_get( self.offset, parent )
        pointer = offset
        total = 0
        shift = 0
        while pointer < len( buffer ):
            test = buffer[pointer]
            pointer += 1
            total += (test & 0x7f) << shift
            shift += 7
            if test & 0x80:
                break
            total += 1 << shift
        return total

    def update_buffer_with_value( self, value, buffer, parent=None ):
        super().update_buffer_with_value( value, buffer, parent )
        offset = mrc.property_get( self.offset, parent )
        length = self.get_size( value, parent )
        remainder = value
        if len( buffer ) < offset + length:
            buffer.extend( b"\x00" * (offset + length - len( buffer )) )
        for i in range( length ):
            buffer[offset + i] = remainder & 0x7f
            remainder >>= 7
            if remainder == 0:
                buffer[offset + i] |= 0x80
                break
            remainder -= 1
        return

    def get_start_offset( self, value, parent=None, index=None ):
        assert index is None
        offset = mrc.property_get( self.offset, parent )
        return offset

    def get_size( self, value, parent=None, index=None ):
        assert index is None
        test = value
        count = 1
        test >>= 7
        while test > 0:
            count += 1
            test >>= 7
            if test == 0:
                break
            test -= 1
        return count

    def validate( self, value, parent=None ):
        if type( value ) != int:
            raise mrc.FieldValidationError(
                f"Expecting type {self.format_type}, not {type( value[i] )}"
            )
        if value < 0:
            raise mrc.FieldValidationError( "Value must be unsigned" )
        return

class XORData( mrc.Bytes ):
    def __init__( self, offset, *args, **kwargs ):
        super().__init__( offset, stream_end=b"\x00", *args, **kwargs )

    def validate( self, value, parent=None ):
        super().validate( value, parent )
        if value.find( b"\x00" ) != -1:
            raise mrc.FieldValidationError( "XOR data can't contain a null character" )
        return

class UPSBlock( mrc.Block ):
    rel_offset = UIntVLV( 0x00 )
    xor_data = XORData( mrc.EndOffset( "rel_offset" ) )

    @property
    def repr( self ):
        return f"rel_offset: 0x{self.rel_offset:x}, size: {len( self.xor_data )}"

class UPS( mrc.Block ):
    STOP_CHECK = lambda buffer, pointer: pointer >= len( buffer ) - 12

    magic = mrc.Const( mrc.Bytes( 0x00, length=4 ), b"UPS1" )
    input_size = UIntVLV( 0x04 )
    output_size = UIntVLV( mrc.EndOffset( "input_size" ) )
    blocks = mrc.BlockField(
        UPSBlock, mrc.EndOffset( "output_size" ), stream=True, stop_check=STOP_CHECK
    )
    input_crc32 = mrc.UInt32_LE( mrc.EndOffset( "blocks" ) )
    output_crc32 = mrc.UInt32_LE( mrc.EndOffset( "input_crc32" ) )
    patch_crc32 = mrc.UInt32_LE( mrc.EndOffset( "output_crc32" ) )


# -------------------------------------------------- #
# Title: Assignment07
# Description: This script demonstrates how Pickling and Structured error
#              handling work via a shopping cart price calculator program.
# ChangeLog (Who,When,What):
# Cli,11-26-2021,Created Script
# -------------------------------------------------- #

import pickle

# Data ---------------------------------------------------------------------- #
# Declare variables and constants
item_str = ''  # Captures the user's item data
cost_flt = 0  # Captures the user's item's cost
table_lst = []  # A list that acts as a 'table' of rows
choice_str = ''  # captures the user's option selection
file_name_str = 'ShoppingCart.dat'  # The name of the data file

# Processing ---------------------------------------------------------------- #
class Processor:
    @staticmethod
    def add_data_to_list(item, cost, list_of_rows):
        """ Creates a new row with inputted item and its cost and adds it to a list of dictionary rows

        :param item: (string) with item to input into list/table of data:
        :param cost: (string) with cost of the item:
        :param list_of_rows: (list) of dictionary rows:
        :return: (list) of dictionary rows with newly added row of item & cost
        """
        if item != '' and cost != 0:
            row = {'Item': item, 'Cost': cost}
            list_of_rows.append(row)
        return list_of_rows

    @staticmethod
    def total_items(list_of_rows):
        """ Calculates the total number of items in the list/table of data

        :param list_of_rows: (list) containing table of data
        :return: (float) with the number of items in the table
        """
        num_of_items = len(list_of_rows)
        return num_of_items

    @staticmethod
    def total_cost(list_of_rows):
        """ Calculates the total cost of the items in the table of data

        :param list_of_rows: (list) of dictionary rows
        :return: (float) with total cost of shopping list
        """
        total_cost = 0
        # print(list_of_rows)  # testing table of dictionary rows
        for row in list_of_rows:
            total_cost += row['Cost']
        return total_cost

    @staticmethod
    def pickling_list(file_name, list_of_rows):
        """ Pickles the list of data into a new binary file

        :param file_name: (string) with name of the binary file
        :param list_of_rows: (list) with list of data
        :return: nothing
        """
        with open(file_name, 'wb') as file:
            pickle.dump(list_of_rows, file)

    @staticmethod
    def unpickling_list(file_name):
        """ Unpickles the list of data from a binary file

        :param file_name: (string) with name of the binary file
        :return: list of data
        """
        with open(file_name, 'rb') as file:
            shopping_list = pickle.load(file)
        return shopping_list

# Presentation -------------------------------------------------------------- #
class IO:
    @staticmethod
    def print_menu_options():
        """  Display a menu of choices to the user

        :return: nothing
        """
        print('''
          Menu of Options
          0) Exit Program
          1) Add a new item and its cost
          2) See total cost & save receipt
          ''')
        print()  # Add an extra line for looks

    @staticmethod
    def input_menu_choice():
        """ Gets the menu choice from a user

        :return: (string) with user's menu selection
        """
        choice = str(input("Which option would you like to perform? [0 to 2] - ")).strip()
        print()  # Add an extra line for looks
        return choice

    @staticmethod
    def input_press_to_continue(optional_message=''):
        """ Pause program and show a message before continuing

        :param optional_message:  An optional message you want to display
        :return: nothing
        """
        print()
        print(optional_message)
        input('Press the [Enter] key to continue.')

    @staticmethod
    def input_shopping_cart_items(item='', cost=0):
        """ User inputs an item and its cost into a list/table of data. Error handling controls the
        type of inputs allowed

        :param item: (string) with the name of the item
        :param cost: (float) with the cost of the item
        :return: (tuple) with item and its cost
        """
        try:
            item = str(input('\nEnter the name of the item in your shopping cart: ')).lower().strip()
            if item.isnumeric():
                raise Exception('Items must be in letters')
            elif len(item) == 0:
                raise Exception('Blank entries are not allowed.')
            cost = float(input('Enter its cost: '))
        except ValueError as e:
            print("Entry error")
            print('Cost must be in numbers')
            # print("Built-In Python error info: ")
            # print(e, e.__doc__, type(e), sep='\n')  # for testing: evaluating errors
        except Exception as e:
            print("Entry error")
            print(e)
            # print("Built-In Python error info: ")
            # print(e, e.__doc__, type(e), sep='\n')  # for testing: evaluating errors
        return item, cost

    @staticmethod
    def print_current_items_in_list(list_of_rows):
        """ Shows the current items in the list of dictionary rows

        :param list_of_rows: (list) of rows you want to display
        :return: nothing
        """
        print("******* Receipt: *******")
        for row in list_of_rows:
            print(row["Item"] + " (" + str(row["Cost"]) + ")")
        print("*************************")
        print()  # Add an extra line for looks

    @staticmethod
    def input_yes_no_choice(message):
        """ Gets a yes or no choice from the user

        :return: string
        """
        print()  # Add an extra line for looks
        return str(input(message)).strip().lower()

# Main Body of Script  ------------------------------------------------------ #
print('This is the Shopping Cart Price Calculator Program')

while True:
    IO.print_current_items_in_list(list_of_rows=table_lst)  # Show current data
    IO.print_menu_options()  # Display a menu of choices to the user
    choice_str = IO.input_menu_choice()  # Get menu option

    if choice_str == '1':  # User input data into list/table & program demos error handling
        item_str, cost_flt = IO.input_shopping_cart_items()
        Processor.add_data_to_list(item=item_str, cost=cost_flt, list_of_rows=table_lst)
        choice_str = IO.input_yes_no_choice('Do you want to enter more entries? [y/n] ')
        while choice_str.lower() == 'y':  # when yes is selected, continuous inputs are allowed
            item_str, cost_flt = IO.input_shopping_cart_items()
            Processor.add_data_to_list(item=item_str, cost=cost_flt, list_of_rows=table_lst)
            choice_str = IO.input_yes_no_choice('Do you want to enter more entries? [y/n] ')
        else:
            continue

    elif choice_str == '2':
        print('Total number of items in the cart is:', Processor.total_items(list_of_rows=table_lst))
        print('Total cost in the cart:', Processor.total_cost(list_of_rows=table_lst))
        # Demo of pickling data
        IO.input_press_to_continue('Saving receipt...')
        Processor.pickling_list(file_name=file_name_str, list_of_rows=table_lst)
        IO.input_press_to_continue('Receipt saved!')
        # Demo of unpickling data
        IO.input_press_to_continue('Retrieving receipt...')
        table_lst = Processor.unpickling_list(file_name=file_name_str)

    elif choice_str == '0':  # Exits Program
        break

    else:
        IO.input_press_to_continue('Please select only 0, 1 or 2!')


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Nov 23 05:10:10 2020

@author: coyote

This is to test overlapping nodes in TntaAt with the other databases
"""

import logging
from neo4j import GraphDatabase
from neo4j.exceptions import ServiceUnavailable
import create_mysql_db
from create_mysql_db import *
import pandas as pd
import numpy as np
import requests
import re
import json

############################################################################
"""
This script also loads data into the MySQL data warehouse
based on the issue that there was not gene name data for
the MEROPS substrates for the 'name' or label of the nodes
in Neo4j
"""
############################################################################

userMySQL='dummy'
passwordMySQL='password'
hostMySQL='127.0.0.1'
dbMySQL = 'protTest'

cnx = connect(userMySQL, passwordMySQL, hostMySQL)
cursor = cnx.cursor(dictionary=True)
use_database(cnx, cursor, dbMySQL)

## testing data and queries for loading into neo4j
from queries import query_merops
cursor.execute(query_merops)
merops_sel = cursor.fetchall()
merops = pd.DataFrame([[i['interactorA'], i['interactorB'],
                        i["interactionID"], i['uniProtID'],
                        i['cleavageID']] for i in merops_sel])
merops
merops.columns = ['interactorA','interactorB',"interactionID", 'uniProtID', 'cleavageID']
merops_IDs = np.unique(merops['cleavageID'])

from queries import query_psp
cursor.execute(query_psp)
psp_sel = cursor.fetchall()
psp = pd.DataFrame([[i['interactorA'], i['interactorB'],
                     i["interactionID"], i['uniProtID']] for i in psp_sel])
psp
psp.columns = ['interactorA','interactorB',"interactionID", 'uniProtID']

from queries import query_ppase
cursor.execute(query_ppase)
ppase_sel = cursor.fetchall()
ppase = pd.DataFrame([[i['interactorA'], i['interactorB'],
                       i["interactionID"], i['uniProtID']] for i in ppase_sel])
ppase
ppase.columns = ['interactorA','interactorB',"interactionID", 'uniProtID']

np.unique([merops['interactorA'], merops['interactorB']])
np.unique([psp['interactorA'], psp['interactorB']])
np.unique(psp['interactorB'])
np.unique([ppase['interactorA'], ppase['interactorB']])
np.unique(ppase['interactorB'])

get_uniprot = np.unique((merops['interactorA'].tolist() +
                         merops['interactorB'].tolist() +
                         psp['interactorA'].tolist() +
                         psp['interactorB'].tolist() +
                         ppase['interactorA'].tolist() +
                         ppase['interactorB'].tolist() +
                         merops['uniProtID'].tolist() +
                         psp['uniProtID'].tolist() +
                         ppase['uniProtID'].tolist()))

get_intact = np.unique((merops['interactionID'].tolist() +
                         psp['interactionID'].tolist() +
                         ppase['interactionID'].tolist()))

len(get_uniprot)
len(get_intact)

###############################################################################
## Not getting good data for gene names for merops substrates
## Need to get merops substrate genes names from UniProt
## Merops only had a description column which was hard to
## get gene names for nodes. Do this with batch request

query_merops3 = """
SELECT DISTINCT s.Uniprot AS substrateUniprot
FROM protTest.Substrate_search s
INNER JOIN
	protTest.proteinsUniprot p
    ON p.meropsID = s.`code` AND s.organism LIKE CONCAT('%%', p.organism, '%%')
WHERE s.Uniprot IS NOT NULL
"""
## non dict cursor
cursor = cnx.cursor()
cursor.execute(query_merops3)
merops_subs = cursor.fetchall()
arr = [i[0] for i in merops_subs]
len(merops_subs)
merops_subs2 = [i[0][0:6] for i in merops_subs]
up_tbl_cols = ["id", "reviewed", "entry name", "genes(PREFERRED)",
               "genes(ALTERNATIVE)", "organism-id", "organism"]
up_tbl_cols = ["accession", "reviewed", "id", "gene_primary",
               "gene_synonym", "organism_id", "organism_name"]

up_tbl_cols = ["accession", "id", "reviewed", "protein_name", "gene_primary",
               "gene_synonym", "gene_names" , "cc_interaction", "organism_id", "organism_name",
               "protein_families", "length", "xref_merops"]
up_tbl_cols = ["id", "entry name", "reviewed", "protein names", "genes(PREFERRED)",
               "genes(ALTERNATIVE)", "genes" , "interactor", "organism-id", "organism",
               "families", "length", "database(MEROPS)"]

# Uniprot batch request docs!
#https://www.biostars.org/p/304247/
#https://www.ebi.ac.uk/training/online/sites/ebi.ac.uk.training.online/files/UniProt_programmatically_py3.pdf
BASE = "http://www.uniprot.org"
TOOL_ENDPOINT = "/uploadlists/"

from requests.adapters import HTTPAdapter, Retry
from urllib.parse import urlparse, parse_qs, urlencode
import zlib
POLLING_INTERVAL = 3

API_URL = "https://rest.uniprot.org"

retries = Retry(total=5, backoff_factor=0.25, status_forcelist=[500, 502, 503, 504])
session = requests.Session()
session.mount("https://", HTTPAdapter(max_retries=retries))

def check_response(response):
    try:
        response.raise_for_status()
    except requests.HTTPError:
        print(response.json())
        raise

def submit_id_mapping(from_db, to_db, ids):
    request = requests.post(
        f"{API_URL}/idmapping/run",
        data={"from": from_db, "to": to_db, "ids": ",".join(ids)},
    )
    check_response(request)
    return request.json()["jobId"]

def get_next_link(headers):
    re_next_link = re.compile(r'<(.+)>; rel="next"')
    if "Link" in headers:
        match = re_next_link.match(headers["Link"])
        if match:
            return match.group(1)

def check_id_mapping_results_ready(job_id):
    while True:
        request = session.get(f"{API_URL}/idmapping/status/{job_id}")
        check_response(request)
        j = request.json()
        if "jobStatus" in j:
            if j["jobStatus"] == "RUNNING":
                print(f"Retrying in {POLLING_INTERVAL}s")
                time.sleep(POLLING_INTERVAL)
            else:
                raise Exception(j["jobStatus"])
        else:
            return bool(j["results"] or j["failedIds"])

def get_batch(batch_response, file_format, compressed):
    batch_url = get_next_link(batch_response.headers)
    while batch_url:
        batch_response = session.get(batch_url)
        batch_response.raise_for_status()
        yield decode_results(batch_response, file_format, compressed)
        batch_url = get_next_link(batch_response.headers)

def combine_batches(all_results, batch_results, file_format="tsv"):
    if file_format == "json":
        for key in ("results", "failedIds"):
            if key in batch_results and batch_results[key]:
                all_results[key] += batch_results[key]
    elif file_format == "tsv":
        return all_results + batch_results[1:]
    else:
        return all_results + batch_results
    return all_results

def get_id_mapping_results_link(job_id):
    url = f"{API_URL}/idmapping/details/{job_id}"
    request = session.get(url)
    check_response(request)
    return request.json()["redirectURL"]

def decode_results(response, file_format, compressed):
    if compressed:
        decompressed = zlib.decompress(response.content, 16 + zlib.MAX_WBITS)
        if file_format == "json":
            j = json.loads(decompressed.decode("utf-8"))
            return j
        elif file_format == "tsv":
            return [line for line in decompressed.decode("utf-8").split("\n") if line]
        elif file_format == "xlsx":
            return [decompressed]
        elif file_format == "xml":
            return [decompressed.decode("utf-8")]
        else:
            return decompressed.decode("utf-8")
    elif file_format == "json":
        return response.json()
    elif file_format == "tsv":
        return [line for line in response.text.split("\n") if line]
    elif file_format == "xlsx":
        return [response.content]
    elif file_format == "xml":
        return [response.text]
    return response.text

def get_xml_namespace(element):
    m = re.match(r"\{(.*)\}", element.tag)
    return m.groups()[0] if m else ""

def merge_xml_results(xml_results):
    merged_root = ElementTree.fromstring(xml_results[0])
    for result in xml_results[1:]:
        root = ElementTree.fromstring(result)
        for child in root.findall("{http://uniprot.org/uniprot}entry"):
            merged_root.insert(-1, child)
    ElementTree.register_namespace("", get_xml_namespace(merged_root[0]))
    return ElementTree.tostring(merged_root, encoding="utf-8", xml_declaration=True)

def print_progress_batches(batch_index, size, total):
    n_fetched = min((batch_index + 1) * size, total)
    print(f"Fetched: {n_fetched} / {total}")

def get_id_mapping_results_search(url):
    parsed = urlparse(url)
    query = parse_qs(parsed.query)
    file_format = query["format"][0] if "format" in query else "json"
    if "size" in query:
        size = int(query["size"][0])
    else:
        size = 500
        query["size"] = size
    compressed = (
        query["compressed"][0].lower() == "true" if "compressed" in query else False
    )
    parsed = parsed._replace(query=urlencode(query, doseq=True))
    url = parsed.geturl()
    request = session.get(url)
    check_response(request)
    results = decode_results(request, file_format, compressed)
    total = int(request.headers["x-total-results"])
    print_progress_batches(0, size, total)
    for i, batch in enumerate(get_batch(request, file_format, compressed), 1):
        results = combine_batches(results, batch, file_format)
        print_progress_batches(i, size, total)
    if file_format == "xml":
        return merge_xml_results(results)
    return results

def get_id_mapping_results_stream(url):
    if "/stream/" not in url:
        url = url.replace("/results/", "/results/stream/")
    request = session.get(url)
    check_response(request)
    parsed = urlparse(url)
    query = parse_qs(parsed.query)
    file_format = query["format"][0] if "format" in query else "json"
    compressed = (
        query["compressed"][0].lower() == "true" if "compressed" in query else False
    )
    return decode_results(request, file_format, compressed)

job_id = submit_id_mapping(
    from_db="UniProtKB_AC-ID", to_db="ChEMBL", ids=["P05067", "P12345"]
)
if check_id_mapping_results_ready(job_id):
    link = get_id_mapping_results_link(job_id)
    results = get_id_mapping_results_search(link)
    # Equivalently using the stream endpoint which is more demanding
    # on the API and so is less stable:
    # results = get_id_mapping_results_stream(link)
    results = get_id_mapping_results_stream(link + '?compressed=true&fields=' + \
                                             ','.join(up_tbl_cols) + \
                                             '&format=tsv')

merops_up_sub_tbl = pd.DataFrame([x.split("\t") for x in results[1:]],
                                 columns=results[0].split("\t"))
job_id='4e69c039e10625d01d2bb20341684fb127220125'  # 6587 results
#job_id='61a46976682a51e5910ae26363a869b98e1342f1'
if check_id_mapping_results_ready(job_id):
    link = get_id_mapping_results_link(job_id)
    results = get_id_mapping_results_search(link)

def map_retrieve(ids2map, source_fmt='ACC+ID', target_fmt='ACC', output_fmt='tab'):
    if hasattr(list(ids2map), 'pop'):
        pass #ids2map = ','.join(ids2map)

    payload = {'ids': ids2map,
               'format': output_fmt,
               'columns': ','.join(up_tbl_cols),
               'from': source_fmt,
               'to': target_fmt}

    response = requests.get("https://rest.uniprot.org/idmapping/run", params=payload)
    response = requests.post("https://rest.uniprot.org/idmapping/run", params=payload)
    response.raise_for_status()

    request = requests.post(
        "https://rest.uniprot.org/idmapping/run",
        data={"from": "UniProtKB_AC-ID", "to": "UniProtKB", "ids": merops_subs2,
              "fields": ','.join(up_tbl_cols)}
    )
    request.raise_for_status()
    request.text

    if response.ok:
        protein_df = pd.read_csv(StringIO(response.text), sep="\t")
        ## pandas is good but need to pass None to MySQL
        df_final = protein_df.replace({np.nan: None})
        df_final2 = df_final.iloc[:, :-1]
        df_final2.columns
        return df_final2
    else:
        print(response.raise_for_status())

chunks = np.array_split(merops_subs2, 20)
len(chunks)
ids2map = chunks[0]
one = map_retrieve(chunks[0])
two
three
frames = [map_retrieve(i) for i in chunks]

merops_up_sub_tbl = pd.concat(frames)
reviewed = merops_up_sub_tbl[merops_up_sub_tbl['Reviewed'] == 'reviewed']

## add new table to database for merops substrate gene names
TABLES = {}
TABLES['meropsSubsUniprot'] = """
CREATE TABLE meropsSubsUniprot (
    uniProtID CHAR(15) NOT NULL PRIMARY KEY,
    entryName VARCHAR(45) NOT NULL,
    geneNamePreferred VARCHAR(45),
    geneNamesAlternative JSON DEFAULT NULL,
    taxid INT,
    organism VARCHAR(100)
    ) ENGINE=InnoDB
"""
for table_name in TABLES:
    table_description = TABLES[table_name]
    try:
        print("Creating table: {}".format(table_name))
        cursor.execute(table_description)
    except mysql.connector.Error as err:
        if err.errno == errorcode.ER_TABLE_EXISTS_ERROR:
            print("Table already exists")
        else:
            print(err.msg)

log_file = "uniprot_merops_subs_not_entered.log"
open(log_file, 'w').close()

for idx, row in reviewed.iterrows():
#for idx, row in row.iterrows():
    print("*****************************",  row['Entry'])
    uniProtID = row['Entry']
    entryName = row['Entry Name']

    if row['Gene Names (primary)']:
        geneNamePreferred = row['Gene Names (primary)'].split(";")[0]
    else:
        geneNamePreferred = None

    ## can be None
    if row['Gene Names (synonym)']==None:
        geneNamesAlternative=None
    else:
        geneNamesAlternative = row['Gene Names (synonym)'].split(' ')

    organismID = row['Organism (ID)']
    organism = row['Organism'].split(" (")[0]
    geneNamesAlternative_ = None if geneNamesAlternative==None else json.dumps(geneNamesAlternative)

    try:
        cursor.execute(
            "INSERT IGNORE INTO meropsSubsUniprot VALUES (%s,%s,%s,%s,%s,%s)",
            (uniProtID, entryName, geneNamePreferred, geneNamesAlternative_,
             organismID, organism))
        cnx.commit()
    except mysql.connector.Error as err:
        print("Something went wrong: {}".format(err))
        with open(log_file, 'a') as f:
            f.write(f"{row['Entry']}\n")

###############################################################################

# Test filling neo4j

# mostly need this host
uri = 'neo4j://localhost:7687'
user = 'neo4j'
# you'll need a password after you start
password = '**'
db = 'protTest'

from fill_neo4j import ProteinExample
prot_db = ProteinExample(uri, user, password)
#prot_db.create_db(db)
#prot_db._drop(db)
prot_db.use_database(db)
prot_db.database

# query MySQL database to get node and edge infromation
# Need to remove comments in queries.py if testing again
from queries import query_up, query_intact, query_merops2, query_psp2, query_ppase2

#https://stackoverflow.com/questions/589284/imploding-a-list-for-use-in-a-python-mysqldb-in-clause
format_strings = ','.join(['%s'] * len(get_uniprot))
cursor.execute(query_up % format_strings, tuple(get_uniprot))
uniprot_data = cursor.fetchall()
len(uniprot_data)

# get query for fill intact neo4j
format_strings_intact = ','.join(['%s'] * len(get_intact))
cursor.execute(query_intact % format_strings_intact, tuple(get_intact))
intact_data = cursor.fetchall()
len(intact_data)

# get query for fill merops neo4j
format_strings_merops = ','.join(['%s'] * len(merops_IDs))
cursor.execute(query_merops2 % format_strings_merops, tuple(merops_IDs))
merops_data = cursor.fetchall()
len(merops_data)

# get query for fill psp neo4j
format_strings_psp = ','.join(['%s'] * len(np.unique(psp['interactorB'])))
cursor.execute(query_psp2 % format_strings_psp, tuple(np.unique(psp['interactorB'])))
psp_data = cursor.fetchall()
len(psp_data)

# get query for fill ppase neo4j
format_strings_ppase = ','.join(['%s'] * len(np.unique(ppase['interactorB'])))
cursor.execute(query_ppase2 % format_strings_ppase, tuple(np.unique(ppase['interactorB'])))
ppase_data = cursor.fetchall()
len(ppase_data)

"""
Tests
prot_db.create_proteins(uniprot_data)
prot_db.create_intact_interactions(intact_data)
prot_db.create_merops_interactions(merops_data)
prot_db.create_psp_interactions(psp_data)
prot_db.create_depod_interactions(ppase_data)
"""


'''
Suppose you have a random list of people standing in a queue. Each person is described by a pair of integers (h, k), where h is the height of the person and k is the number of people in front of this person who have a height greater than or equal to h. Write an algorithm to reconstruct the queue.

Note:
The number of people is less than 1,100.
'''

class Solution(object):
    def reconstructQueue(self, people):
        """
        :type people: List[List[int]]
        :rtype: List[List[int]]
        """

        # Method 1: Sorted directly.
#         queue = sorted(people, key = lambda x: (-x[0], x[1]))

#         res = []
#         for p in queue:
#             res.insert(p[1], p)
#         return res

        # Method 2: heap
        queue_heap = []
        for p in people:
            heapq.heappush(queue_heap, (-p[0], p[1]))

        res = []
        while len(queue_heap) > 0:
            height, pos = heapq.heappop(queue_heap)

            if pos >= len(res):
                res.append([-height, pos])
            else:
                res.insert(pos, [-height, pos])

        return res

Python/LeetCode/406-Queue Reconstruction by Height.py


from django.shortcuts import render, redirect
from django.utils.decorators import method_decorator
from django.contrib.auth.models import User, Group, Permission
from django.views.decorators.csrf import csrf_protect
from django.contrib import messages
from django.db.models import Q
from rest_framework import fields
from rest_framework.serializers import ModelSerializer, PrimaryKeyRelatedField
from rest_framework.viewsets import ModelViewSet

from QuanLyNhaSach.views.base import BaseITSAdminView

class ProfileView(BaseITSAdminView):
    """
    Profile View (Information my account)
    """

    def __init__(self):
        super(ProfileView, self).__init__()
        self.template_name = 'pages/profile.html'

class UserListView(BaseITSAdminView):
    """
    List users view
    """

    def __init__(self):
        super(UserListView, self).__init__()
        self.template_name = 'pages/user_list.html'

class UserEditView(BaseITSAdminView):
    """
    Update users view
    """

    def __init__(self):
        super(UserEditView, self).__init__()
        self.template_name = 'pages/user_edit.html'

    @method_decorator(csrf_protect)
    def get(self, request, user_id, **params):
        user = User.objects.get(pk=user_id)
        selected_permissions = Permission.objects.filter(Q(user=user), Q(content_type__app_label='QuanLyNhaSach') | Q(
            content_type__app_label='auth'))
        unselected_permissions = Permission.objects.exclude(user=user).filter(
            Q(content_type__app_label='QuanLyNhaSach') | Q(
                content_type__app_label='auth'))
        selected_groups = Group.objects.filter(user=user)
        unselected_groups = Group.objects.exclude(user=user)
        self.extra.update({'selected_permissions': selected_permissions,
                           'unselected_permissions': unselected_permissions,
                           'selected_groups': selected_groups,
                           'unselected_groups': unselected_groups,
                           'user_selected': user,
                           }
                          )
        params.update(self.extra)
        return render(request, self.template_name, params)

class UserAddView(BaseITSAdminView):
    """
    Add user view
    """

    def __init__(self):
        super(UserAddView, self).__init__()
        self.template_name = 'pages/user_add.html'

    @method_decorator(csrf_protect)
    def post(self, request, **kwargs):
        email = request.POST['email']
        first_name = request.POST['first_name']
        last_name = request.POST['last_name']
        user_exist = User.objects.filter(email=email, username=email).count()
        if user_exist:
            messages.warning(request, 'Your email was exist please use another')
            return redirect('QuanLyNhaSach:users_add')

        user = User(username=email, email=email, first_name=first_name,
                    last_name=last_name)
        user.save()
        user.username = user.id
        user.save()
        unselected_permissions = Permission.objects.filter(Q(content_type__app_label='QuanLyNhaSach') | Q(
            content_type__app_label='auth'))
        unselected_groups = Group.objects.all()
        self.extra.update({'unselected_permissions': unselected_permissions,
                           'unselected_groups': unselected_groups,
                           'user_selected': user,
                           }
                          )

        return render(request, 'pages/user_edit.html', self.extra)

class ErrorView(BaseITSAdminView):
    def __init__(self):
        super(ErrorView, self).__init__()
        self.template_name = 'pages/error.html'

"""
API users
include: Serializer class, Permission class, ModelViewSet class
"""

class UserSerializer(ModelSerializer):
    id = fields.CharField(required=False)
    email = fields.EmailField(required=False)
    username = fields.CharField(required=False)
    first_name = fields.CharField(required=False, allow_blank=True, allow_null=True)
    last_name = fields.CharField(required=False, allow_blank=True, allow_null=True)
    last_login = fields.DateTimeField(required=False)
    date_joined = fields.DateTimeField(required=False)
    is_active = fields.BooleanField(required=False, default=True)
    is_staff = fields.BooleanField(required=False, default=True)
    is_superuser = fields.BooleanField(required=False, default=True)
    groups_ids = PrimaryKeyRelatedField(required=False, many=True, read_only=False, queryset=Group.objects.all(),
                                        source='groups')
    permissions_ids = PrimaryKeyRelatedField(required=False, many=True, read_only=False,
                                             queryset=Permission.objects.filter(
                                                 Q(content_type__app_label='QuanLyNhaSach') | Q(
                                                     content_type__app_label='auth')),
                                             source='user_permissions')

    class Meta:
        model = User
        fields = (
            'id', 'username', 'email', 'first_name', 'last_name', 'last_login', 'date_joined', 'is_active',
            'groups_ids', 'permissions_ids', 'is_staff', 'is_superuser')
        databases_always_serialize = (
            'id', 'username', 'email', 'first_name', 'last_name', 'last_login', 'date_joined', 'is_active',
            'groups_ids', 'permissions_ids', 'is_staff', 'is_superuser')

class UserListViewSet(ModelViewSet):
    queryset = User.objects.all()
    serializer_class = UserSerializer


from PIL import Image
import cv2
import matplotlib.pyplot as plt
import numpy as np
import sys
sys.setrecursionlimit(10000000)

def fj(imgpre,img,vxs=0,vxe=0,h=True):
    x = img.shape[0]
    y = img.shape[1]
    if h:
        fj = y // 70
        dis = 30
        s = [0 for i in range(x)]
        for i in range(x):
            for j in range(y):
                s[i] += img[i][j]
    else:
        fj = 5
        dis = 30
        s = [0 for i in range(y)]
        for i in range(x):
            for j in range(y):
                s[j] += img[i][j]
        x = y
    print(s)
    rec = []
    ch = []
    now = False
    for i in range(x):
        if s[i] > fj:
            if now == False:
                rec.append(i)
                now = True
        else:
            if now == True:
                rec.append(i)
                now = False
    if now:
        rec.append(x)
    ps = rec[0]
    pe = rec[1]
    for i in range(2,len(rec),2):
        if rec[i] - pe < dis:
            pe = rec[i+1]
        else:
            if pe-ps > 70:
                ch.append(ps)
                ch.append(pe)
            ps = rec[i]
            pe = rec[i+1]
    ch.append(ps)
    ch.append(pe)
    ret = []
    for i in range(0,len(ch),2):
        if h:
            ret.append((imgpre[ch[i]:ch[i+1]+1],img[ch[i]:ch[i+1]+1],ch[i],ch[i+1]))
        else:
            ret.append((imgpre[:,ch[i]:ch[i+1]+1],vxs,vxe,ch[i],ch[i+1]))
        #plt.imshow(ret[-1][0])
        #plt.show()
    return ret

def fz(img):
    x = img.shape[0]
    y = img.shape[1]
    ret = []
    return ret

def fj2(preimg,img):
    print(img)
    x = img.shape[0]
    y = img.shape[1]
    vis = [[0 for j in range(y)] for i in range(x)]
    ret = []
    for i in range(x):
        for j in range(y):
            if vis[i][j] == 0 and img[i,j] == 1:
                cnt = 0
                xmin = i
                xmax = i
                ymin = j
                ymax = j
                bfs = []
                bfs.append((i,j))
                vis[i][j]=1
                cnt = 1
                cc = 1
                p = 0
                while cc>0:
                    now = bfs[p]
                    cc-=1
                    p+=1
                    for z in ((-1,0),(1,0),(0,-1),(0,1)):
                        na = now[0]+z[0]
                        nb = now[1]+z[1]
                        if na>=0 and na<x and nb>=0 and nb<y and vis[na][nb]==0 and img[na,nb]==1:
                            bfs.append((na,nb))
                            xmin=min(xmin,na)
                            xmax=max(xmax,na)
                            ymin=min(ymin,nb)
                            ymax=max(ymax,nb)
                            vis[na][nb]=1
                            cc+=1
                            cnt+=1
                if cnt>5500:
                    ret.append((0,xmin,xmax,ymin,ymax))
    return ret

out = cv2.imread('5.jpg', 0)
plt.imshow(out,cmap='gray')
plt.show()
ret, th = cv2.threshold(out, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
imgpre = cv2.adaptiveThreshold(out, 1, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY_INV,51,5)
plt.imshow(imgpre,cmap='gray')
plt.show()
img = cv2.fastNlMeansDenoising(imgpre,h=90)
plt.imshow(img,cmap='gray')
plt.show()

kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (40, 40))
img = cv2.dilate(img, kernel)
plt.imshow(img)
plt.show()
zz = fj2(imgpre,img)
#ret, th = cv2.threshold(img, 0, 1, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
#hh = fj(imgpre,img)
#zz = []
#for i in hh:
#    zz += fj(i[0],i[1],i[2],i[3],False)
for i in zz:
    print(i)
    for j in range(i[1],i[2]+1):
        for k in range(20):
            out[j,i[3]+k]=0
            out[j,i[4]-k]=0
    for j in range(i[3],i[4]+1):
        for k in range(20):
            out[i[1]+k,j]=0
            out[i[2]-k,j]=0
plt.imshow(out,cmap='gray')
plt.show()


import matplotlib.pyplot as plt
from random_walk import RandomWalk

while True:
    # Make a random walk
    rw = RandomWalk(num_points=5000)
    rw.fill_walk()
    plt.figure(dpi=180, figsize=(14,6.5))
    point_numbers = list(range(rw.num_points))
    plt.scatter(rw.x_values, rw.y_values, edgecolor='none', c=point_numbers, cmap=plt.cm.rainbow, s=30)

    # Emphasis the first and last points
    plt.scatter(0, 0, c='green', edgecolor='none', s=100)
    plt.scatter(rw.x_values[-1], rw.y_values[-1], c='black', edgecolor='none', s=100)
    plt.axis('off')
    plt.savefig('random_walk.png')

    status = input("Make another walk?[y/n] ")
    if status == 'n':
        break


import sys
input = sys.stdin.readline

t = int(input().rstrip())
for i in range(1, t+1):
    arr = list(sorted(map(int, input().rstrip().split())))
    if arr[0] + arr[1] > arr[2]:
        if arr[0] == arr[1] == arr[2]: print('Case #%d: equilateral'%i)
        elif arr[0] == arr[1] or arr[1] == arr[2]: print('Case #%d: isosceles'%i)
        else: print('Case #%d: scalene'%i)
    else: print('Case #%d: invalid!'%i)


from __future__ import division
import time
import torch
import torch.nn as nn
from torch.autograd import Variable
import numpy as np
import cv2
from util import prep_image, write_results
from argparse import ArgumentParser
import os
import os.path as osp
from darknet import Darknet
import pickle as pkl
import pandas as pd
import random

def arg_parse():
    """
    Parse arguements to the detect module
    """
    parser = ArgumentParser(description='YOLO v3 Detection Module')
    parser.add_argument("--images", dest='images',
                        help="Image / Directory containing images to perform detection upon",
                        default="imgs", type=str)

    parser.add_argument("--det", dest='det',
                        help="Image / Directory to store detections to",
                        default="det", type=str)

    parser.add_argument("--bs", dest="bs", help="Batch size", default=1,
                        type=int)
    parser.add_argument("--confidence", dest="confidence",
                        help="Object Confidence to filter predictions",
                        default=0.5, type=float)
    parser.add_argument("--nms_thresh", dest="nms_thresh",
                        help="NMS Threshhold", default=0.4, type=float)
    parser.add_argument("--cfg", dest='cfgfile', help="Config file",
                        default="cfg/yolov3.cfg", type=str)
    parser.add_argument("--weights", dest='weightsfile', help="weightsfile",
                        default="yolov3.weights", type=str)
    parser.add_argument("--reso", dest='reso',
                        help="Input resolution of the network. Increase to increase accuracy. Decrease to increase speed",
                        default="416", type=int)

    return parser.parse_args()

def load_classes(namesfile):
    with open(namesfile, 'r') as f:
        return f.read().splitlines()

if __name__ == '__main__':
    args = arg_parse()

    images = args.images
    batch_size = args.bs
    confidence = args.confidence
    nms_thresh = args.nms_thresh

    cuda = torch.cuda.is_available()

    classes = load_classes('./data/coco.names')
    with open('./data/pallete', 'rb') as f:
        colors = pkl.load(f)

    # Load the neural network
    print('Loading the network...')
    model = Darknet(args.cfgfile)
    model.load_weights(args.weightsfile)
    print('Network succesfully loaded')

    in_dim = args.reso
    model.net_info['height'] = in_dim
    assert in_dim % 32 == 0, 'Argument resolution --reso must be divisible by 32'
    assert in_dim > 32, 'Argument resolution --reso must be greater than 32'

    if cuda:
        model.cuda()

    # Set the model in evaluation mode
    model.eval()

    # Load the images
    t_read_dir = time.time()
    try:
        if osp.isdir(images):
            imlist = [osp.join(osp.realpath('.'), images, img) for img in
                      os.listdir(images) if osp.isfile(osp.join(images, img))]
        else:
            imlist = [osp.join(osp.realpath('.'), images)]

    except Exception.FileNotFoundError:
        print('No file or directory with the name: {}'.format(images))
        exit()

    if not osp.exists(args.det):
        os.mkdir(args.det)

    t_load_batch = time.time()
    loaded_img = [cv2.imread(im) for im in imlist]

    # PyTorch Variables for images
    im_batches = list(map(prep_image, loaded_img,
                      [in_dim for _ in range(len(imlist))]))

    # List containing dimension of original images
    im_dim_list = [(im.shape[1], im.shape[0]) for im in loaded_img]
    im_dim_list = torch.FloatTensor(im_dim_list).repeat(1, 2)

    if cuda:
        im_dim_list = im_dim_list.cuda()

    # Create the batches
    leftover = 0
    if len(im_dim_list) % batch_size:
        leftover = 1

    if batch_size != 1:
        num_batches = len(imlist) // batch_size + leftover
        im_batches = [torch.cat((im_batches[i * batch_size:min((i + 1) * batch_size,
                       len(im_batches))])) for i in range(num_batches)]

    # Predict
    write = False
    output = []
    t_detection = time.time()
    for idx, batch in enumerate(im_batches):
        t_loop_start = time.time()

        if cuda:
            batch = batch.cuda()

        pred = model(Variable(batch, volatile=True), cuda)
        pred = write_results(pred, confidence, len(classes),
                             nms_conf=nms_thresh, cuda=cuda)
        t_loop_end = time.time()

        if type(pred) == int:
            for im_num, image in enumerate(imlist[i * batch_size:
                                                  min((i + 1) * batch_size,
                                                      len(imlist))]):
                im_id = idx * batch_size + im_num
                print('{0:20s} predicted in {1:6.3f} seconds'.format(image.split('/')[-1],
                                                                     (t_loop_end - t_loop_start) / batch_size))
                print('{0:20s} {1:s}'.format('Objects Detected:', ''))
                print("----------------------------------------------------------")
            continue

        pred[:, 0] += idx * batch_size  # Transform idx batch to idx images

        if not write:
            output = pred
            write = True
        else:
            output = torch.cat((output, pred))

        for im_num, image in enumerate(imlist[idx * batch_size:
                                              min((idx + 1) * batch_size,
                                                  len(imlist))]):
            im_id = idx * batch_size + im_num
            objs = [classes[int(x[-1])] for x in output if int(x[0]) == im_id]
            #objs = [classes[int(x[-1])] for x in pred if x[0] == im_num]
            print('{0:20s} predicted in {1:6.3f} seconds'.format(image.split('/')[-1],
                                                                 (t_loop_end - t_loop_start) / batch_size))
            print('{0:20s} {1:s}'.format('Objects Detected:', ' '.join(objs)))
            print("----------------------------------------------------------")

        if cuda:
            torch.cuda.synchronize()

    if not len(output) > 0:
        print('No detections were made')
        exit()

    # Convert the coordinates to img coord without gray space padding
    im_dim_list = torch.index_select(im_dim_list, 0, output[:, 0].long())
    scaling_factor = torch.min(in_dim / im_dim_list, 1)[0].view(-1, 1)

    output[:, [1, 3]] -= (in_dim - scaling_factor * im_dim_list[:, 0].view(-1, 1)) / 2
    output[:, [2, 4]] -= (in_dim - scaling_factor * im_dim_list[:, 1].view(-1, 1)) / 2
    output[:, 1:5] /= scaling_factor

    t_output_processing = time.time()

    # Restrict bboxes with vertices outside the image
    for idx in range(output.shape[0]):
        output[idx, [1, 3]] = torch.clamp(output[idx, [1, 3]], min=0.0,
                                        max=im_dim_list[idx, 0])
        output[idx, [2, 4]] = torch.clamp(output[idx, [2, 4]], min=0.0,
                                        max=im_dim_list[idx, 1])

    def write(x, results):
        c1 = tuple(x[1:3].int())
        c2 = tuple(x[3:5].int())
        img = results[int(x[0])]
        cls = int(x[-1])
        color = random.choice(colors)
        label = "{0}".format(classes[cls])
        cv2.rectangle(img, c1, c2,color, 1)
        t_size = cv2.getTextSize(label, cv2.FONT_HERSHEY_PLAIN, 1 , 1)[0]
        c2 = c1[0] + t_size[0] + 3, c1[1] + t_size[1] + 4
        cv2.rectangle(img, c1, c2,color, -1)
        cv2.putText(img, label, (c1[0], c1[1] + t_size[1] + 4), cv2.FONT_HERSHEY_PLAIN, 1, [225,255,255], 1);
        return img

    t_draw = time.time()
    list(map(lambda x: write(x, loaded_img), output))

    det_names = pd.Series(imlist).apply(lambda x: "{}/det_{}".format(args.det,x.split("/")[-1]))
    list(map(cv2.imwrite, det_names, loaded_img))

    t_end = time.time()

    print("SUMMARY")
    print("----------------------------------------------------------")
    print("{:25s}: {}".format("Task", "Time Taken (in seconds)"))
    print()
    print("{:25s}: {:2.3f}".format("Reading addresses", t_load_batch - t_read_dir))
    print("{:25s}: {:2.3f}".format("Loading batch", t_detection- t_load_batch))
    print("{:25s}: {:2.3f}".format("Detection (" + str(len(imlist)) +  " images)", t_output_processing - t_detection))
    print("{:25s}: {:2.3f}".format("Output Processing", t_draw - t_output_processing))
    print("{:25s}: {:2.3f}".format("Drawing Boxes", t_end - t_draw))
    print("{:25s}: {:2.3f}".format("Average time_per_img", (t_end - t_load_batch)/len(imlist)))
    print("----------------------------------------------------------")


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#     ____
#    / ___| _   _ _ __   __ _  ___ _ __ ___
#    \___ \| | | | '_ \ / _` |/ _ \ '__/ _ \
#     ___) | |_| | |_) | (_| |  __/ | | (_) |
#    |____/ \__,_| .__/ \__,_|\___|_|  \___/
#   ____       _ |_|       _   _ _       ____ _       _
#  |  _ \ ___ | |__   ___ | |_(_) | __  / ___| |_   _| |__
#  | |_) / _ \| '_ \ / _ \| __| | |/ / | |   | | | | | '_ \
#  |  _ < (_) | |_) | (_) | |_| |   <  | |___| | |_| | |_) |
#  |_| \_\___/|_.__/ \___/ \__|_|_|\_\  \____|_|\__,_|_.__/

# pyright: reportMissingImports=false

#################################################################
#																#
# 							IMPORTS 							#
#																#
#################################################################

import os, sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import smach
import time
from an_utils import log_errs, log_info, patch_frame_br

# Import publishers
from an_comm import cherries_pub, add_score, action_error
from an_sm_states.sm_displacement import set_next_destination

# Import utils and constants
from an_const import ACTIONS_SCORE, DISPLACEMENT

#################################################################
# CONSTANTS
#################################################################

ACTION_TIMEOUT = 5
ARM_TIMING_TAKE = 3
ARM_TIMING_DEPOSE = 3

NEG_ACCURATE_DISP = -30
POS_ACCURATE_DISP = +15

#################################################################
# TakeCherries - For cherries (perpendicular or against the wall)
#################################################################

class TakeCherries(smach.State) :
    def __init__(self) :
            smach.State.__init__(   self,
                                    outcomes=['fail', 'preempted','done'],
                                    input_keys=['nb_actions_done','cb_arm', 'nb_errors','cherries_loaded'],
                                    output_keys=['nb_actions_done', 'nb_errors','cherries_loaded','cb_arm'])

    def execute(self, userdata):

        #Deploying an arm to take cherries on a rack
        log_info("Ordering take cherries")
        userdata.cb_arm[0] = -1
        cherries_pub.publish(0) # Publish 0 to tell to the BN to deploy the arm in order to recover the cherries

        #Wait for the end of action

        time.sleep(3)
        userdata.nb_actions_done[0] += 1
        return 'done'

        begin_time = time.time()
        while (time.time() - begin_time < ARM_TIMING_TAKE) :
            time.sleep(0.01)
            if self.preempt_requested():
                self.service_preempt()
                return 'preempted'

            #React to the feedback
            if userdata.cb_arm[0] == 0 : #Success
                log_info("Take cherries : success")
                userdata.cherries_loaded[0] = 1
                userdata.nb_actions_done[0] += 1
                return 'done'
            elif userdata.cb_arm[0] == 1 : #Error durring the recovery
                log_info("Take cherries : error encountered")
                userdata.nb_errors[0] += 1
                return 'fail'

        if time.time() - begin_time >= ARM_TIMING_TAKE :
            log_info('Timeout reached')
            return action_error("fail")

#####################################################################################
# DepositBucket - Put the cherries in the bucket (DepositCherries was already taken)
#####################################################################################

class DepositBucket(smach.State) :
    def __init__(self) :
        smach.State.__init__(self,
                            outcomes=['preempted','done','fail'],
                            input_keys=['nb_actions_done','cb_arm','cherries_loaded','nb_errors'],
                            output_keys=['nb_actions_done','cherries_loaded','cb_arm','nb_errors'])

    def execute(self, userdata):

        #Deploying an arm to depose the cherries
        log_info("Ordering Depose Cherries in the Bucket")
        userdata.cb_arm[0] = -1
        cherries_pub.publish(1) # Publish 1 to tell to the BN to deploy the arm in order to recover the cherries

        time.sleep(3)
        userdata.nb_actions_done[0] += 1
        return 'done'

        begin_time = time.time()
        while (time.time() - begin_time < ARM_TIMING_DEPOSE) :
            if self.preempt_requested():
                self.service_preempt()
                return 'preempted'
            time.sleep(0.1)
            #React to feedback
            if userdata.cb_arm[0] == 0 : #No problem
                log_info("Deposit sample arm : success")
                userdata.nb_actions_done[0] += 1
                return 'done'
            elif userdata.cb_arm[0] == 1 :
                log_errs("Deposit sample arm : fail")
                userdata.nb_errors[0] += 1
                return 'fail'

        if time.time() - begin_time >= ARM_TIMING_TAKE :
            log_info('Timeout reached')
            return action_error("fail")

        log_info("No response BN")
        return 'fail'

#TODO : Gestion d'erreur
#########################################################################################################
# TakeCherriesError - Substate managing potential errors induced by taking cherries on a rack
#########################################################################################################

class TakeCherriesPerpendicularError(smach.State) :
    def __init__(self) :
        smach.State.__init__(   self,
                                outcomes=['preempted','done', 'littleDisp'],
                                input_keys=['nb_actions_done','nb_errors','next_pos','cb_arm', 'cb_pos', 'color'],
                                output_keys=['nb_actions_done','nb_errors', 'next_pos','cb_arm'])

    def execute(self, userdata):
        if self.preempt_requested() :
            self.service_preempt()
            return 'preempted'

        if userdata.cb_arm[0] == 1 :
            if userdata.nb_errors[0] == 1 :
                x,y,theta = userdata.cb_pos[0]
                set_next_destination(userdata,x,y+POS_ACCURATE_DISP,theta,DISPLACEMENT['accur_av'])
                return 'littleDisp'

            elif userdata.nb_errors[0] == 2 :
                x,y,theta = (userdata.cb_pos[0][0], userdata.cb_pos[0][1], userdata.cb_pos[0][2])
                set_next_destination(userdata,x,y+NEG_ACCURATE_DISP,theta,DISPLACEMENT['accur_ar'])
                return 'littleDisp'

            elif userdata.nb_errors[0] == 3: # TODO Find n, the number of attempts we can make
                userdata.nb_errors[0] = 0
                userdata.nb_actions_done[0] += 1
                return 'done'

        elif userdata.cb_arm[0] == 2 : # We just skip the action
            userdata.nb_errors[0] = 0
            userdata.nb_actions_done[0] += 1
            return 'done'


# Import built-in modules
import logging
import os

# Import third-party modules
from PySide2 import QtCore
from PySide2 import QtGui
from PySide2 import QtWidgets

class CustomTableView(QtWidgets.QTableWidget):
    def __init__(self):
        super(CustomTableView, self).__init__()
        self.setObjectName("TableView")
        self.setup()

    def setup(self):
        header = self.horizontalHeader()
        header.setSectionResizeMode(QtWidgets.QHeaderView.ResizeToContents)
        vh = self.verticalHeader()
        vh.setVisible(False)
        vh.setMinimumHeight(1000)

    def resizeEvent(self, event):
        """ Resize all sections to content and user interactive """
        super(CustomTableView, self).resizeEvent(event)
        header = self.horizontalHeader()
        for column in range(header.count()):
            header.setSectionResizeMode(column,
                                        QtWidgets.QHeaderView.ResizeToContents)
            width = header.sectionSize(column)
            header.setSectionResizeMode(column,
                                        QtWidgets.QHeaderView.Interactive)
            header.resizeSection(column, width)

    def set_data(self, headers, data):
        self.setColumnCount(len(headers))
        self.setHorizontalHeaderLabels([header.NAME for header in headers])
        self.setRowCount(len(data))
        self.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
        self.setSizePolicy(QtWidgets.QSizePolicy(
            QtWidgets.QSizePolicy.Expanding,
            QtWidgets.QSizePolicy.Expanding))
        for index, info in enumerate(data):
            for column, header in enumerate(headers):
                item = QtWidgets.QTableWidgetItem()

                # if key == "thumbnail":
                #     self
                # else:
                item.setText(str(info[header.TEXT]))
                self.setItem(index, column, item)

class Thumbnail(QtWidgets.QLabel):
    def __init__(self, image_path):
        super(Thumbnail, self).__init__()
        self._pixomap = QtGui.QPixmap(image_path)
        self.setPixmap(self._pixomap)

class DropLabel(QtWidgets.QLabel):
    """Custom label that supports dropping and adds a file browser."""

    STYLE_DROPLABEL_NORMAL = """
    QLabel {
        background-color: #2f2f2f;
        border: 2px dotted #404040;
        color: #666666
    }
    """

    # Style of the DropLabel when having a selection hovering over it.
    STYLE_DROPLABEL_HOVER = """
    QLabel {
        background-color: #262626;
        border: 2px dotted #505050;
        color: #666666
    }
    """

    file_dropped = QtCore.Signal(str)
    clicked = QtCore.Signal()

    def __init__(self):
        """Initialize the DropLabel instance."""
        super(DropLabel, self).__init__()

        self.setAcceptDrops(True)

        self.setAlignment(QtCore.Qt.AlignCenter)
        self.setWordWrap(True)
        self.set_style(hover=False)
        self.setText("drop a folder.")

    def set_style(self, hover=True):
        """Set style for the widget.

        We need to set the style over a string. Using a style sheet won't work
        because we are updating the style in the main window afterwards.

        Args:
            hover (bool, optional): If True use hover style, otherwise use
                normal style.

        """
        style = self.STYLE_DROPLABEL_HOVER if hover else \
            self.STYLE_DROPLABEL_NORMAL
        self.setStyleSheet(style)

    def dragEnterEvent(self, event):  # pylint: disable=invalid-name
        """Overwrite dragEnterEvent to set mimeData."""
        event.accept()
        self.set_style(hover=True)

    def dragLeaveEvent(self, event):  # pylint: disable=invalid-name
        """Overwrite dragEnterEvent to set mimeData."""
        event.accept()
        self.set_style(hover=False)

    @staticmethod
    def dragMoveEvent(event):  # pylint: disable=invalid-name
        """Overwrite dragMoveEvent to accept drags."""
        event.setDropAction(QtCore.Qt.CopyAction)
        event.accept()

    @staticmethod
    def _get_valid_url(event):
        """Accepts only single existing directory with any kind of content:
        files, subdirectories etc.

        Args:
            event (QtGui.QDropEvent): Event from drag and drop process.

        Returns:
            str: Valid url if the content is valid.

        """
        if event.mimeData().hasUrls and (len(event.mimeData().urls()) == 1):
            url = event.mimeData().urls()[0]
            path = str(url.toLocalFile())
            if os.path.exists(path) and os.path.isdir(path):
                return path
        return None

    def dropEvent(self, event):  # pylint: disable=invalid-name
        """Fire file_dropped signal."""
        path = self._get_valid_url(event)
        if path:
            event.setDropAction(QtCore.Qt.CopyAction)
            event.accept()
            self.file_dropped.emit(path)
        else:
            logger = logging.getLogger(__name__)
            logger.warning('Please drop the folder try again.')

    def mousePressEvent(self, event):  # pylint: disable=invalid-name
        """Emit signal when clicked the label.

        Args:
            event (QtQore.QEvent): Event instance.

        """
        super(DropLabel, self).mousePressEvent(event)
        self.clicked.emit()

class MessageDisplay(QtWidgets.QDialog):
    """Shows dialogues as stand alone application i.e. can be called from scripts
    that have no QApplication running.
    """

    INFO = QtWidgets.QMessageBox.information
    WARNING = QtWidgets.QMessageBox.warning
    CRITICAL = QtWidgets.QMessageBox.critical
    ABOUT = QtWidgets.QMessageBox.about

    def __init__(self, title, message, dialog=None):
        """
        Args:
            title (str): Window title.
            message (str): Text of body.
            dialog (QtWidgets.QMessageBox): information, warning, critical, about
                                         Default is INFO.

        """
        if not dialog:
            dialog = MessageDisplay.INFO
        self.app = QtWidgets.QApplication.instance()
        if self.app is None:
            self.app = QtWidgets.QApplication([])
        super(MessageDisplay, self).__init__()
        dialog(self, title, message)


from Get_ETH_Balance_L2 import get_eth_l2
from Get_L1_Balances_v2 import Get_L1_Balances_v2

def eth_checker():
    eth_balance_l2 = int(get_eth_l2())
    L1_balances = Get_L1_Balances_v2({})
    eth_balance_l1 = int(L1_balances[0]['ethBalance'])

    eth_balance_l1_formatted = "{:,.2f}".format(eth_balance_l1 / 10**18)
    eth_balance_l2_formatted = "{:,.2f}".format(eth_balance_l2 / 10**18)
    difference = eth_balance_l2 - eth_balance_l1
    difference_formatted = "{:,.2f}".format(difference / 10**18)
    percentage_difference = abs(difference / eth_balance_l1) * 100
    percentage_difference_formatted = "{:.2f}%".format(percentage_difference)

    print("+-----------------+-----------------+-----------------+-----------------+-----------------+")
    print("|     Balance     |       L1        |       L2        |    Difference   | % Difference    |")
    print("+-----------------+-----------------+-----------------+-----------------+-----------------+")
    print("| Eth             | {:>15} | {:>15} | {:>15} | {:>15} |".format(eth_balance_l1_formatted, eth_balance_l2_formatted, difference_formatted, percentage_difference_formatted))
    print("+-----------------+-----------------+-----------------+-----------------+-----------------+")

if __name__ == '__main__':
    try:
        eth_checker()
    except Exception as e:
        print(f"Error: {e}")


import speech_recognition as sr
import webbrowser as wb
import pyttsx3

def Reconocer_voz():
    sr.Microphone(device_index = 0) #INICIALIZA EL MICRÓFONO 0 = POR DEFECTO
    print("Micrfono encontrado en la PC:  \n {sr.Microphone.list_micophpne_names()}")
    #CREAMOS UN OBJETO DE RECONOCIMIENTO
    reconocimiento = sr.Recognizer()

    def hablar(frase,texto):
        engine = pyttsx3.init()
        voces = engine.getProperty('voices')
        engine.setProperty('Voz',voces[0].id)
        engine.say(texto)
        engine.say(frase)
        engine.runAndWait()

    '''index = 0
        for voice in voces:
        print(f'index-> {index} -- {voice.name}')
        index +=1'''

    reconocimiento.energy_threshold = 50000 #TENEMOS VALORES DE 150 A 4000 son los buenos, PONEMOS EL QUE QUEDE
    reconocimiento.dynamic_energy_threshold = False

    with sr.Microphone() as source:
        print("Por favor hable fuerte y claro: ")
        reconocimiento.adjust_for_ambient_noise(source) #REDUCE EL RUIDO AMBIENTAL
        audioDelMicrofono = reconocimiento.listen(source) #TOMA LA VOZ DEL MICRÓFONO
        try:
            frase = reconocimiento.recognize_google(audioDelMicrofono, language = "es-MX")
            texto = "Dijiste algo como?"
            hablar(frase,texto)
            url = "https://www.google.com/search?q="
            buscar  = url + frase
            #wb.open(buscar)
        except TimeoutException as msg:
            print(msg)
        except WaitTimeoutError:
            print("Tardaste mucho en hablar")
            quit()
        # SI NO ENTIENDE EL AUDIO
        except LookupError:
            print("No entendí lo que quisiste decir")
        else:
            print("Tu resultado se mostrará en un momento. Adiós :) ")

Reconocer_voz()
#pip install pipwin
#pipwin install pyaudio
#pip uninstall pipwin


def solution(tickets):
    route = {}
    stack = ['ICN']
    visit = []

    for t in tickets:
        route[t[0]] = route.get(t[0],[]) + [t[1]]

    while stack:
        node = stack[-1]
        if node not in route or len(route[node])==0:
            visit.append(stack.pop())
        else:
            stack.append(route[node].pop())
    visit.reverse()
    return visit

print(solution([["ICN", "SFO"], ["ICN", "ATL"], ["SFO", "ATL"], ["ATL", "ICN"], ["ATL","SFO"]]))

print(solution([["ICN", "SFO"], ["ICN", "ATL"], ["SFO", "ATL"], ["ATL", "ICN"], ["ATL","SFO"]]))

# 현재 공항에서 갈 수 있는 모든 공항을 큐에 삽입
# 기존의 리스트에서는 삭제
#


import os

import pandas as pd
import matplotlib.pyplot as plt

#### load data
metricsDF = pd.read_csv('C:/Users/Darren Cook/Documents/PhD Research/csa_chats/predicting_predators/models/model1b_metrics.csv')
metricsDF = metricsDF.iloc[: , 1:]

#### Comparing performance between algorithms

## Visualization

# compare performance of algorithms
metricsDF_mean = metricsDF.groupby('model').mean()

# # add baseline averages
# baseline = pd.Series(data={'Precision':0.95,'Recall':0.86,'F1':0.91}, name='bs')
# metricsDF_mean = metricsDF_mean.append(baseline, ignore_index=False)

# boxplot
metricsDF.boxplot(
                  by='model',
                  layout=(3,5),
                  figsize=(8,8),
                  fontsize=10)
plt.suptitle('')
plt.savefig("plots/algorithm_comparisonbox_model1b.png", bbox_inches='tight')
plt.show()

## ANOVA
import scipy.stats as stats

# recall
recall = metricsDF.drop(['Precision','F1'], axis=1)
recall_unstacked = recall.pivot(columns='model')['Recall']
recall_unstacked = recall_unstacked.apply(lambda x: pd.Series(x.dropna().values))

# run test
fvalue, pvalue = stats.f_oneway(recall_unstacked['lr'],
                                recall_unstacked['nb'],
                                recall_unstacked['rf'],
                                recall_unstacked['svm'])
print(fvalue, pvalue)

# run post-hocs (Tukey HSD)
from statsmodels.stats.multicomp import pairwise_tukeyhsd

recall_posthoc = pairwise_tukeyhsd(endog=recall['Recall'],
                          groups=recall['model'],
                          alpha=0.01)

print(recall_posthoc)

# F1
f1 = metricsDF.drop(['Precision','Recall'], axis=1)
f1_unstacked = f1.pivot(columns='model')['F1']
f1_unstacked = f1_unstacked.apply(lambda x: pd.Series(x.dropna().values))

# run test
fvalue, pvalue = stats.f_oneway(f1_unstacked['lr'],
                                f1_unstacked['nb'],
                                f1_unstacked['rf'],
                                f1_unstacked['svm'])
print(fvalue, pvalue)

# run post-hocs (Tukey HSD)
f1_posthoc = pairwise_tukeyhsd(endog=f1['F1'],
                          groups=f1['model'],
                          alpha=0.01)

print(f1_posthoc)


import re
from django.db.models import Manager
from django.db.models.query import QuerySet

def get_class_name(obj=None):
    # Get name of parent object.
    if obj is None:
        return "Unnamed"
    else:
        return obj.__class__.__name__

class HashableList(list):
    def __hash__(self):
        return id(self)

class HashableDict(dict):
    """
    Hashable Dictionary

    Hashables should be immutable -- not enforcing this but TRUSTING you not to mutate a
    dict after its first use as a key.

    https://stackoverflow.com/questions/1151658/python-hashable-dicts
    """

    def __hash__(self):
        vals = ()
        for v in self.values():
            try:
                hash(v)
                vals += (str(v),)
            except TypeError:
                if isinstance(v, list):
                    for x in v:
                        vals += (str(x),)
                else:
                    vals += (str(v),)
        return hash((frozenset(self), frozenset(vals)))

def normalize_path(path):
    if path.startswith("."):
        path = path[1:]
    if path.endswith("."):
        path = path[:-1]
    return path

def get_path_parts(obj, path, base_name=None):
    pattern = re.compile(r"(\w+\.\w+)")
    parts = [normalize_path(x) for x in pattern.split(path, 1) if len(x)]
    parts_final = []
    for part in parts:
        try:
            part_field = part.split(".")[1]
        except IndexError:
            part_field = part
        parts_final.append([part_field, part])

    ret = (parts_final[0][0], parts_final[0][1])
    if len(parts_final) > 1:
        ret += (parts_final[1][0], parts_final[1][1])
    else:
        ret += ("", "")
    return ret

def get_object(obj):
    if isinstance(obj, Manager):
        obj = obj.all()
    if isinstance(obj, QuerySet):
        obj = obj.first()
    return obj

class DictDiffer(object):
    """
    Calculate the difference between two dictionaries as:
    (1) items added
    (2) items removed
    (3) keys same in both but changed values
    (4) keys same in both and unchanged values
    """

    def __init__(self, current_dict, past_dict):
        self.current_dict, self.past_dict = current_dict, past_dict
        self.current_keys, self.past_keys = (
            set(current_dict.keys()),
            set(past_dict.keys()),
        )
        self.intersect = self.current_keys.intersection(self.past_keys)

    def added(self):
        """ Find keys that have been added """
        return self.current_keys - self.intersect

    def removed(self):
        """ Find keys that have been removed """
        return self.past_keys - self.intersect

    def changed(self):
        """ Find keys that have been changed """
        return set(
            o for o in self.intersect if self.past_dict[o] != self.current_dict[o]
        )

    def unchanged(self):
        """ Find keys that are unchanged """
        return set(
            o for o in self.intersect if self.past_dict[o] == self.current_dict[o]
        )

    def new_or_changed(self):
        """ Find keys that are new or changed """
        # return set(k for k, v in self.current_dict.items()
        #           if k not in self.past_keys or v != self.past_dict[k])
        return self.added().union(self.changed())

def remove_redundant_paths(paths):
    """
    Returns a list of unique paths.
    """
    results = []
    for path in paths:
        redundant = False
        paths_copy = paths[:]
        paths_copy.pop(paths.index(path))
        for p in paths_copy:
            if p.startswith(path) and len(p) > len(path):
                redundant = True
        if redundant is False:
            results.append(path)
    return results

def sort_field_paths(field_paths):
    """
    Clean up a list of field paths by removing duplicates, etc.
    """
    result = list(set(field_paths))
    result = remove_redundant_paths(result)
    result = [x for x in result if len(x)]
    return result


#!/usr/bin/env python3

import collections
import subprocess
import datetime
import requests
import shutil
import pandas
import numpy
import copy
import sys
import re
import os

parent = os.path.abspath(os.path.join(os.getcwd(), os.pardir))
utils = os.path.join(parent,'utils')
sys.path.append(utils)

from plotting_utilities import *

def extractData(input_path,output_path,parameter,sensor,subsystem):
	outliers = 0

	pattern = '{}.*{}'.format(subsystem,sensor)
	status = grep(pattern,input_path,output_path,useAnd=True)
	if status > 0: # grep failed for some reason
		if os.path.exists(output_path):
			os.unlink(output_path)
		return

	cut(',','1,{}'.format(data_column),output_path,output_path)

	outliers = trimOutliers(output_path)
	# delFirstLine(output_path)

	return outliers

def getDates(days=1):
	date_list = []
	for i in range(1,days+1):
		date_list.append((datetime.datetime.utcnow()-datetime.timedelta(i)).strftime("%Y-%m-%d"))
	return date_list

if __name__ == '__main__':
	project_dir = sys.argv[1]
	data_column = sys.argv[2]
	cwd = os.getcwd()

	dicts = getNodes(project_dir)
	nodes_dict = dicts[0]
	details_dict = dicts[1]

	dicts = getSensors(project_dir)
	parameter_to_sensor_subsystem_dict = dicts[0]
	ontology_dict = dicts[1]
	subcategory_to_hrf_unit_dict = dicts[2]
	hrf_unit_to_sensor_dict = dicts[3]

	date_list = getDates(30)

	deleted_old_data = False

	# extract ontology/parameter data from daily data sets
	no_data_nodes = []
	for node in nodes_dict:

		# check if the node has anby data, if it doesn't then delete it from nodes dict later
		path = '{project_dir}/data/{node}'.format(project_dir=project_dir,node=node)
		# if a node has no data, skip it
		if not os.path.exists(path):
			no_data_nodes.append(node)
			continue

		# get details for the node to avoid extraneous operations
		details_dict_verbose = {'C':'chemsense','A':'alphasense','P':'plantower'}
		details = details_dict[node]
		for i,item in enumerate(details):
			details[i] = details_dict_verbose[item]

		# iterate over every date for which data exists
		for file in os.listdir(path):
			if file == 'final':
				continue
			print('Extracting {}/{}...'.format(node,file))

			input_path = os.path.join(path,file,'{}.csv'.format(file))

			outlier_dict = {}

			# iterate over every parameter from sensors.csv
			for parameter in parameter_to_sensor_subsystem_dict:
				input_path = os.path.join(path,file,'{}.csv'.format(file))

				# get the sensors and subsystems associated with the parameter
				sensor_list = []
				subsystem_list = []
				for value in parameter_to_sensor_subsystem_dict[parameter]:

					sensor_list.append(value[0])
					subsystem_list.append(value[1])

				if (len(sensor_list) == 0 or len(subsystem_list) == 0):
					continue

				# extract just the parameter data from each day
				output_path = '{project_dir}/data/{node}/{file}/{parameter}.csv'.format(project_dir=project_dir,node=node,parameter=parameter,file=file)
				# # if the data has already been extracted, don't do it again
				# if os.path.exists(output_path) or (deleted_old_data and file != datetime.datetime.utcnow().strftime("%Y-%m-%d")):
				# 	continue
				# if not os.path.isfile(output_path):
				# 	with open(output_path,'w+') as f:
				# 		f.close()

				status = grep(parameter,input_path,output_path)
				if status > 0:
					continue

				input_path = output_path

				# extract data for each sensor individually
				for i,sensor in enumerate(sensor_list):

					subsystem = subsystem_list[i]

					output_path = '{project_dir}/data/{node}/{file}/{parameter}-{sensor}-{subsystem}.csv'.format(project_dir=project_dir,node=node,file=file,sensor=sensor,parameter=parameter,subsystem=subsystem)
					# again, if the data has already been extracted, don't do it again
					if os.path.exists(output_path) or (deleted_old_data and file != datetime.datetime.utcnow().strftime("%Y-%m-%d")):
						continue
					outliers = extractData(input_path,output_path,parameter,sensor,subsystem)
					outlier_dict[parameter] = outliers

			# remove any empty files or files with only headers
			sub_path = os.path.join(path,file)
			for data_file in os.listdir(sub_path):
				p = re.compile('-')
				m = p.search(data_file)
				if not m:
					continue
				data_file_path = os.path.join(sub_path,data_file)

				try:
					if os.path.isfile(data_file_path) and os.stat(data_file_path).st_size <= 56:
						os.unlink(data_file_path)
					#elif os.path.isdir(data_file_path): shutil.rmtree(data_file_path)
				except Exception as e:
					print(e)
	# remove nodes from nodes_dict for which no data exists
	for node in no_data_nodes:
		del nodes_dict[node]

	time_periods = collections.OrderedDict([('day',1),('week',7),('month',30)])

	# combine files for plotting
	for node in nodes_dict:

		path = '{project_dir}/data/{node}'.format(project_dir=project_dir,node=node)
		final_path = os.path.join(path,'final')
		if not os.path.exists(final_path):
			os.makedirs(final_path)

		details_dict_verbose = {'C':'chemsense','A':'alphasense','P':'plantower'}
		details = details_dict[node]

		for time_period in time_periods:
			print('Combining data for {node} for the last {time}...'.format(node=node,time=time_period))
			time_period_path = os.path.join(final_path,time_period)
			if not os.path.exists(time_period_path):
				os.makedirs(time_period_path)

			# if old data was deleted then files to to be recombined, and i don't want to delete the first day of every
			# single file then append the most recent day because combining them doesn't take very long.
			if deleted_old_data:
				clearDir(time_period_path)

			for file in os.listdir(time_period_path):
				file_path = os.path.join(time_period_path,file)
				if os.path.exists(file_path) and os.stat(file_path).st_size > 56:
					# print(file_path)
					start_date = getStartDate(file_path)
					datecutoff = (datetime.datetime.utcnow()-datetime.timedelta(time_periods[time_period]-1)).strftime("%Y/%m/%d 00:00:00")
					# print(start_date,datecutoff)
					if start_date < datecutoff:
						# print('deleting...')
						if os.path.exists(file_path):
							os.unlink(file_path)

			date_list = getDates(time_periods[time_period])
			chronological = list(reversed(date_list))

			sensor_list = []
			subsystem_list = []

			for parameter in parameter_to_sensor_subsystem_dict:

				for pair in parameter_to_sensor_subsystem_dict[parameter]:

					sensor_list.append(pair[0])
					subsystem_list.append(pair[1])

				if (len(sensor_list) == 0 or len(subsystem_list) == 0):
					continue

				for i,sensor in enumerate(sensor_list):
					subsystem = subsystem_list[i]
					param_sens_subs = '{}-{}-{}'.format(parameter,sensor,subsystem)

					time_period_data_path = '{path}/{string}-{time}.csv'.format(path=time_period_path,string=param_sens_subs,time=time_period)
					if os.path.exists(time_period_data_path):
						continue

					with open(time_period_data_path,'w+') as f:
						for date in chronological:
							day_path = '{data}/{date}/{string}.csv'.format(data=path,date=date,string=param_sens_subs)
							# print(day_path)
							if os.path.exists(day_path):

								with open(day_path,'r') as r:
									f.write(r.read())

			#delete empty files in each time period
			cleanupEmptyFilesInDir(time_period_path)

		# delete any empty files that somehow ended up in the final folder
		cleanupEmptyFilesInDir(final_path)

	# plot generation
	plots_path = '{project_dir}/plots'.format(project_dir=project_dir)
	if not os.path.exists(plots_path):
		os.makedirs(plots_path)

	plot_logs_dir = os.path.join(project_dir,'plot_logs')
	if not os.path.exists(plot_logs_dir):
		os.makedirs(plot_logs_dir)

	removed_dict = {}
	for time_period in time_periods:

		date_list = getDates(time_periods[time_period])
		x_range = "['{start} 00:00:00':'{end} 23:59:59']".format(start=date_list[-1].replace('-','/'),end=date_list[0].replace('-','/'))
		# tomorrow: datetime.date.today()+datetime.timedelta(days=1).strftime()
		for node in nodes_dict:
			with open('{}/{}.log'.format(plot_logs_dir,node),'w+') as f:
				f.write('subsystem,sensor,parameter\n')
			if node not in removed_dict:
				removed_dict[node] = []

			node_plot_path = os.path.join(plots_path,node,time_period)
			if not os.path.exists(node_plot_path):
				os.makedirs(node_plot_path)

			data_path = '{project_dir}/data/{node}/final/{time}'.format(project_dir=project_dir,node=node,time=time_period)

			vsn = nodes_dict[node][0]
			address = nodes_dict[node][1]
			address = re.sub('"','',address)

			description = nodes_dict[node][2]

			temp_ontology_dict = copy.deepcopy(ontology_dict)

			pm_flag = False

			for ontology in temp_ontology_dict:
				if ontology == '/sensing/air_quality/particulates/pm_2_5' or ontology == '/sensing/air_quality/particulates/pm_10':
					continue

				for date in date_list:
					remove = []
					for triplet in temp_ontology_dict[ontology]:
						if not os.path.exists('{data}/{}-{}-{}-{}.csv'.format(triplet[0],triplet[1],triplet[2],time_period,data=data_path)):
							remove.append(triplet)
				for triplet in remove:
					temp_ontology_dict[ontology].remove(triplet)
					removed_dict[node].append(triplet)
					with open('{}/{}.log'.format(plot_logs_dir,node),'a') as f:
						f.write(triplet[2]+','+triplet[1]+','+triplet[0]+'\n')
				if len(temp_ontology_dict[ontology]) == 0:
					# print(ontology)
					continue
				# for key in temp_ontology_dict:
				# 	print(key,temp_ontology_dict[key])
				# print(remove)

				ontology_list = ontology.split('/')
				parameter = ontology_list[-1]

				sensor_list = []
				sensor_string = ''
				parameter_list = []
				parameter_string = ''
				plot = ''
				logscale = ''

				# get a list of sensors, parameters and subsytems, as well as
				# a string for each for the plot
				sensor_list = []
				sensor_set = collections.OrderedDict([])
				sensor_string = ''
				parameter_list = []
				parameter_set = collections.OrderedDict([])
				parameter_string = ''
				subsystem_list = []

				for triplet in temp_ontology_dict[ontology]:
					sensor_list.append(triplet[1])
					sensor_set[triplet[1]] = None
					parameter_list.append(triplet[0])
					parameter_set[triplet[0]] = None
					subsystem_list.append(triplet[2])
				for parameter in parameter_set:
					if len(parameter_string) == 0:
						parameter_string = parameter
					else:
						parameter_string = '{}+{}'.format(parameter_string,parameter)
				for sensor in sensor_set:
					if len(sensor_string) == 0:
						sensor_string = sensor
					else:
						sensor_string = '{}+{}'.format(sensor_string,sensor)

				# get labeles for the y axis of each plot
				ylabel_list = []
				for unit in subcategory_to_hrf_unit_dict[ontology_list[3]]:
					if unit not in ylabel_list:
						ylabel_list.append(unit)
						if unit == 'lux' and ontology == '/system/other/light':
							if 'lux' in ylabel_list:
								ylabel_list.remove('lux')
				if 'RH' in ylabel_list:
					for i,item in enumerate(ylabel_list):
						if item == 'RH':
							ylabel_list[i] = '%RH'
				if 'C' in ylabel_list:
					for i,item in enumerate(ylabel_list):
						if item == 'C':
							ylabel_list[i] = 'deg C'

				if ontology_list[-1] == 'particle_count':
					if 'pms7003' in sensor_list:
						# ylabel_list.remove('counts')
						1
					if 'opc_n2' in sensor_list:
						ylabel_list.remove('mg/m^3')

				# if ontology_list[3] == 'pressure':
				# 	ylabel_list.remove('C')

				#plotting pm1, pm2_5, and pm10 on one graph requires a different setup because all 3 have different ontologies
				if ontology_list[-2] == 'particulates' and ontology_list[-1] != 'particle_count':
					sensor_list = []
					sensor_string = ''
					parameter_list = []
					parameter_string = ''
					parameter_set = collections.OrderedDict([])
					subsystem_list = []

					if not pm_flag:
						pm_flag = True
						pm_list = []
						for ontology in temp_ontology_dict:
							p = re.compile('/sensing/air_quality/particulates/pm_[0125_]*')
							m = p.match(ontology)
							if m:
								pm_list.append(temp_ontology_dict[ontology])
						for triplet in pm_list:
							# print('TRPIPLET',triplet)
							sensor_list.append(triplet[0][1])
							if len(sensor_string) == 0:
								sensor = triplet[0][1]
							else:
								sensor = '{}+{}'.format(sensor,triplet[0][1])
							parameter_list.append(triplet[0][0])
							parameter_set[triplet[0][0]] = None
							subsystem_list.append(triplet[0][2])
						for parameter in parameter_set:
							if len(parameter_string) == 0:
								parameter_string = parameter
							else:
								parameter_string = '{}+{}'.format(parameter_string,parameter)
					ontology = '{}/{}/{}'.format(ontology_list[0],ontology_list[1],ontology_list[2])

				#generate the plot script for plotting only one unit
				if (len(ylabel_list) == 1):
					ylabel = ylabel_list[0]
					plot = ''#"['{start} 00:00:00':'{end} 23:59:59']".format(start=date_list[-1],end=date_list[0])

					for i,sensor in enumerate(sensor_list):
						dataset_path = "{dir}/data/{node}/final/{time}/{param}-{sensor}-{subsystem}-{time}.csv".format(dir=project_dir,sensor=sensor,node=node,param=parameter_list[i],subsystem=subsystem_list[i],time=time_period)
						if not os.path.exists(dataset_path):
							continue
						elif os.stat(dataset_path).st_size == 0:
							continue
						if i == 0:
							plot = '{} "{path}" using 1:2 with lines title "{param_upper} by sensor {sensor}"'.format(plot,path=dataset_path,sensor=sensor.replace('_','\\\\_'),param=parameter_list[i].replace('_','\\\\_'),param_upper=parameter_list[i].replace('_','\\\\_'),node=node,subsystem=subsystem_list[i])
						else:
							plot = "{}, {}".format(plot,'"{path}" using 1:2 with lines title "{param_upper} by sensor {sensor}"'.format(path=dataset_path,sensor=sensor.replace('_','\\\\_'),param=parameter_list[i].replace('_','\\\\_'),param_upper=parameter_list[i].replace('_','\\\\_'),node=node,subsystem=subsystem_list[i]))
					tics = 'tics'

				#generate plot script, alternate ylabel and alternate tics for plotting w/ 2 y axes
				elif len(ylabel_list) == 2:
					ylabel = '{}"\nset y2label "{}'.format(ylabel_list[0],ylabel_list[1])
					plot = ''#"['{start} 00:00:00':'{end} 23:59:59']".format(start=date_list[-1],end=date_list[0])

					for j,unit in enumerate(ylabel_list):
						for i,sensor in enumerate(sensor_list):
							dataset_path = "{dir}/data/{node}/final/{time}/{param}-{sensor}-{subsystem}-{time}.csv".format(dir=project_dir,sensor=sensor,node=node,param=parameter_list[i],subsystem=subsystem_list[i],time=time_period)
							if sensor not in hrf_unit_to_sensor_dict[unit]:
								continue
							#check if a dataset with data exists
							if  not os.path.exists(dataset_path):
								continue
							elif os.stat(dataset_path) == 0:
								continue

							#add lines to the plot script
							if i == 0:
								plot = '{} "{path}" using 1:2 with lines title "{param_upper} by sensor {sensor} ({unit})" noenhanced'.format(plot,path=dataset_path,sensor=sensor.replace('_','\\\\_'),param=parameter_list[i].replace('_','\\\\_'),param_upper=parameter_list[i].replace('_','\\\\_'),node=node,subsystem=subsystem_list[i],unit=unit)
							else:
								plot = "{}, {}".format(plot,'"{path}" using 1:2 with lines title "{param_upper} by sensor {sensor} ({unit})" noenhanced'.format(path=dataset_path,sensor=sensor.replace('_','\\\\_'),param=parameter_list[i].replace('_','\\\\_'),param_upper=parameter_list[i].replace('_','\\\\_'),node=node,subsystem=subsystem_list[i],unit=unit))
							if (j == 1):
								plot = "{} {}".format(plot, 'axes x1y2')
					tics = 'ytics nomirror autofreq tc lt 1\nset y2tics nomirror autofreq tc lt 2'
					# print(plot,ylabel_list)

				# 																	  #background,borders,x,y,plotting....
				plot_template = '''
set terminal png medium size 1920,1080 enhanced font "Helvetica,16" #ffffff #000000 #404040 #ff0000 #e41a1c #377eb8 #4daf4a #984ea3 #ff7f00 #ffff33 #a65628 #f781bf #999999
set tics font "Helvetica,14"
set output "{path}/{node}-{ontology_underscores}-{time_period}.png"
set title "{vsn} | {description} | {address} | {ontology}\\n{{/*0.7 Created: {today}, Data: {data_address}, Code: Version {version} of {{/Courier {code}}}, Command: {{/Courier ./digest.sh {project}}}}}"
set xlabel "Time (UTC)"
set ylabel noenhanced "{ylabel}"
set datafile separator ','
set timefmt '%Y/%m/%d %H:%M:%S'
set xdata time
set autoscale
set xrange {xrange}
{logscale}
set format x "%H.%M\\n%m/%d"
set {tics}
set grid
plot {plot}
'''

				context = {
					'address':address,
					'code':'gen\\\\_digest\\\\_plots.py',
					'description':description,
					'data_address':project_dir.replace('_','\\\\_').replace('scratch/',''),
					'logscale':logscale,
					'parameter':parameter_string,
					'plot':plot,
					'project':project_dir.replace('_','\\\\_').replace('scratch/',''),
					'node':node,
					'ontology':ontology.replace('_','\\\\_'),
					'ontology_underscores':ontology.strip('/').replace('/','_'),
					'path':node_plot_path,
					'sensor':sensor_string,
					'time_period':time_period,
					'today':datetime.datetime.utcnow().strftime("%Y-%m-%d"),
					'tics':tics,
					'version':'1',
					'vsn':vsn,
					'xrange':x_range,
					'ylabel':ylabel,
				}

				output = plot_template.format(**context).strip()
				# print(output.strip())

				with open('graph.plt','w') as f:
					f.write(output)

				print("Plotting {node} {time_period} {ontology}".format(node=node,time_period=time_period,ontology=ontology))

				subprocess.run(['gnuplot', 'graph.plt'])

	# for key in removed_dict:
	# 	print(key,removed_dict[key])


# Load the WCS information from a fits header, and use it
# to convert pixel coordinates to world coordinates.

from __future__ import division, print_function
import math
import numpy
from astropy import wcs
from astropy.io import fits
import sys

def load_wcs_from_file(filename):
    # Load the FITS hdulist using astropy.io.fits
    hdulist = fits.open(filename, memmap=True)
    hdulist.info()
#    hdulist[0].header['CTYPE1']
#    hdulist.writeto('backup.fits')
    # Parse the WCS keywords in the primary HDU
    w = wcs.WCS(hdulist[0].header)
    # Print out the "name" of the WCS, as defined in the FITS header
    #print(w.wcs.name)
    # Print out all of the settings that were parsed from the header
    #w.wcs.print_contents()

    # Some pixel coordinates of interest.
    pixcrd = numpy.array([[0., 0.], [1., 1.]])

    # Convert pixel coordinates to world coordinates
    # The second argument is "origin" -- in this case we're declaring we
    # have 1-based (Fortran-like) coordinates.
    world = w.wcs_pix2world(pixcrd, 1)
    print(convert(pixcrd))
    print(world )
    # Convert the same coordinates back to pixel coordinates.
#    pixcrd2 = w.wcs_world2pix(world, 1)
#    print(pixcrd2)

    # These should be the same as the original pixel coordinates, modulo
    # some floating-point error.
  #  assert numpy.max(numpy.abs(pixcrd - pixcrd2)) < 1e-6

def convert(list):
    for x in list:
        x[0] = (x[0]-1030.)*(-0.008333333333333333333)
        if(x[0] < 0):
            x[0] = x[0] + 360
        x[1] = (x[1]-808.)*(0.00833333333333333333333)
        x[1] = math.asin(x[1]*(math.pi/180.))*180./math.pi
    return list

hdu=fits.PrimaryHDU()
hdu.header

if __name__ == '__main__':
    load_wcs_from_file(sys.argv[-1])