from parameters import ParameterSet
import numpy as np
from matplotlib import pyplot as plt

# Add multi core parallelization
from concurrent.futures import ThreadPoolExecutor, as_completed

# Import main simulation function
import simulation_main as sMAIN #import main simulation function

class simulation_screen(object):
    """
    Run screen of simulations over 1 to 3 parameters

    Parameters
    ----------
    sim_parameters: dict
        Dictionary including all simulation parameters
    param1_name: str
        Name of parameter 1 to screen (use name according to sim_parameter naming).
    params1: list
        List of values to use for screening of parameter 1.
    param2_name: str, optional
        Name of parameter 2 to screen (use name according to sim_parameter naming).
    params1: list, optional
        List of values to use for screening of parameter 2.
    param3_name: str, optional
        Name of parameter 3 to screen (use name according to sim_parameter naming).
    params3: list, optional
        List of values to use for screening of parameter 3.
    """
    def __init__(self, sim_parameters, param1_name, params1,
                 param2_name=None, params2=[0],
                 param3_name=None, params3=[0]):
        self.param1_name = param1_name
        self.param2_name = param2_name
        self.param3_name = param3_name
        self.params1 = params1
        self.params2 = params2
        self.params3 = params3
        self.sim_parameters = sim_parameters
        self.dist_compare_cat = []
        self.dist_compare_contact = []

        # Initialize output variables:
        dim1 = len(self.params1)
        dim2 = len(self.params2)
        dim3 = len(self.params3)
        # Output as numpy arrays:
        self.SCAN_dt = np.zeros((dim1,dim2,dim3))
        self.SCAN_mean_cat_times = np.zeros((dim1,dim2,dim3))
        self.SCAN_mean_contact_times = np.zeros((dim1,dim2, dim3))
        # Output as lists:
        self.SCAN_EB_comet_sum = [[[0]*dim3 for i in range(dim2)] for j in range(dim1)]
        self.SCAN_EB_comet_sum_barrier = [[[0]*dim3 for i in range(dim2)] for j in range(dim1)]
        self.SCAN_cat_times = [[[0]*dim3 for i in range(dim2)] for j in range(dim1)]
        self.SCAN_cat_length = [[[0]*dim3 for i in range(dim2)] for j in range(dim1)]
        self.SCAN_barrier_contact_times = [[[0]*dim3 for i in range(dim2)] for j in range(dim1)]
        self.SCAN_cap_end = [[[0]*dim3 for i in range(dim2)] for j in range(dim1)]

        # Initialize screening parameter sets:
        parameter_collection = []
        for i, param1 in enumerate(self.params1):
            if self.param2_name == None: # screen 1 parameter
                sim_parameters = self.sim_parameters
                sim_parameters[self.param1_name] = param1
                print(sim_parameters)
                parameter_collection.append([i, 0, 0, sim_parameters.copy()])
            elif self.param3_name == None: # screen 2 parameters
                for j, param2 in enumerate(self.params2):
                    sim_parameters = self.sim_parameters
                    sim_parameters[self.param1_name] = param1
                    sim_parameters[self.param2_name] = param2
                    parameter_collection.append([i, j, 0, sim_parameters.copy()])
            else: # screen 3 parameter
                for j, param2 in enumerate(self.params2):
                    for k, param3 in enumerate(self.params3):
                        sim_parameters = self.sim_parameters
                        sim_parameters[self.param1_name] = param1
                        sim_parameters[self.param2_name] = param2
                        sim_parameters[self.param3_name] = param3
                        parameter_collection.append([i, j, k, sim_parameters.copy()])

        self.parameter_collection = parameter_collection

    def run_screen_multicore(self, barrier_or_free = 'both', num_workers=4):
        """
        Run simulation parameter scan on multiple threads.

        Parameters
        ----------
        barrier_or_free: str
            Must be "barrier', 'free', or 'both'.
        num_workers: int
            Number of threads for parallelization.
            Usually the number of CPUs or
        """

        #Create a pool of processes. By default, one is created for each CPU in your machine.
        #with concurrent.futures.ProcessPoolExecutor() as executor:
        print("Run parameter scan on ", num_workers, "number of workers.")
        with ThreadPoolExecutor(max_workers=num_workers) as executor:
            futures = [executor.submit(self.single_run, X, barrier_or_free=barrier_or_free) for X in self.parameter_collection]

        for future in as_completed(futures):
            print("Single simulation run done.")
            #print(future.result())

    def run_screen(self, barrier_or_free = 'both'):
        """
        Run simulation parameter scan without parallelization.

        Parameters
        ----------
        barrier_or_free: str
            Must be "barrier', 'free', or 'both'.
        """

        print("Run parameter scan without parallelization.")
        for X in self.parameter_collection:
            self.single_run(X, barrier_or_free=barrier_or_free)

    def single_run(self, parameter_collect, barrier_or_free = 'both'):
        """
        Run simulations for one single parameter set

        Parameters
        ----------
        parameter_collect: list
            List including the position along the screening parameters,
            and the respective parameter set
            --> [i,j,k, sim_parameters]
            with sim_parameters a dictionary containing the parameters
        barrier_or_free: str
            Must be "barrier', 'free', or 'both'.
        """

        i, j, k, sim_parameters = parameter_collect
        print('\n', '==='*5, 'Start run ', i,' - ',j, ' - ', k, )

        if (barrier_or_free == 'free') or (barrier_or_free == 'both'):
            if sim_parameters['barrier']:
                barrier = sim_parameters['barrier']
            else:
                print("No barrier distance given in parameter file")

        if (barrier_or_free == 'free') or (barrier_or_free == 'both'):
            sim_parameters['barrier'] = False
            simPa = ParameterSet(sim_parameters)

            dt, MT_length_sum, CATASTROPHE_TIMES, CATASTROPHE_LENGTH, barrier_contact_times, \
            EB_comet_sum, cap_end, frame_rate_actual, EB_profiles  = sMAIN.MT_RUN(simPa)

            self.SCAN_dt[i,j,k] = dt
            self.SCAN_EB_comet_sum[i][j][k] = EB_comet_sum
            self.SCAN_cat_times[i][j][k] = CATASTROPHE_TIMES
            self.SCAN_cat_length[i][j][k] = CATASTROPHE_LENGTH
            if len(CATASTROPHE_TIMES) > 10: # needs to have minimum 10 events to count !
                tau_c = np.mean(np.array(CATASTROPHE_TIMES))
                print(len(CATASTROPHE_TIMES),' catastrophes | catastrophe time: %.2f s' %tau_c)
            else:
                tau_c = 2000
            self.SCAN_mean_cat_times[i,j,k] = tau_c

        if (barrier_or_free == 'barrier') or (barrier_or_free == 'both'):
            print('barrier at: ', barrier)

            sim_parameters['barrier'] = barrier
            simPa = ParameterSet(sim_parameters)
            dt, MT_length_sum, CATASTROPHE_TIMES, CATASTROPHE_LENGTH, barrier_contact_times, \
            EB_comet_sum, cap_end, frame_rate_actual, EB_profiles  = sMAIN.MT_RUN(simPa)

            if (barrier_or_free == 'barrier'):
                self.SCAN_dt[i,j,k] = dt
            self.SCAN_barrier_contact_times[i][j][k] = barrier_contact_times

            if barrier_contact_times.shape[0] > 0:
                mean_contact = np.mean(barrier_contact_times)
                print('Barrier contact time: %.2f s' %mean_contact)
            else:
                mean_contact = 0

            self.SCAN_mean_contact_times[i,j,k] = mean_contact
            self.SCAN_EB_comet_sum_barrier[i][j][k] = EB_comet_sum

    def compare_to_results(self, cum_hist_exp, type = 'free'):
        """ Compare results from simulation screening to empirical data

        Parameters
        ----------
        cum_hist_exp: numpy array
            Cumulative distribution of contact or catastrophe times.
        type: str
            Type of empirical data, freely growing (type='free') or stalled ('barrier').
        """
        dim1 = len(self.params1)
        dim2 = len(self.params2)
        dim3 = len(self.params3)
        dist_compare_cat = np.zeros((dim1,dim2,dim3))
        dist_compare_contact = np.zeros((dim1,dim2,dim3))

        # Loop through all screening simulations
        for i in range(dim1):
            for j in range(dim2):
                for k in range(dim3):
                    if type == 'free':
                        # freely growing MTs -------------------------------------
                        cum_hist = np.array(self.SCAN_cat_times[i][j][k]) # load respective model curve
                        cum_hist = np.append(cum_hist, 0)
                        cum_hist = np.sort(cum_hist)

                        if cum_hist.shape[0] < 5:
                                 dist_compare_cat[i,j,k] = 0
                        else:
            #                Previously used KS score:
            #                stats_shifted = np.zeros(20+1)
            #                for m in range(0,20+1):
            #                    #stats_shifted[m] = stats.ks_2samp(cum_hist_exp, (1 + FACT.shift_ks_cat - FACT.shift_ks_cat/10*m)*cum_hist)[1]
            #                    stats_shifted[m] = distribution_compare(cum_hist_exp, (1 + FACT.shift_ks_cat - FACT.shift_ks_cat/10*m)*cum_hist)
            #
            #                dist_compare_cat[i,j,k] = min(stats_shifted)
                             dist_compare_cat[i,j,k] = distribution_compare(cum_hist_exp, cum_hist)

                    elif type == 'barrier':
                        # barrier stalled MTs -------------------------------------
                        cum_hist = np.array(self.SCAN_barrier_contact_times[i][j][k]) #load respective model curve
                        cum_hist = np.append(cum_hist, 0)
                        cum_hist = np.sort(cum_hist)

                        if cum_hist.shape[0] < 5:
                                dist_compare_contact[i,j,k] = 0
                        else:
            #                Previously used KS score:
            #                stats_shifted = np.zeros(20+1)
            #                for m in range(0,20+1):
            #                    #stats_shifted[m] = stats.ks_2samp(cum_hist_exp, (1 + FACT.shift_ks_contact - FACT.shift_ks_contact/10*m)*cum_hist)[1]
            #                    stats_shifted[m] = distribution_compare(cum_hist_exp, (1 + FACT.shift_ks_contact - FACT.shift_ks_contact/10*m)*cum_hist)
            #
            #                dist_compare_contact[i,j,k] = min(stats_shifted)
                            dist_compare_contact[i,j,k] = distribution_compare(cum_hist_exp, cum_hist)

                    else:
                        print("Given type must be 'free', 'barrier'.")

        if type == 'free':
            dist_compare_cat[np.where(self.dist_compare_cat == 0)] = np.max(dist_compare_cat)
            self.dist_compare_cat = dist_compare_cat
        elif type == 'barrier':
            dist_compare_contact[np.where(dist_compare_contact == 0)] = np.max(dist_compare_contact)
            self.dist_compare_contact = dist_compare_contact

    def plot_dist_compare(self, scan_x_label, scan_y_label, type = 'free'):
        """ Plot MSE

        Parameters
        ----------
        scan_x_label: str
            Label of x axis
        scan_y_label: str
            Label of y axis
        type: str
            Type of empirical data, freely growing (type='free') or stalled ('barrier'), or 'both'.
        """
        dim1 = len(self.params1)
        dim2 = len(self.params2)
        dim3 = len(self.params3)

        if (type == 'barrier') or (type == 'both'):
            cols = min(dim3,2)
            rows = int(np.ceil(dim3/cols))

            fig, axs = plt.subplots(nrows=rows, ncols=cols, sharex='col', sharey='row',
                                    figsize=(6 * cols, 6* dim1/dim2 *rows ))

            for i in range(rows):
                for j in range(cols):
                    count = j * rows + i
                    if count < dim3:
                        if cols == 1:
                            ax = axs
                        elif rows >1:
                            ax = axs[j,i]
                        else:
                            ax = axs[count]

                        map = ax.imshow(self.dist_compare_contact[:,:,count]**0.5, aspect='auto',
                                        vmin = 0, vmax= np.max(self.dist_compare_contact[:,:,count]**0.5),
                                        cmap='gist_heat_r', interpolation='nearest')
                        ax.set_ylabel(scan_y_label)
                        ax.set_xlabel(scan_x_label)
                        fig.colorbar(map, ax = ax, label = 'sqrt(MSE)')

                    ax.set_xticks(np.arange(dim1))
                    ax.set_yticks(np.arange(dim2))
                    ax.set_xticklabels(self.params1)
                    ax.set_yticklabels(self.params2)

            fig.suptitle('MSE between cumulative contact time distributions', fontsize=16)

#            if self.sim_parameters["record_data"]:
    #            filename = foldername + "/" + DATE + '_screen_04'
    #            plt.savefig(filename+'.pdf', format='pdf', dpi=1000) #, transparent=True)
    #            plt.savefig(filename+'.png', format='png', dpi=200) #, transparent=True )
            plt.show()

        if (type == 'free') or (type == 'both'):
            cols = min(dim3,2)
            rows = int(np.ceil(dim3/cols))

            fig, axs = plt.subplots(nrows=rows, ncols=cols, sharex='col', sharey='row',
                                    figsize=(6 * cols, 6* dim1/dim2 *rows ))

            for i in range(rows):
                for j in range(cols):
                    count = j * rows + i
                    if count < dim3:
                        if cols == 1:
                            ax = axs
                        elif rows >1:
                            ax = axs[j,i]
                        else:
                            ax = axs[count]

                        map = ax.imshow(self.dist_compare_cat[:,:,count]**0.5, aspect='auto',
                                        vmin = 0, vmax= np.max(self.dist_compare_cat[:,:,count]**0.5),
                                        cmap='gist_heat_r', interpolation='nearest')

                        ax.set_ylabel(scan_y_label)
                        ax.set_xlabel(scan_x_label)
                        fig.colorbar(map, ax = ax, label = 'sqrt(MSE)')#'P (Kolmogorov-Smirnov)')

                    ax.set_xticks(np.arange(dim1))
                    ax.set_yticks(np.arange(dim2))
                    ax.set_xticklabels(self.params1)
                    ax.set_yticklabels(self.params2)

            fig.suptitle('MSE between cumulative catastrophe time distributions', fontsize=16)

#            if self.sim_parameters["record_data"]:
#                filename = foldername + "/" + DATE + '_screen_05'
#                plt.savefig(filename+'.pdf', format='pdf', dpi=1000) #, transparent=True)
#                plt.savefig(filename+'.png', format='png', dpi=200) #, transparent=True )

            plt.show()

        if type == 'both':
            cols = min(dim3,2)
            rows = int(np.ceil(dim3/cols))

            fig, axs = plt.subplots(nrows=rows, ncols=cols, sharex='col', sharey='row',
                                    figsize=(6 * cols, 6* dim1/dim2 *rows ))

            for i in range(rows):
                for j in range(cols):
                    count = j * rows + i
                    if count < dim3:
                        if cols == 1:
                            ax = axs
                        elif rows >1:
                            ax = axs[j,i]
                        else:
                            ax = axs[count]

                        map = ax.imshow(np.multiply(self.dist_compare_cat[:,:,count]**0.5,self.dist_compare_contact[:,:,count]**0.5),
                                        vmin= 0, vmax= np.mean(self.dist_compare_cat[:,:,count]**0.5)*np.mean(self.dist_compare_contact[:,:,count]**0.5),
                                        aspect='auto', cmap='gist_heat_r', interpolation='nearest')
                        ax.set_ylabel(scan_y_label)
                        ax.set_xlabel(scan_x_label)
                        fig.colorbar(map, ax = ax, label = 'sqrt(MSE*MSE)')#'P (Kolmogorov-Smirnov)')

                    ax.set_xticks(np.arange(dim1))
                    ax.set_yticks(np.arange(dim2))
                    ax.set_xticklabels(self.params1)
                    ax.set_yticklabels(self.params2)

            fig.suptitle('Combined MSE between cumulative distributions (=MSE1 * MSE2)', fontsize=16)

#            if self.sim_parameters["record_data"]:
#                filename = foldername + "/" + DATE + '_screen_06'
#                plt.savefig(filename+'.pdf', format='pdf', dpi=1000) #, transparent=True)
#                plt.savefig(filename+'.png', format='png', dpi=200) #, transparent=True )

            plt.show()

# ----------------------------------------------------------------------------
# Helper functions
# ----------------------------------------------------------------------------
from scipy.interpolate import interp1d

# Alternative version I tested and played around with
#def distribution_compare(Cum_hist1, Cum_hist2, num_interpol=1000):
#    """ Compare two distributions
#
#    Parameters
#    ----------
#    Cum_hist1: numpy array
#        (Cumulative) distribution 1
#    Cum_hist2: numpy array
#        (Cumulative) distribution 1
#    num_interpol: int, optional
#        Number of interpolation points.
#        In general: More is better. Less is faster. Much more than resolution of data will not add anything.
#    """
#    y1 = 1/(len(Cum_hist1)-1) * np.arange(0, len(Cum_hist1) , 1)
#    y2 = 1/(len(Cum_hist2)-1) * np.arange(0, len(Cum_hist2) , 1)
#    fit1 = interp1d(y1, Cum_hist1, kind='cubic')
#    fit2 = interp1d(y2, Cum_hist2, kind='cubic')
#    ynew = np.linspace(0,0.95, num=num_interpol) # only look at first 95% (avoid overly strong influence of 'weird' tails)
#    return (np.mean((fit1(ynew) - fit2(ynew))**2) / np.mean([np.mean(Cum_hist1), np.mean(Cum_hist2)]))

def distribution_compare(Cum_hist1, Cum_hist2, num_interpol=10000):
    """ Compare two distributions

    Parameters
    ----------
    Cum_hist1: numpy array
        (Cumulative) distribution 1
    Cum_hist2: numpy array
        (Cumulative) distribution 1
    num_interpol: int, optional
        Number of interpolation points.
        In general: More is better. Less is faster. Much more than resolution of data will not add anything.
    """
    y1 = 1/(len(Cum_hist1)-1) * np.arange(0, len(Cum_hist1) , 1)
    y2 = 1/(len(Cum_hist2)-1) * np.arange(0, len(Cum_hist2) , 1)
    fit1 = interp1d(Cum_hist1, y1, kind='nearest')
    fit2 = interp1d(Cum_hist2, y2, kind='nearest')
    xnew = np.linspace(0,min(max(Cum_hist1),max(Cum_hist2)), num=num_interpol) # only compare the actual overlapping part
    return (np.mean((fit1(xnew) - fit2(xnew))**2))


#!/usr/bin/env python

# =============================================================================
# MODULE DOCSTRING
# =============================================================================

"""
Test classes and functions in tfep.nn.graph.
"""

# =============================================================================
# GLOBAL IMPORTS
# =============================================================================

import pytest
import torch

from tfep.nn.masked import create_autoregressive_mask
from tfep.nn.graph import (
    get_all_edges,
    fix_edges_batch_size,
    compute_edge_distances,
    prune_long_edges,
    unsorted_segment_sum,
)

# =============================================================================
# TEST MODULE CONFIGURATION
# =============================================================================

_old_default_dtype = None

def setup_module(module):
    global _old_default_dtype
    _old_default_dtype = torch.get_default_dtype()
    torch.set_default_dtype(torch.double)

def teardown_module(module):
    torch.set_default_dtype(_old_default_dtype)

# =============================================================================
# TESTS
# =============================================================================

@pytest.mark.parametrize('batch_size', [1, 5])
@pytest.mark.parametrize('n_nodes', [1, 3])
@pytest.mark.parametrize('autoregressive_mask', [False, True])
def test_get_all_edges(batch_size, n_nodes, autoregressive_mask):
    """Test that get_all_edges determine all the masked edges of the graph."""
    # Create the mask.
    if autoregressive_mask is True:
        degrees_nodes = torch.arange(0, n_nodes)
        mask = create_autoregressive_mask(degrees_nodes, degrees_nodes)
    else:
        mask = None

    # Compute all masked edges.
    edges = get_all_edges(batch_size, n_nodes, mask)

    # Check that no mask and full mask are the same thing.
    if mask is None:
        mask = -(torch.eye(n_nodes)-1)
        edges2 = get_all_edges(batch_size, n_nodes, mask=mask)
        assert torch.all(edges == edges2)

    # Check that we have the expected number of edges.
    n_edges = int(mask.sum())
    assert edges.shape == (2, n_edges * batch_size)

    # Check that the edges are correct.
    for edge_src, edge_dst in edges.t():
        batch_idx  = edge_src.tolist() // n_nodes
        assert batch_idx < batch_size

        edge_src = edge_src - batch_idx * n_nodes
        edge_dst = edge_dst - batch_idx * n_nodes

        assert mask[edge_src, edge_dst] == 1.0

    # Check that fix_edges_batch_size correctly fixes the number of batches.
    new_batch_size = 2
    expected_edges = get_all_edges(new_batch_size, n_nodes, mask)
    fixed_edges = fix_edges_batch_size(edges, new_batch_size, n_edges, n_nodes)
    assert fixed_edges.shape == (2, new_batch_size*n_edges)
    assert torch.all(expected_edges == fixed_edges)

def test_compute_edge_distances():
    """Test compute_edge_distances() function."""
    batch_size = 2
    n_particles = 3

    # We compute distances only for a subset of the possible edges.
    edges = torch.tensor([
        [0, 0, 2, 3, 3, 5],
        [1, 2, 1, 4, 5, 4],
    ])
    n_edges = len(edges[0])

    # Create random (but reproducible) input and compute distances with torch.
    generator = torch.Generator()
    generator.manual_seed(0)
    x = torch.randn(batch_size, n_particles, 3, generator=generator)
    ref_distances = torch.cdist(x, x)

    # Compute the distances and direction with our implementation.
    x_batch_fmt = x.view(batch_size*n_particles, 3)
    distances, directions = compute_edge_distances(
        x_batch_fmt, edges, normalize_directions=True)
    assert distances.shape == (n_edges,)
    assert directions.shape == (n_edges, 3)

    # Compare distances with pytorch-computed reference.
    for edge_idx, (src, dest) in enumerate(edges.t()):
        batch_idx = int(src) // n_particles
        ref_dest, ref_src = dest % n_particles, src % n_particles
        ref_dist = ref_distances[batch_idx, ref_src, ref_dest]
        assert torch.allclose(ref_dist, distances[edge_idx])

        ref_dir = x[batch_idx, ref_dest] - x[batch_idx, ref_src]
        ref_dir = torch.nn.functional.normalize(ref_dir.unsqueeze(0))[0]
        assert torch.allclose(ref_dir, directions[edge_idx])

    # Check normalization and inverse direction.
    distances2, directions2 = compute_edge_distances(
        x_batch_fmt, edges, normalize_directions=False, inverse_directions=True)
    directions2 = directions2 / distances2.unsqueeze(-1)
    assert torch.allclose(directions, -directions2)

def test_prune_long_edges():
    """Test that prune_long_edges() discards edges with distance greater than the cutoff."""
    r_cutoff = 1.0

    # We compute distances only for a subset of the possible edges.
    edges = torch.tensor([
        [0, 0, 2, 3, 3, 5],
        [1, 2, 1, 4, 5, 4],
    ])

    # Create an input.
    x = torch.tensor([
        # In the first batch sample, consecutive particles are within the
        # cutoff, but non consecutive ones aren't.
        [0, 0, 0.0],
        [0, 0, r_cutoff],
        [0, 0, 2*r_cutoff],
        # In the second batch sample only the first and third atoms are within
        # the cutoff.
        [0.0, 0, 0],
        [5*r_cutoff, 0, 0],
        [r_cutoff, 0, 0],
    ])

    # We expect edges[expected_edges_indices[i]] to be NOT pruned.
    expected_edges_indices = torch.tensor([0, 2, 4])
    n_expected_edges = len(expected_edges_indices)

    # Compute distances and prune them.
    distances, directions = compute_edge_distances(x, edges)
    p_edges, p_distances, p_directions = prune_long_edges(r_cutoff, edges, distances, directions)

    # Check result of the pruning.
    for arr, ref_arr in zip([p_edges[0], p_edges[1], p_distances, p_directions],
                            [edges[0], edges[1], distances, directions]):
        assert arr.shape[0] == n_expected_edges
        assert torch.allclose(arr, ref_arr[expected_edges_indices])

def test_unsorted_segment_sum():
    """Test the unsorted_segment_sum function."""
    data = torch.tensor([
        [1.0, 1, 1],
        [2.0, 2, 2],
        [3.0, 3, 3],
        [4.0, 4, 4],
        [5.0, 5, 5],
    ])
    segment_ids = torch.tensor([0, 1, 0, 2, 2])
    n_segments = 3

    expected_result = torch.tensor([
        [4.0, 4, 4],
        [2.0, 2, 2],
        [9.0, 9, 9],
    ])

    result = unsorted_segment_sum(data, segment_ids, n_segments)
    assert torch.all(result == expected_result)


from django.shortcuts import render, redirect, get_object_or_404
from django.contrib.auth.decorators import login_required
from .models import Club, ClubMembership, ClubMessage
from .forms import ClubForm, ClubMessageForm
from django.contrib import messages as django_messages

def clubs(request):
    return render(request, 'clubs/club_list.html',{'club': clubs})

from django.urls import reverse

def club_detail(request):
    return render(request, "clubs/club_detail.html")
@login_required
def create_club_view(request):
    if request.method == 'POST':
        form = ClubForm(request.POST, request.FILES)
        if form.is_valid():
            club = form.save(commit=False)
            club.save()
            membership = ClubMembership(user=request.user, club=club)
            membership.save()
            return redirect('/clubs/club_detail', club_id=club.id)
    else:
        form = ClubForm()
    return render(request, 'clubs/create_club.html', {'form': form})

@login_required
def join_club_view(request, club_id):
    club = get_object_or_404(Club, id=club_id)
    if club.members.filter(id=request.user.id).exists():
        django_messages.warning(request, 'You are already a member of this club.')
    else:
        membership = ClubMembership(user=request.user, club=club)
        membership.save()
        django_messages.success(request, 'You have successfully joined the club.')
    return redirect('club_detail', club_id=club_id)

@login_required
def club_list(request):
    clubs = Club.objects.all()
    return render(request, "clubs/club_list.html", {'clubs': clubs})


import pandas as pd
import pickle
from pathlib import Path
from sklearn.model_selection import StratifiedKFold
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn.metrics import f1_score

if not(Path(f"dataset/cvss_v2.csv").is_file()) or not(Path(f"dataset/cvss_v3.csv").is_file()):
  print('Carregando Banco de Dados . . .')
  ds_completo = pd.read_csv("dataset/cvss.csv", delimiter= '¨', engine = 'python')
  ds_completo.drop(['CVE_DESC', 'CVSS2_AV', 'CVSS2_AC', 'CVSS2_AU', 'CVSS2_C', 'CVSS2_I', 'CVSS2_A', 'CVSS3_AV', 'CVSS3_AC', 'CVSS3_PR', 'CVSS3_UI', 'CVSS3_S', 'CVSS3_C', 'CVSS3_I', 'CVSS3_A'], axis='columns', inplace=True)

if Path(f"dataset/cvss_v2.csv").is_file():
  print('Carregando Banco de Dados CVSS v2 . . .')
  ds_cvss_v2 = pd.read_csv("dataset/cvss_v2.csv", delimiter= '¨', engine = 'python')
else:
  print('Filtrando CVSS v2 . . .')
  ds_cvss_v2 = ds_completo[(ds_completo.CVSS2_C_AV!=-1) & (ds_completo.CVSS2_C_AC!=-1) & (ds_completo.CVSS2_C_AU!=-1) & (ds_completo.CVSS2_C_C!=-1) & (ds_completo.CVSS2_C_I!=-1) & (ds_completo.CVSS2_C_A!=-1)].copy(deep=True)
  ds_cvss_v2.drop(['CVSS3_C_AV', 'CVSS3_C_AC', 'CVSS3_C_PR', 'CVSS3_C_UI', 'CVSS3_C_S', 'CVSS3_C_C', 'CVSS3_C_I', 'CVSS3_C_A'], axis='columns', inplace=True)
  print('Salvando Banco de Dados CVSS v2 . . .')
  ds_cvss_v2.to_csv(f"dataset/cvss_v2.csv", sep= '¨', index=False)

if Path(f"dataset/cvss_v3.csv").is_file():
  print('Carregando Banco de Dados CVSS v3 . . .')
  ds_cvss_v3 = pd.read_csv("dataset/cvss_v3.csv", delimiter= '¨', engine = 'python')
else:
  print('Filtrando CVSS v3 . . .')
  ds_cvss_v3 = ds_completo[(ds_completo.CVSS3_C_AV!=-1) & (ds_completo.CVSS3_C_AC!=-1) & (ds_completo.CVSS3_C_PR!=-1) & (ds_completo.CVSS3_C_UI!=-1) & (ds_completo.CVSS3_C_S!=-1) & (ds_completo.CVSS3_C_C!=-1) & (ds_completo.CVSS3_C_I!=-1) & (ds_completo.CVSS3_C_A!=-1)].copy(deep=True)
  ds_cvss_v3.drop(['CVSS2_C_AV', 'CVSS2_C_AC', 'CVSS2_C_AU', 'CVSS2_C_C', 'CVSS2_C_I', 'CVSS2_C_A'], axis='columns', inplace=True)
  print('Salvando Banco de Dados CVSS v3 . . .')
  ds_cvss_v3.to_csv(f"dataset/cvss_v3.csv", sep= '¨', index=False)

versoes = {'CVSS2': {'metricas': ['AV', 'AC', 'AU', 'C', 'I', 'A'],
                     'dataset': ds_cvss_v2
                    },
           'CVSS3': {'metricas': ['AV', 'AC', 'PR', 'UI', 'S', 'C', 'I', 'A'],
                     'dataset': ds_cvss_v3
                    }
          }

COLUNAS_R_GERAL = ['VERSAO',
                   'METRICA',
                   'FOLD',
                   'AMOSTRAS_TRAIN',
                   'AMOSTRAS_TEST',
                   'ACCURACY',
                   'PRECISION_MACRO',
                   'PRECISION_MICRO',
                   'PRECISION_WEIGHTED',
                   'RECALL_MACRO',
                   'RECALL_MICRO',
                   'RECALL_WEIGHTED',
                   'F1_MACRO',
                   'F1_MICRO',
                   'F1_WEIGHTED'
                  ]
resumo_r_geral = pd.DataFrame(columns=COLUNAS_R_GERAL)
n_splits = 5

for versao in versoes:
  for metrica in versoes[versao]['metricas']:
    print('Processando a métrica ' + metrica + ' do ' + versao)

    nm_col = versao + '_C_' + metrica

    if Path(f"modelos/train_indexes_{nm_col}.var").is_file() and Path(f"modelos/test_indexes_{nm_col}.var").is_file():
      print(f'Carregando índices de treinamento e teste {versao} {metrica}')
      train = pickle.load(open(f"modelos/train_indexes_{nm_col}.var",'rb'))
      test = pickle.load(open(f"modelos/test_indexes_{nm_col}.var",'rb'))
    else:
      print(f'Criando índices de treinamento e teste {versao} {metrica}')
      skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1)
      train = []
      test = []
      for train_index, test_index in skf.split(versoes[versao]['dataset'][nm_col], versoes[versao]['dataset'][nm_col]):
        train.append(train_index)
        test.append(test_index)
      print(f'Salvando índices de treinamento e teste {versao} {metrica}')

    t_a_s = 0
    t_ps_macro = 0
    t_ps_micro = 0
    t_ps_weighted = 0
    t_r_macro = 0
    t_r_micro = 0
    t_r_weighted = 0
    t_f1_macro = 0
    t_f1_micro = 0
    t_f1_weighted = 0
    for k in range(0, len(train)):
      if Path(f"modelos/vectorizer_{nm_col}_k{k}.tfidf").is_file() and Path(f"modelos/x_{nm_col}_k{k}.tfidf").is_file():
        print(f"Carregando vectorizer k{k} e X k{k} do {versao} {metrica}")
        vectorizer = pickle.load(open(f"modelos/vectorizer_{nm_col}_k{k}.tfidf",'rb'))
        X = pickle.load(open(f"modelos/x_{nm_col}_k{k}.tfidf",'rb'))
      else:
        # Criar pacote de termos do modelo (treinamento)
        print(f"Criando vectorizer k{k} e X k{k} do {versao} {metrica}")
        vectorizer = TfidfVectorizer(sublinear_tf=True, min_df=2, norm='l2', ngram_range = (1,3), max_features=5000)
        X = vectorizer.fit_transform([versoes[versao]['dataset']['CVE_DESC_NLP'].to_numpy()[indice] for indice in train[k]]).toarray()

      if not Path(f"modelos/classifier_{nm_col}_{k}.model").is_file():
        print(f"Treinando modelo k = {k} do {versao} {metrica}")
        classifier = LogisticRegression(random_state=0, verbose=True).fit(X, versoes[versao]['dataset'][nm_col].to_numpy()[train[k]])
      else:
        print(f"Carregando modelo k = {k} do {versao} {metrica}")
        classifier = pickle.load(open(f"modelos/classifier_{nm_col}_{k}.model",'rb'))

      print(f"Classificando k = {k} do {versao} {metrica}")
      y_pred = classifier.predict(vectorizer.transform([versoes[versao]['dataset']['CVE_DESC_NLP'].to_numpy()[indice] for indice in test[k]]))
      y_true = versoes[versao]['dataset'][nm_col].to_numpy()[test[k]]
      # Accuracy
      a_s = accuracy_score(y_true, y_pred)
      t_a_s += a_s
      # Precision (Macro)
      ps_macro = precision_score(y_true, y_pred, average='macro')
      t_ps_macro += ps_macro
      # Precision (Micro)
      ps_micro = precision_score(y_true, y_pred, average='micro')
      t_ps_micro += ps_micro
      # Precision (Weighted)
      ps_weighted = precision_score(y_true, y_pred, average='weighted')
      t_ps_weighted += ps_weighted
      # Recall (Macro)
      r_macro = recall_score(y_true, y_pred, average='macro')
      t_r_macro += r_macro
      # Recall (Micro)
      r_micro = recall_score(y_true, y_pred, average='micro')
      t_r_micro += r_micro
      # Recall (Weighted)
      r_weighted = recall_score(y_true, y_pred, average='weighted')
      t_r_weighted += r_weighted
      # F1 (Macro)
      f1_macro = recall_score(y_true, y_pred, average='macro')
      t_f1_macro += f1_macro
      # F1 (Micro)
      f1_micro = recall_score(y_true, y_pred, average='micro')
      t_f1_micro += f1_micro
      # F1 (Weighted)
      f1_weighted = recall_score(y_true, y_pred, average='weighted')
      t_f1_weighted += f1_weighted

      resumo_r_geral.loc[len(resumo_r_geral.index)] = [versao,
                                                       metrica,
                                                       f'F{k:02}',
                                                       len(train[k]),
                                                       len(test[k]),
                                                       a_s,
                                                       ps_macro,
                                                       ps_micro,
                                                       ps_weighted,
                                                       r_macro,
                                                       r_micro,
                                                       r_weighted,
                                                       f1_macro,
                                                       f1_micro,
                                                       f1_weighted]
      resumo_r_geral.to_csv(f"resultados/validacao_cruzada.csv", sep= '¨', index=False)
    resumo_r_geral.loc[len(resumo_r_geral.index)] = [versao,
                                                     metrica,
                                                     'MEDIA',
                                                     0,
                                                     0,
                                                     t_a_s / n_splits,
                                                     t_ps_macro / n_splits,
                                                     t_ps_micro / n_splits,
                                                     t_ps_weighted / n_splits,
                                                     t_r_macro / n_splits,
                                                     t_r_micro / n_splits,
                                                     t_r_weighted / n_splits,
                                                     t_f1_macro / n_splits,
                                                     t_f1_micro / n_splits,
                                                     t_f1_weighted / n_splits]
    resumo_r_geral.to_csv(f"resultados/validacao_cruzada.csv", sep= '¨', index=False)


import configparser
import re
import logging
import os

import pandas as pd
from pandas.errors import EmptyDataError

INPSECTED_ARTICLES_OUTPUT_DIR = 'articles_processed'
LOCATIONS_KEYWORDS = 'keywords'  # location of keywords (victims, infrastructures)
LOG_DIR = 'logs'

def get_config(config_file):
    config = configparser.ConfigParser()
    config.read(config_file)
    return config['main']

def get_keywords(config_file):
    config = configparser.ConfigParser()
    config.read(config_file)
    return config['keywords']

def get_scraped_article_output_dir(config):
    return 'Articles_{keyword}_{country}'.format(
        keyword=config['keyword'], country=config['country'])

def get_articles_summary_output_filename(config):
    return 'articles_summary_{keyword}_{country}.csv'.format(
        keyword=config['keyword'], country=config['country'])

def get_inspected_articles_output_filename(config):
    return 'articles_all_topical_{keyword}_{country}.csv'.format(
        keyword=config['keyword'], country=config['country'])

def get_pattern_entity(loc_string, target):
    pattern_entity = '({loc_string}(.*){target}|{target}(.*){loc_string})'
    pattern_entity = pattern_entity.format(loc_string=re.escape(loc_string), target=re.escape(target))
    return re.compile(str(pattern_entity), re.IGNORECASE)

def read_keyword_csv(filename):
    try:
        return pd.read_csv(os.path.join(LOCATIONS_KEYWORDS,  filename),
                           header=None, encoding='latin-1')[0].tolist()
    except EmptyDataError:
        return []

def set_log_level(is_debug_mode=False, log_filename=None):
    logging.basicConfig()
    log_level = 'DEBUG' if is_debug_mode else 'INFO'
    root_logger = logging.getLogger()
    root_logger.setLevel(log_level.upper())

    if log_filename is not None:
        if not os.path.isdir(LOG_DIR):
            os.mkdir(LOG_DIR)
        file_handler = logging.FileHandler(os.path.join(LOG_DIR, log_filename))
        formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
        file_handler.setFormatter(formatter)
        file_handler.setLevel(log_level.upper())
        root_logger.addHandler(file_handler)


from telethon.tl.functions.photos import GetUserPhotosRequest
from telethon.tl.functions.users import GetFullUserRequest
from telethon.tl.types import MessageEntityMentionName
from telethon.utils import get_input_location
import flag
import html, os
from countryinfo import CountryInfo
from GroupMenter import telethn as borg
from GroupMenter.events import register

@register(pattern="^/country (.*)")
async def msg(event):
    if event.fwd_from:
        return
    input_str = event.pattern_match.group(1)
    lol = input_str
    country = CountryInfo(lol)
    try:
	    a = country.info()
    except:
	    await event.reply("Ülke Şu anda Kullanılamıyor")
    name = a.get("name")
    bb= a.get("altSpellings")
    hu = ''
    for p in bb:
    	hu += p+",  "

    area = a.get("area")
    borders = ""
    hell = a.get("borders")
    for fk in hell:
	    borders += fk+",  "

    call = ""
    WhAt = a.get("callingCodes")
    for what in WhAt:
	    call+= what+"  "

    capital = a.get("capital")
    currencies = ""
    fker = a.get("currencies")
    for FKer in fker:
	    currencies += FKer+",  "

    HmM = a.get("demonym")
    geo = a.get("geoJSON")
    pablo = geo.get("features")
    Pablo = pablo[0]
    PAblo = Pablo.get("geometry")
    EsCoBaR= PAblo.get("type")
    iso = ""
    iSo = a.get("ISO")
    for hitler in iSo:
      po = iSo.get(hitler)
      iso += po+",  "
    fla = iSo.get("alpha2")
    nox = fla.upper()
    okie = flag.flag(nox)

    languages = a.get("languages")
    lMAO=""
    for lmao in languages:
	    lMAO += lmao+",  "

    nonive = a.get("nativeName")
    waste = a.get("population")
    reg = a.get("region")
    sub = a.get("subregion")
    tik = a.get("timezones")
    tom =""
    for jerry in tik:
	    tom+=jerry+",   "

    GOT = a.get("tld")
    lanester = ""
    for targaryen in GOT:
	    lanester+=targaryen+",   "

    wiki = a.get("wiki")

    caption = f"""Bilgi Başarıyla Toplandı

Ülke Adı:- {name}
Alternatif Yazımlar:- {hu}
Ülke Alanı:- {area} kilometrekare
Sınırlar:- {borders}
Arama Kodları:- {call}
Ülkenin Başkenti:- {capital}
Ülkenin para birimi:- {currencies}
Ülkenin Bayrağı:- {okie}
Demonym:- {HmM}
Ülke Türü:- {EsCoBaR}
ISO Adları:- {iso}
Diller:- {lMAO}
Yerel Ad:- {nonive}
nüfus:- {waste}
Bölge:- {reg}
Alt Bölge:- {sub}
Saat Dilimleri:- {tom}
Üst Düzey Etki Alanı:- {lanester}
Wikipedia:- {wiki}

Emilia Tarafından Toplandı.
"""

    await borg.send_message(
        event.chat_id,
        caption,
        parse_mode="HTML",
    )

__help__ = """
Bir ülke hakkında bilgi vereceğim

  ❍ `/country <ülke adı>`*:* Verilen ülke hakkında bilgi toplama
"""

__mod_name__ = "ÜLKE"


#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
@Project ：LeetCodePthonVersion
@File ：781. 森林中的兔子.py
@Author ：HuntingGame
@Date ：2022-11-08 9:30
C'est la vie!!! enjoy ur day :D
'''
import collections
from typing import List

class Solution:
    def numRabbits(self, answers: List[int]) -> int:
        myset = set()
        ans = 0

        cnts = collections.Counter(answers)
        for key,val in cnts.items():
            if key == 0:
                ans +=val
                continue
            if val <=key+1: # key=1,val=2,val=1
                ans +=key + 1
            else:# key=1,val = 3 很明显需要4个兔子起码
                if val %(key+1) == 0:
                    ans += (key + 1) * (val // (key+1))

                else:

                    ans += (key + 1) * (val // (key + 1)) + (key + 1)
        return ans

print(Solution().numRabbits([0,1,0,2,0,1,0,2,1,1]))

贪心/781. 森林中的兔子.py


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Implement the crnn model mentioned in An End-to-End Trainable Neural Network for Image-based Sequence
Recognition and Its Application to Scene Text Recognition paper
"""
import numpy as np
import tensorflow as tf
from tensorflow.contrib import layers as tflayers
from tensorflow.contrib import rnn
slim = tf.contrib.slim
import sys
import mobilenet_v2
import telenet_basenet

class TeleNet(telenet_basenet.TeleBaseModel):
    """
        Implement the crnn model for squence recognition
    """
    def __init__(self, phase, hidden_nums, layers_nums, seq_length, num_classes,use_bn=False):
        """

        :param phase:
        """
        super(TeleNet, self).__init__()  # call  TeleBaseModel __int__
        self.__phase = phase
        self.__hidden_nums = hidden_nums
        self.__layers_nums = layers_nums
        self.__seq_length = seq_length
        if False:
            self.__num_classes = num_classes
        else:
            self.__num_classes=1   #  use  sigmoid
        self.__use_bn=use_bn
        return

    @property
    def phase(self):
        """

        :return:
        """
        return self.__phase

    @phase.setter
    def phase(self, value):
        """

        :param value:
        :return:
        """
        if not isinstance(value, str):
            raise TypeError('value should be a str \'Test\' or \'Train\'')
        if value.lower() not in ['test', 'train']:
            raise ValueError('value should be a str \'Test\' or \'Train\'')
        self.__phase = value.lower()
        return

    def __conv_stage(self, inputdata, out_dims, name=None):
        """
        Traditional conv stage in VGG format
        :param inputdata:
        :param out_dims:
        :return:
        """
        conv = self.conv2d(inputdata=inputdata, out_channel=out_dims, kernel_size=3, stride=1, use_bias=False, name=name)
        relu = self.relu(inputdata=conv)
        max_pool = self.maxpooling(inputdata=relu, kernel_size=2, stride=2)
        return max_pool

    def __feature_sequence_extraction(self, inputdata):
        """
        Implement the 2.1 Part Feature Sequence Extraction
        :param inputdata: eg. batch*32*100*3 NHWC format
        :return:
        """
        conv1 = self.__conv_stage(inputdata=inputdata, out_dims=64, name='conv1')  # batch*16*50*64
        conv2 = self.__conv_stage(inputdata=conv1, out_dims=128, name='conv2')  # batch*8*25*128
        conv3 = self.conv2d(inputdata=conv2, out_channel=256, kernel_size=3, stride=1, use_bias=False, name='conv3')  # batch*8*25*256
        relu3 = self.relu(conv3) # batch*8*25*256
        conv4 = self.conv2d(inputdata=relu3, out_channel=256, kernel_size=3, stride=1, use_bias=False, name='conv4')  # batch*8*25*256
        relu4 = self.relu(conv4)  # batch*8*25*256
        max_pool4 = self.maxpooling(inputdata=relu4, kernel_size=[2, 1], stride=[2, 1], padding='VALID')  # batch*4*25*256
        conv5 = self.conv2d(inputdata=max_pool4, out_channel=512, kernel_size=3, stride=1, use_bias=False, name='conv5')  # batch*4*25*512
        relu5 = self.relu(conv5)  # batch*4*25*512
        if self.phase.lower() == 'train':
            bn5 = self.layerbn(inputdata=relu5, is_training=True)
        else:
            bn5 = self.layerbn(inputdata=relu5, is_training=False)  # batch*4*25*512
        conv6 = self.conv2d(inputdata=bn5, out_channel=512, kernel_size=3, stride=1, use_bias=False, name='conv6')  # batch*4*25*512
        relu6 = self.relu(conv6)  # batch*4*25*512
        if self.phase.lower() == 'train':
            bn6 = self.layerbn(inputdata=relu6, is_training=True)
        else:
            bn6 = self.layerbn(inputdata=relu6, is_training=False)  # batch*4*25*512
        max_pool6 = self.maxpooling(inputdata=bn6, kernel_size=[2, 1], stride=[2, 1])  # batch*2*25*512
        conv7 = self.conv2d(inputdata=max_pool6, out_channel=512, kernel_size=2, stride=[2, 1], use_bias=False, name='conv7')  # batch*1*25*512
        relu7 = self.relu(conv7)  # batch*1*25*512
        return relu7
    def __mobilenetV2feature_sequence_extraction(self, inputdata,reuseflag):
        """
        use inception V3 model
        :param inputdata: eg. batch*128*128*1
        :return:
        """
        # arg_scope = inception_v3_arg_scope()
        # with slim.arg_scope(arg_scope) as scope:
        #     shape=inputdata.get_shape().as_list()
        #     # if self.phase.lower() == 'train':
        #     #     logits, end_points = inception_v3(inputdata, is_training=True,reuse=reuseflag,num_classes=1001)
        #     # else:
        #     #     logits, end_points = inception_v3(inputdata, is_training=False, reuse=reuseflag, num_classes=1001)
        #     logits, end_points = inception_v3(inputdata, is_training=False, reuse=reuseflag, num_classes=1001)
        #     # fc_out=slim.fully_connected(logits,512,scope='fc512')
        #     return end_points['PreLogits']   # batch ,1,1,2048

        # with tf.contrib.slim.arg_scope(mobilenet_v2.training_scope(is_training=False)):
        #     logits, endpoints = mobilenet_v2.mobilenet(inputdata, depth_multiplier=1.0,reuse=reuseflag)
        #     return endpoints['global_pool']  # batch ,1,1,1280

        if self.__use_bn:
            with tf.contrib.slim.arg_scope(mobilenet_v2.training_scope(is_training=True)):
                logits, endpoints = mobilenet_v2.mobilenet(inputdata, depth_multiplier=1.0, reuse=reuseflag)
                return endpoints['global_pool']  # batch ,1,1,1280
        else:
            with tf.contrib.slim.arg_scope(mobilenet_v2.training_scope(is_training=False)):
                logits, endpoints = mobilenet_v2.mobilenet(inputdata, depth_multiplier=1.0, reuse=reuseflag)
                return endpoints['global_pool']  # batch ,1,1,1280

    def __downfc(self, input_data,reuseflag):
        """
        Implement the map to sequence part of the network mainly used to convert the cnn feature map to sequence used in
        later stacked lstm layers
        :param inputdata:
        :return:
        """
        with tf.variable_scope('Downfc',reuse=reuseflag):
            if self.phase.lower() == 'train':
                input_data = self.dropout(inputdata=input_data, keep_prob=0.8)
            conv1 = tf.layers.conv2d(inputs=input_data, filters=640, kernel_size=[1, 1],padding="same", activation=tf.nn.relu)
            shape = conv1.get_shape().as_list()[1:]
            if None not in shape:
                conv1 = tf.reshape(conv1, [-1, int(np.prod(shape))])
            else:
                conv1 = tf.reshape(conv1, tf.stack([tf.shape(conv1)[0], -1]))
            fc_out1 = tf.layers.dense(conv1, 320,activation=tf.nn.relu)
            if self.phase.lower() == 'train':
                fc_out1 = tf.layers.dropout(inputs=fc_out1, rate=0.2, training=True)
            fc_out = tf.layers.dense(fc_out1, self.__hidden_nums,activation=tf.nn.relu)
            #  原来只有一层全连接
            # fc_out=self.fullyconnect(inputdata=input_data,out_dim=self.__hidden_nums)
            outdata=tf.expand_dims(fc_out, 1)
            return outdata
        # return self.squeeze(inputdata=inputdata, axis=1)

    def __sequence_label(self, inputdata):
        """
        Implement the sequence label part of the network
        :param inputdata:
        :return:
        """
        with tf.variable_scope('LSTMLayers'):
            # construct stack lstm rcnn layer
            # # forward lstm cell
            # fw_cell_list = [rnn.BasicLSTMCell(nh, forget_bias=1.0) for nh in [self.__hidden_nums, self.__hidden_nums]]
            # # Backward direction cells
            # bw_cell_list = [rnn.BasicLSTMCell(nh, forget_bias=1.0) for nh in [self.__hidden_nums, self.__hidden_nums]]
            list_f = list(self.__hidden_nums*np.ones((self.__layers_nums)))
            list_b = list(self.__hidden_nums*np.ones((self.__layers_nums)))
            fw_cell_list = [rnn.BasicLSTMCell(nh, forget_bias=1.0) for nh in list_f]
            # Backward direction cells
            bw_cell_list = [rnn.BasicLSTMCell(nh, forget_bias=1.0) for nh in list_b]

            stack_lstm_layer, _, _ = rnn.stack_bidirectional_dynamic_rnn(fw_cell_list, bw_cell_list, inputdata,dtype=tf.float32)

            if self.phase.lower() == 'train':
                stack_lstm_layer = self.dropout(inputdata=stack_lstm_layer, keep_prob=0.8)

            # # 用处不大
            raw_pred = tf.layers.dense(stack_lstm_layer[:, -1, :], self.__num_classes*2,activation=tf.nn.relu)  # output based on the last output step
            # logits = tf.nn.softmax(raw_pred)
            # # 用处不大 作为中间变量
            #raw_pred = tf.layers.dense(stack_lstm_layer[:, -1, :],100)  # output based on the last output step
        return raw_pred  # stack_lstm_layer[:, -1, :]

    def __lr_label(self, candidata,browsedseq,dnnout):   # batch ,50   ;  batch, 100 ; batch , 50
        """
        Implement the sequence label part of the network
        :param inputdata:
        :return:
        """
        with tf.variable_scope('CTRLayers'):
            all_f=tf.concat([candidata,browsedseq,dnnout],axis=1)
            #  增加 fc 层
            if True:
                fc_pred=tf.layers.dense(all_f, 256,activation=tf.nn.relu)
                raw_pred = tf.layers.dense(fc_pred, self.__num_classes)
            else:
                raw_pred = tf.layers.dense(all_f, self.__num_classes)  # output based on the last output step 默认没有激活函数
            #all_f=tf.concat([candidata,browsedseq,dnnout],axis=1)
            #raw_pred = tf.layers.dense(all_f, self.__num_classes)  # output based on the last output step 默认没有激活函数
            logits = tf.nn.sigmoid(raw_pred)

        return logits , raw_pred

    def __dnnseq(self, inputdata):
        """
        Implement the sequence label part of the network
        :param inputdata:    batch *1 * 512
        :return:
        """
        with tf.variable_scope('DNNLayers'):
            shape = inputdata.get_shape().as_list()   # batch , per user browse num, lstm hidden num
            inputdata_new=tf.reshape(inputdata,shape=[-1,1,shape[1]*shape[2]]) #

            if self.phase.lower() == 'train':
                inputdata_new = self.dropout(inputdata=inputdata_new, keep_prob=0.8)
            dnn1 = tf.layers.dense(inputdata_new, 512,activation=tf.nn.relu)  # output based on the last output step
            if self.phase.lower() == 'train':
                dnn1 = self.dropout(inputdata=dnn1, keep_prob=0.8)
            dnn2 = tf.layers.dense(dnn1, 512,activation=tf.nn.relu)  # output based on the last output step
            if self.phase.lower() == 'train':
                dnn2 = self.dropout(inputdata=dnn2, keep_prob=0.8)
            dnn3 = tf.layers.dense(dnn2, 128,activation=tf.nn.relu)  # output based on the last output step
            dnn3 = tf.squeeze(dnn3, [1])
        return dnn3

    def build_telenet(self, candidata,browsedseq):
        """

        :param inputdata:
        :return:
        """
        #1、 candidate image
        candidata_f = self.__mobilenetV2feature_sequence_extraction(inputdata=candidata,reuseflag=False)
        c_f = self.__downfc(candidata_f,False)  # batch ,1,50
        c_f = tf.squeeze(c_f,[1])
        if True:
            #2、 user browsedseq image
            # first apply the cnn feature extraction stage
            shape = browsedseq.get_shape().as_list()  # batch, per user browsed num ,h,w,c
            Timestep=self.__seq_length
            browsed_f = self.__mobilenetV2feature_sequence_extraction(inputdata=browsedseq[:, 0, :, :, :], reuseflag=True)
            # second apply the map to sequence stage
            sequence = self.__downfc(browsed_f, True)
            for i in range(shape[1]-1):
                browsed_f = self.__mobilenetV2feature_sequence_extraction(inputdata=browsedseq[:,i+1,:,:,:],reuseflag=True)
                # second apply the map to sequence stage
                sequence = tf.concat([self.__downfc(browsed_f,True),sequence],axis=1)
            # third apply the sequence label stage
            blstm_out = self.__sequence_label(inputdata=sequence)
            #3、 dnn out
            dnn_out=self.__dnnseq(sequence)
            crt_out,logits_before=self.__lr_label(c_f,blstm_out,dnn_out)
            return crt_out, logits_before
        else:   #  浏览记录test
            raw_pred = tf.layers.dense(c_f, self.__num_classes)  # output based on the last output step
            logits = tf.nn.sigmoid(raw_pred,name='test_sigmoid')
            return logits,raw_pred


#!/usr/bin/env python3
"""
based on encoderlayer of:
https://www.tensorflow.org/tutorials/text/transformer#encoder_layer
Encoder Block
remember to use tensorflow 1.15 or higher
"""

import tensorflow as tf
MultiHeadAttention = __import__('6-multihead_attention').MultiHeadAttention

class EncoderBlock(tf.keras.layers.Layer):
    """
    class Encoder block
    """
    def __init__(self, dm, h, hidden, drop_rate=0.1):
        """
        * dropout1 - the first dropout layer
        * dropout2 - the second dropout layer
        """
        super().__init__()
        self.mha = MultiHeadAttention(dm, h)
        self.dense_hidden = tf.keras.layers.Dense(units=hidden,
                                                  activation='relu')
        self.dense_output = tf.keras.layers.Dense(units=dm)
        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
        self.dropout1 = tf.keras.layers.Dropout(drop_rate)
        self.dropout2 = tf.keras.layers.Dropout(drop_rate)

    def call(self, x, training, mask=None):
        """
        Returns: a tensor of shape (batch, input_seq_len, dm) containing
        the block’s output
        """
        attn_output, _ = self.mha(x, x, x, mask)
        attn_output = self.dropout1(attn_output, training=training)
        out1 = self.layernorm1(x + attn_output)
        hidden_output = self.dense_hidden(out1)
        output_output = self.dense_output(hidden_output)
        ffn_output = self.dropout2(output_output, training=training)
        final_output = self.layernorm2(out1 + ffn_output)
        return final_output


# -*- coding:utf-8 -*-
# 数组中有一个数字出现的次数超过数组长度的一半，请找出这个数字。
# 例如输入一个长度为9的数组{1,2,3,2,2,2,5,4,2}。
# 由于数字2在数组中出现了5次，超过数组长度的一半，因此输出2。如果不存在则输出0。

class Solution:
    # 使用字典进行计数
    def MoreThanHalfNum_Solution(self, numbers):
        dict = {}
        for i in numbers:
            if i not in dict:
                dict[i] = 1
            else:
                dict[i] = dict[i] + 1
            if dict[i] > len(numbers) / 2:
                return i
        return 0

# 想象下面示例的情况：
# 1 1 1 1 1 1
# 2 3 4 5
class Solution1:
    # 使用不同数字进行对消的方法
    def MoreThanHalfNum_Solution(self, numbers):
        if numbers == []:
            return 0
        val, cnt = None, 0
        for num in numbers:
            if cnt == 0:
                val, cnt = num, 1
            elif val == num:
                cnt += 1
            else:
                cnt -= 1
        return val if numbers.count(val) * 2 > len(numbers) else 0

class Solution2:
    # 使用默认词典的方法
    def MoreThanHalfNum_Solution(self, numbers):
        length = len(numbers)
        d = {}
        d[1] = 2
        for num in numbers:
            d[num] = d.setdefault(num, 0) + 1
        for key in d.keys():
            if d[key] * 2 > length:
                return int(key)
        return 0

    def MoreThanHalfNum_Solution1(self, numbers):
        from collections import Counter
        count_dic = Counter(numbers)
        length = len(numbers)
        for k, v in count_dic.items():
            if v * 2 > length:
                return k
        return 0

if __name__ == '__main__':
    S = Solution1()
    print(S.MoreThanHalfNum_Solution([1, 2, 2]))

28.数组中出现次数超过一半的数字.py


import os
import json
import argparse
from tqdm import tqdm

def parse_arguments():
    parser = argparse.ArgumentParser(description='BDD100K to COCO format')
    parser.add_argument(
          "-l", "--label_dir",
          default="/path/to/bdd/label/",
          help="root directory of BDD label Json files",
    )
    parser.add_argument(
          "-s", "--save_path",
          default="/save/path",
          help="path to save coco formatted label file",
    )
    return parser.parse_args()

def bdd2coco_detection(id_dict, labeled_images, fn):

    images = list()
    annotations = list()

    counter = 0
    for i in tqdm(labeled_images):
        counter += 1
        image = dict()
        image['file_name'] = i['name']
        image['height'] = 720
        image['width'] = 1280

        image['id'] = counter

        empty_image = True

        for label in i['labels']:
            annotation = dict()
            if label['category'] in id_dict.keys():
                empty_image = False
                annotation["iscrowd"] = 0
                annotation["image_id"] = image['id']
                x1 = label['box2d']['x1']
                y1 = label['box2d']['y1']
                x2 = label['box2d']['x2']
                y2 = label['box2d']['y2']
                annotation['bbox'] = [x1, y1, x2-x1, y2-y1]
                annotation['area'] = float((x2 - x1) * (y2 - y1))
                annotation['category_id'] = id_dict[label['category']]
                annotation['ignore'] = 0
                annotation['id'] = label['id']
                annotation['segmentation'] = [[x1, y1, x1, y2, x2, y2, x2, y1]]
                annotations.append(annotation)

        if empty_image:
            continue

        images.append(image)

    attr_dict["images"] = images
    attr_dict["annotations"] = annotations
    attr_dict["type"] = "instances"

    print('saving...')
    json_string = json.dumps(attr_dict)
    with open(fn, "w") as file:
        file.write(json_string)

if __name__ == '__main__':

    args = parse_arguments()

    attr_dict = dict()
    attr_dict["categories"] = [
        {"supercategory": "none", "id": 1, "name": "person"},
        {"supercategory": "none", "id": 2, "name": "rider"},
        {"supercategory": "none", "id": 3, "name": "car"},
        {"supercategory": "none", "id": 4, "name": "bus"},
        {"supercategory": "none", "id": 5, "name": "truck"},
        {"supercategory": "none", "id": 6, "name": "bike"},
        {"supercategory": "none", "id": 7, "name": "motor"},
        {"supercategory": "none", "id": 8, "name": "traffic light"},
        {"supercategory": "none", "id": 9, "name": "traffic sign"},
        {"supercategory": "none", "id": 10, "name": "train"}
    ]

    attr_id_dict = {i['name']: i['id'] for i in attr_dict['categories']}

    # create BDD training set detections in COCO format
    print('Loading training set...')
    with open(os.path.join(args.label_dir,
                           'bdd100k_labels_images_train.json')) as f:
        train_labels = json.load(f)
    print('Converting training set...')

    out_fn = os.path.join(args.save_path,
                          'bdd100k_labels_images_det_coco_train.json')
    bdd2coco_detection(attr_id_dict, train_labels, out_fn)

    print('Loading validation set...')
    # create BDD validation set detections in COCO format
    with open(os.path.join(args.label_dir,
                           'bdd100k_labels_images_val.json')) as f:
        val_labels = json.load(f)
    print('Converting validation set...')

    out_fn = os.path.join(args.save_path,
                          'bdd100k_labels_images_det_coco_val.json')
    bdd2coco_detection(attr_id_dict, val_labels, out_fn)


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

data = pd.read_csv(r'C:\Users\manoj\OneDrive\Documents\Cornell University\ENMGT 5980 (Decision Framing and Analytics)\Final Project\FilteredAmesHousing.csv')

x = data['SalePrice'].values
y = data['Gr Liv Area'].values
y2 = data['Year Built'].values
y3 = data['Lot Area'].values
y4 = data['Bedroom AbvGr'].values

fig, ax = plt.subplots(2,2, figsize=(12,10))
fig.tight_layout(pad=4.0)
ax[0,0].scatter(x,y,s=2, color='red')

ax[0,0].set_title("Sales Price vs Living Area Space")
ax[0,0].set_xlabel("Sales Price")
ax[0,0].set_ylabel("Living Area Space")

ax[0,1].scatter(x,y2,s=2, color='red')

ax[0,1].set_title("Sales Price vs Year Built")
ax[0,1].set_xlabel("Sales Price")
ax[0,1].set_ylabel("Year Built")

ax[1,0].scatter(x,y3,s=2, color='red')

ax[1,0].set_title("Sales Price vs Lot Area")
ax[1,0].set_xlabel("Sales Price")
ax[1,0].set_ylabel("Lot Area")

ax[1,1].scatter(y4,x,s=2,color='red')

ax[1,1].set_title("Sales Price vs Bedrooms")
ax[1,1].set_xlabel("Bedrooms")
ax[1,1].set_ylabel("Sales Price")

plt.show()

"""
ax[2].scatter(x,y,s=2)

ax[2].set_title("Sales Price vs Year Built")
ax[2].set_xlabel("Sales Price")
ax[2].set_ylabel("Year Built")

plt.show()
"""


import numpy as np
import csv
import random
import xlrd
import itertools
import json
import os
from PIL import Image

def split(image_dict,split_ratio,tag:str='test'):
    """This function extract a new image dict from the given image dict by the split_ratio

    Args:
        image_dict (dict): {'class_label': [image_index1, image_index2, image_index3....]}
        split_ratio (float): eg.0.8,  the number of samples per class is 80% of that of the original image dict

    Returns:
        dict: image_dict
        dict: a dict records the state  and the index for each unique image from the original image_dict
    """
    keep_split_ratio = 1 - split_ratio
    keep_image_dict  = {}
    other_image_dict  = {}
    keys = sorted(list(image_dict.keys()))
    values = np.unique(list(itertools.chain.from_iterable(image_dict.values())))
    flag =  {ind:"undefine" for ind in values}
    for key in keys:
        samples_ind = image_dict[key]
        random.shuffle(samples_ind)
        sample_num = len(samples_ind)
        keep_image_dict[key] =[]
        other_image_dict[key] =[]
        # check if there are some sample id already in train/nontrain
        for ind in samples_ind:
            if flag[ind] =="undefine":
                if len(keep_image_dict[key])< int(sample_num*keep_split_ratio):
                    keep_image_dict[key].append(ind)
                    flag[ind] = "non"+tag
                else:
                    if len(other_image_dict[key])< (sample_num - int(sample_num*keep_split_ratio)):
                        other_image_dict[key].append(ind)
                        flag[ind] = tag
            elif flag[ind] == "non"+tag:
                if len(keep_image_dict[key])< int(sample_num*keep_split_ratio):
                    keep_image_dict[key].append(ind)
            else:
                if len(other_image_dict[key])< (sample_num - int(sample_num*keep_split_ratio)):
                    other_image_dict[key].append(ind)
    return keep_image_dict,flag

def read_csv(csv_filename,datapath):
    """reads a csv file and returns a list of file paths

    Args:
        csv_filename (str): file path, this file contains the image name and its multi-hot labels
        datapath (str): the source of dataset

    Returns:
        list: [image_path, multi-hot label]
    """
    file_list, file_label =[],[]
    with open(csv_filename) as csv_file:
        csv_reader = csv.reader(csv_file, delimiter=',')
        for row in csv_reader:
            file_list.append([datapath + str(row[0]),np.array(row[1:],dtype=int)])
    return file_list

def create_csv_split(csv_dir,datapath):
    """Split the dataset to train/val/test with ratio 50%/10%/40%
    Keep this ratio among classes
    Write the results to csv files

    Args:
        csv_dir (str): folder to store the resulted csv files
        datapath (str): eg. /scratch/CV4RS/Dataset/MLRSNet
    """
    category = {}
    category_path = datapath + '/Categories_names.xlsx'
    book = xlrd.open_workbook(category_path)
    sheet = book.sheet_by_index(1)
    for i in range(2,sheet.nrows):
        category_name = sheet.cell_value(rowx=i, colx=1)
        temp_label_name = np.unique(np.array([sheet.cell_value(rowx=i, colx=j).strip() for j in range(2,sheet.ncols) if sheet.cell_value(rowx=i, colx=j)!=""]))
        if "chapparral" in temp_label_name: temp_label_name[np.where(temp_label_name=="chapparral")]= "chaparral"
        category[category_name] = temp_label_name

    label_folder = datapath +'/labels/'
    image_folder  = datapath +'/Images/'
    image_list =[] # image path
    image_labels =[]

    for entry in os.listdir(label_folder):
        if entry.split('.')[-1] =="csv" :
            with open(label_folder + entry) as csv_file:
                csv_reader = csv.reader(csv_file, delimiter=',')
                label_names =next(csv_reader,None)[1:]
                if len(label_names)==60:
                    sort_ind = np.argsort(label_names)
                    for row in csv_reader:
                        image_path = image_folder + entry.split('.')[0] +'/'+row[0]
                        #image_list.append(image_path)
                        image_list.append('/Images/'+ entry.split('.')[0] +'/'+row[0])
                        temp = np.array(row[1:])
                        image_labels.append(temp[sort_ind])
                else:
                    print(entry)

    label_names = np.sort(label_names)
    # to record the label names and its id
    label_names_dict= {i:x for i,x in enumerate(label_names)}

    for key in category.keys():
        labels = np.array(category[key])
        label_ind = [str(np.where(label_names==item)[0][0]) for item in labels]
        category[key] = label_ind

    image_list = np.array(image_list)
    image_labels = np.array(image_labels,dtype=int)
    image_dict  = {i:np.where(image_labels[:,i]==1)[0] for i in range(len(label_names))}

    # split data into nontest/test 70%/30% balanced in class.
    temp_image_dict, flag_test =split(image_dict, 0.3,'test')
    # split train into train/val 40%/30% balanced in class
    _,flag_val =split(temp_image_dict, 0.43,'val')

    test   = [[image_list[ind]]+list(image_labels[ind,:]) for ind in sorted(list(flag_test.keys())) if flag_test[ind]=="test"]
    val  = [[image_list[ind]]+list(image_labels[ind,:]) for ind in sorted(list(flag_val.keys())) if flag_val[ind]=="val"]
    train   = [[image_list[ind]]+list(image_labels[ind,:]) for ind in sorted(list(flag_val.keys())) if flag_val[ind]=="nonval"]

    with open(csv_dir +'/label_name.json', 'w+') as label_file:
        json.dump(label_names_dict, label_file,separators=(",", ":"),allow_nan=False,indent=4)
    with open(csv_dir +'/category.json', 'w+') as category_file:
        json.dump(category, category_file,separators=(",", ":"),allow_nan=False,indent=4)
    with open(csv_dir +'/train.csv', 'w', newline='') as file:
        writer = csv.writer(file)
        writer.writerows(train)
    with open(csv_dir +'/test.csv', 'w', newline='') as file:
        writer = csv.writer(file)
        writer.writerows(test)
    with open(csv_dir +'/val.csv', 'w', newline='') as file:
        writer = csv.writer(file)
        writer.writerows(val)

def Give(datapath,dset_type):
    """Given a dataset path generate a list of image paths and multi-hot labels .

    Args:
        datapath (str): eg. /scratch/CV4RS/DatasetBigEarthNet
        dset_type (str): choose from {'train','val','test'}

    Returns:
        list: contains [image_path, multi-hot label]
    """
    csv_dir = os.path.dirname(__file__) + '/MLRSNet_split'
    # check the split train/test/val existed or not
    if not os.path.exists(csv_dir +'/train.csv'):
        create_csv_split(csv_dir,datapath)

    with open(csv_dir +'/category.json') as json_file:
        category = json.load(json_file)
    with open(csv_dir +'/label_name.json') as json_file:
        conversion= json.load(json_file)
    train_list = read_csv(csv_dir +'/train.csv',datapath)
    val_list = read_csv(csv_dir +'/val.csv',datapath)
    test_list= read_csv(csv_dir +'/test.csv',datapath)

    dsets = {'train': train_list , 'val': val_list , 'test': test_list}
    return dsets[dset_type],conversion


#%%
"""
Programa lectura DS18B20 en Raspberry PI.

-Funciona en RPi4.
- DS18B20 conectado en PIN 4 del GPIO.
- Protocolo 1-Wire (habilitarlo desde consola "sudo raspi-config")
- Resistencia de 10K entre Data y +VCC 3V3

Libreria Adafruit:
    https://learn.adafruit.com/adafruits-raspberry-pi-lesson-11-ds18b20-temperature-sensing

"""
#%%

import glob
import time
import datetime

#%%

base_dir = '/sys/bus/w1/devices/'
device_folder = glob.glob(base_dir + '28*')[0]
device_file = device_folder + '/w1_slave'

#%%

def print_temp():
    '''
    Pido la temperatura al sensor con read_temp, que a su vez llama a read_temp_raw.
    Si el sensor falla (por ejemplo no tiene tension) atrapo el error e imprimo "Error Sensor"
    '''
    try:
        horaMedicion = datetime.datetime.now()
        print(f'{str(horaMedicion.strftime("%Y%m%d_%H%M%S"))} Temp °C = {read_temp()}')
    except:
        print(f'{str(horaMedicion.strftime("%Y%m%d_%H%M%S"))} Temp °C = Error sensor')
    return

def read_temp_raw():
    f = open(device_file, 'r')
    lines = f.readlines()
    f.close()
    return lines

def read_temp():
    lines = read_temp_raw()
    while lines[0].strip()[-3:] != 'YES':
        time.sleep(0.2)
        lines = read_temp_raw()
    equals_pos = lines[1].find('t=')
    if equals_pos != -1:
        temp_string = lines[1][equals_pos+2:]
        temp_c = float(temp_string) / 1000.0
        return temp_c

print_temp() # corro la funcion que larga la medicion e imprime el valor de T en °C


from tqdm import tqdm
import numpy as np
from sarvam.colorful_logger import *
##  Data : http://www-nlp.stanford.edu/data/glove.840B.300d.zip

from nltk import word_tokenize
from nltk.corpus import stopwords
stop_words = stopwords.words('english')

# load the GloVe vectors in a dictionary:
def load_embeddings(glove_text_path):
    embeddings_index = {}
    f = open(glove_text_path)
    for line in tqdm(f):
        values = line.split()
        word = values[0]
        coefs = np.asarray(values[-300:], dtype='float32')
        embeddings_index[word] = coefs
    f.close()

    print('Found %s word vectors.' % len(embeddings_index))

    return embeddings_index

 #this function creates a normalized vector for the whole sentence
def sent2vec(s, embeddings_index):
    words = str(s)#.lower()#.decode('utf-8')
    words = word_tokenize(words)
    words = [w for w in words if not w in stop_words]
    words = [w for w in words if w.isalpha()]
    M = []
    for w in words:
        try:
            M.append(embeddings_index[w])
        except:
            continue
    M = np.array(M)
    v = M.sum(axis=0)
    if type(v) != np.ndarray:
        return np.zeros(300)
    return v / np.sqrt((v ** 2).sum())

def data_2_glove_vec(data, embeddings_index):
    '''

    :param data: List of text
    :return:
    '''
    # create sentence vectors using the above function for training and validation set
    sentence_vectors = [sent2vec(x, embeddings_index) for x in tqdm(data)]
    return np.array(sentence_vectors)

def spacy_word2vec(word, spacy_nlp):
    lex = spacy_nlp(word)
    if lex.has_vector:
        return lex.vector
    else:
        return spacy_nlp.vocab[0].vector  # return all zeros for Out of vocab


import pandas as pd

def split_date(df, col_name):
    # Function to split dates ("MM/DD/YYYY", etc.) into multiple columns
    aux_col = pd.to_datetime(df[col_name], infer_datetime_format=True)

    # Extract components from date_recorded, then drop the original column
    df['year'] = aux_col.dt.year
    df['month'] = aux_col.dt.month
    df['day'] = aux_col.dt.day

    return df

def check_nulls(df):
    #Check a dataframe for nulls, print/report them in a nice "pretty" format
    nulls = df.isnull().sum()

    for row in nulls.iteritems():
      print('The column', row[0], 'has', row[1], 'nulls.')

    return


import bpy
from . import prop, workspace

classes = (
    prop.BIMCoveringProperties,
    workspace.Hotkey,
)

def register():
    if not bpy.app.background:
        bpy.utils.register_tool(workspace.CoveringTool, after={"bim.structural_tool"}, separator=False, group=False)
    bpy.types.Scene.BIMCoveringProperties = bpy.props.PointerProperty(type=prop.BIMCoveringProperties)
#    bpy.types.Object.BIMObjectSpatialProperties = bpy.props.PointerProperty(type=prop.BIMObjectSpatialProperties)
#    bpy.types.Scene.BIMSpatialManagerProperties = bpy.props.PointerProperty(type=prop.BIMSpatialManagerProperties)

def unregister():
    if not bpy.app.background:
        bpy.utils.unregister_tool(workspace.CoveringTool)
    del bpy.types.Scene.BIMCoveringProperties
#    del bpy.types.Object.BIMObjectSpatialProperties
#    del bpy.types.Scene.BIMSpatialManagerProperties


"""
URL configuration for project project.

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/4.2/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.urls import path
from blog.views import (CategoryListView, CreatedByListView, PostListView,
                        SearchListView, TagListView, PageDetailView,
                        PostDetailView)

app_name = 'blog'

urlpatterns = [
    path('', PostListView.as_view(), name='index'),
    path('post/<slug:slug>/', PostDetailView.as_view(), name='post'),
    path('page/<slug:slug>/', PageDetailView.as_view(), name='page'),
    path(
        'created_by/<int:author_pk>/',
        CreatedByListView.as_view(),
        name='created_by'
    ),
    path('category/<slug:slug>/', CategoryListView.as_view(), name='category'),
    path('tag/<slug:slug>/', TagListView.as_view(), name='tag'),
    path('search/', SearchListView.as_view(), name='search'),
]


from spack import *

class PyPygeos(PythonPackage):
    """PyGEOS is a C/Python library with vectorized geometry functions.

    The geometry operations are done in the open-source geometry library GEOS.
    PyGEOS wraps these operations in NumPy ufuncs providing a performance
    improvement when operating on arrays of geometries."""

    homepage = "https://github.com/pygeos/pygeos"
    pypi = "pygeos/pygeos-0.8.tar.gz"

    maintainers = ['adamjstewart']

    version('0.8', sha256='45b7e1aaa5fc9ff53565ef089fb75c53c419ace8cee18385ec1d7c1515c17cbc')

    depends_on('python@3:', type=('build', 'link', 'run'))
    depends_on('py-setuptools', type='build')
    depends_on('py-numpy@1.10:', type=('build', 'link', 'run'))
    depends_on('geos@3.5:')


# !/usr/bin/python
# -*- coding: UTF-8 -*-
import DepthFirstOrder
class KosarajuSharirSCC(object):
    def __init__(self,g):
        self.G = g
        self.dfs = DepthFirstOrder(self.G.reverse())  # 用图的逆序数进行排列
        #得到图的强连通的结果
        self.marked = [False]*self.G.V
        self.id = [0]*self.G.V  # 给定点标号确定是不是在同一个强连通域内
        self.count = 0
        for v in self.dfs.reversePost():
            if not self.marked[v]:
                self.dfs(v)
                self.count += 1
    def dfs(self,v):
        self.marked[v] = True
        self.id[v] = self.count
        for w in self.G.adj[v]:
            if not self.marked[w]:
                self.dfs(w)
    def count(self):  # 图中强连通分量的数量
        return self.count
    def stronglyConnected(self,v,w):  # 判断图中是不是强连通分量
        return self.id[v] == self.id[w]
# 总结：有向图的可达性有别与无向图的，无向图的可达性就是连通性就是CC
# 可是有向图不同，有向图的强连通性与可达是两个概念所以需要新的方法解
# 这个问题


# -*- coding: utf-8 -*-
# -*- 弱小和无知不是生存的障碍，傲慢才是! -*-
# @author  : Fighter.Ma
# @Email   : fighter_ma1024@163.com
# @file    : NLP_Test.py
# @time    : 2022/2/2418:52
# @Software: PyCharm

import os
import docx
from ocr import api_call
from filereader import FILEREADER
from NLP_mysql import SqlFiles
import subprocess
import pandas as pd

file_reader_engine = 'http://10.80.49.137:80/service/'
ocr_server = "http://10.80.49.137:8507"
scene = "chinese_print"

def xw_toExcel(data, ID):
    fileName = "/root/GuoDianTou/Nlps_2022517.xlsx"
    rb = pd.read_excel(io=fileName, sheet_name="Sheet1", dtype={'ID':str})
    rb.loc[rb.ID==str(ID), 'Content'] = data
    rb.to_excel(fileName)

def Identification(file_path, ID):
    file = open(file_path, "r", encoding="utf-8", errors='ignore')
    content = file.read()
    file.close()
    msg = "".join(content).replace("\n", "").replace(" ", "").replace("（","(").replace("）", ")")
    msgs = msg.split('。')[0:3]
    ms = "".join(msgs)
    xw_toExcel(ms, ID)

def Doc_Ident(file_path, ID):
    try:
        output = subprocess.check_output(["antiword", file_path])
        # 解码
        output = output.decode("utf8")
        outputs = output.split('。')[0:3]
        ou = "".join(outputs)
        xw_toExcel(ou, ID)
    except Exception as subprocess_CalledProcessError:
        print(subprocess_CalledProcessError)

def Docx_Ident(file_path, ID):
    content = ""
    try:
        document = docx.Document(file_path)
        for paragraph in document.paragraphs:
            content += paragraph.text
        msg = ''.join(content).replace(" ","").replace("\t","").replace("\n","")
        msgs = msg.split('。')[0:3]
        ms = "".join(msgs)
        print(ms)
        xw_toExcel(ms, ID)
    except Exception as FileNotFound:
        message = "File NotFound Error~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
        print(message)
        with open("FileNotFound.txt", "a+", encoding="utf-8") as f:
            f.write(file_path)
            f.write("\n")

def Ocr_Ident(file_path, scene, ID):
    content = api_call(file_path, scene)
    data = content['data']['json']['general_ocr_res']['texts']
    msg = ""
    for da in data:
        msg += da
    msgs = msg.split('。')[0:3]
    ms = "".join(msgs)
    print(ms)
    xw_toExcel(ms, ID)

def Filereader_Ident(file_reader_engine, file_path, ID):
    filereader = FILEREADER(file_reader_engine, r"%s" % file_path)
    try:
        filereader.f_upload()
        filereader.f_inquire()
        msg = filereader.f_download()
        content = "".join(msg).replace('\n','')
        contents = content.split('。')[0:3]
        co = "".join(contents)
        print(co)
        xw_toExcel(co, ID)
    except Exception as FileNotFound:
        message = "File NotFound Error~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
        print(message)
        with open("FileNotFound.txt", "a+", encoding="utf-8") as f:
            f.write(file_path)
            f.write("\n")

def SqlPath():
    num = 0
    for pa in SqlFiles():
        path = '/root/GuoDianTou' + pa[1] + pa[2]
        num += 1
        print(num,path)
        if os.path.splitext(path)[-1] in [".pdf", ".PDF"]:
            Filereader_Ident(file_reader_engine, path, pa[0])
        elif os.path.splitext(path)[-1] in ['.png', '.jpg', '.jpeg']:
            Ocr_Ident(path, scene, pa[0])
        elif os.path.splitext(path)[-1] == '.txt':
            Identification(path, pa[0])
        elif os.path.splitext(path)[-1] == '.docx':
            Docx_Ident(path, pa[0])
        elif os.path.splitext(path)[-1] == '.doc':
            Doc_Ident(path, pa[0])
        else:
            message = 'File suffix error'
            print(message)

SqlPath()


import cv2
import numpy as np
import os

##KNN Code
def euclidean(x,y):
    return np.sqrt(((x-y)**2).sum())

def knn(train,test,k=5):
    dist = []

    for i in range(train.shape[0]):
    	ix = train[i,:-1]
    	iy = train[i,-1]

    	d = euclidean(test,ix)
    	dist.append([d,iy])

    dk = sorted(dist,key = lambda x:x[0])[:k]

    labels = np.array(dk)[:,-1]

    output = np.unique(labels,return_counts=True)

    index = np.argmax(output[1])
    return output[0][index]

#######

cap = cv2.VideoCapture(0)

cascade = cv2.CascadeClassifier('haarcascade_frontalface_alt.xml')

skip = 0
face_data = []
dataset_path = './data/'

labels = []
class_id = 0 # Labels for given file
names = {} # Mapping between id and name

for fx in os.listdir(dataset_path):
	if fx.endswith('.npy'):
		#Create mapping between class_id and name
		names[class_id] = fx[:-4]
		print("File Name : " +fx)
		data_item = np.load(dataset_path+fx)
		face_data.append(data_item)

		#Create labels for class
		target = class_id*np.ones((data_item.shape[0],))
		class_id += 1
		labels.append(target)

face_dataset = np.concatenate(face_data,axis=0)
face_labels = np.concatenate(labels,axis=0).reshape((-1,1))

print(face_dataset.shape)
print(face_labels.shape)

trainset = np.concatenate((face_dataset,face_labels),axis=1)
print(trainset.shape)

##Testing

while True:
	ret,frame = cap.read()

	faces = cascade.detectMultiScale(frame,1.3,5)

	for face in faces:
		x,y,w,h = face

		offset = 10
		face_section = frame[y-offset:y+h+offset,x-offset:x+w+offset]
		face_section = cv2.resize(face_section,(100,100))

		out = knn(trainset,face_section.flatten())

		#Display name on the screen and rectangle around it
		pred_name = names[int(out)]
		cv2.putText(frame,pred_name,(x,y-10),cv2.FONT_HERSHEY_SIMPLEX,1,(255,0,0),cv2.LINE_4)
		cv2.rectangle(frame,(x,y),(x+w,y+h),(0,255,255),2)

	cv2.imshow("Faces",frame)

	key_pressed = cv2.waitKey(1) & 0xFF
	if key_pressed == ord('q'):
		break

cap.release()
cv2.destroyAllWindows()


import matplotlib as mpl
mpl.use('Agg')

import matplotlib.pyplot as plt
import numpy as np

val = ( 10, 30, 20, 50,10 )
pos = np.arange(len(val))+.5

plt.barh(pos , val , align='center' )  # on utilise la fonction sinus de Numpy

# Les Ticks
plt.yticks( pos, ( "DEMO_IDX", "DEMO_DAT", "TOTO_DAT", "TOTO_IDX", "USERS" ) )
plt.xticks (np.arange(0,101,25), ("0%", "25%", "50%", "75%") )

## Les Label
plt.ylabel('TableSpace')
plt.xlabel("% Pourcentage Occupation")

plt.grid(True)

plt.savefig("toto.png", format="png")