M1Flag = True
M2Flag = True
iMat1, iMat2, iMat3, iMat4 = [], [] ,[] , []
while True:
    iChoice = int(input("""Enter 1: Enter value in Matrix 1
      2: Enter value in Matrix 2
      3: Addition
      4: Substraction
      5: Display Matrix
      6: Exit
=>"""))
    if iChoice == 1:
        row1 = int(input("Input Number of Rows:"))
        col1 = int(input("Input Number of Cols:"))
        if M2Flag == False:
            if row1 != row2 or col1 != col2:
                print("Warning !!! Dimension of Matrix 2 is " + str(row2)+ "x" + str(col2))
        iMat1 = [[int(input('index['+str(i)+"]["+str(j)+"]:")) for j in range(col1)]for i in range(row1)]
        M1Flag = False
    elif iChoice == 2:
        row2 = int(input("Input Number of Rows:"))
        col2 = int(input("Input Number of Cols:"))
        if M1Flag == False:
            if row1 != row2 or col1 != col2:
                print("Warning !!! Dimension of Matrix 1 is " + str(row1)+ "x" + str(col1))
        iMat2 = [[int(input('index['+str(i)+"]["+str(j)+"]:")) for j in range(col2)]for i in range(row2)]
        M2Flag = False
    elif iChoice == 3:
        if row1 == row2 and col1 == col2:
            iMat3 = [[iMat1[i][j] + iMat2[i][j] for j in range(col2)]for i in range(row1)]
        else:
            print("Dimensions of Matrices are not same!!!")
            print("Matrix 1: " + str(row1)+ "x" + str(col1))
            print("Matrix 2: " + str(row2)+ "x" + str(col2))
    elif iChoice == 4:
        if row1 == row2 and col1 == col2:
            iMat4 = [[iMat1[i][j] - iMat2[i][j] for j in range(col2)]for i in range(row1)]
        else:
            print("Dimensions of Matrices are not same!!!")
            print("Matrix 1: " + str(row1)+ "x" + str(col1))
            print("Matrix 2: " + str(row2)+ "x" + str(col2))
    elif iChoice == 5:
        dChoice = int(input("""Which Matrix do you want to see,
Enter 1, for Matrix 1
      2, for Matrix 2
      3, for Addition Matrix
      4, for Substraction Matrix
=>"""))
        if dChoice == 1 and len(iMat1) != 0:
            print("Matrix 1:")
            for i in range(row1):
                for j in range(col1):
                    print(iMat1[i][j], end = " ")
                print()
        elif dChoice == 2 and len(iMat2) != 0:
            print("Matrix 2:")
            for i in range(row2):
                for j in range(col2):
                    print(iMat2[i][j], end = " ")
                print()
        elif dChoice == 3 and len(iMat3) != 0:
            print("Addition Matrix:")
            for i in range(row1):
                for j in range(col1):
                    print(iMat3[i][j], end = " ")
                print()
        elif dChoice == 4 and len(iMat4) != 0:
            print("Substraction Matrix:")
            for i in range(row2):
                for j in range(col2):
                    print(iMat4[i][j], end = " ")
                print()
        else:
            print('Wrong Choice, Maybe the Matrix is not initialize.')

    elif iChoice == 6:
        break
    else:
        print("Wrong Choice")


# INSTRUCTIONS

# This time no story, no theory. The examples below show you how to write function accum:
# Examples:
# accum("abcd") -> "A-Bb-Ccc-Dddd"
# accum("RqaEzty") -> "R-Qq-Aaa-Eeee-Zzzzz-Tttttt-Yyyyyyy"
# accum("cwAt") -> "C-Ww-Aaa-Tttt"
# The parameter of accum is a string which includes only letters from a..z and A..Z.

# SOLUTION

def accum(s):
    finalStr = ''
    letters = list(s)
    for i in range(len(letters)):
        secondaryStr = (letters[i].lower()*(i+1)).capitalize()
        finalStr = finalStr + secondaryStr + "-"
    return finalStr[:-1]


import nltk     # for NLP

from biz.ritter.nlp.Sentence import Sentence

from java.nio.file import Path, Files
from java.io import IOException
from java.lang import Object
from biz.ritter.nlp.util import Toolkit

class Text(Object):
    '''
    classdocs
    '''

    def __init__(self):
        '''
        Constructor
        '''
        pass

    def setSource (self, path : Path):
        self.source = path

    def load (self) -> 'Text': # Python 'fearture' self type result, needed for FluentInterfacePattern
        if None == self.source :
            raise IOException ()
        self.text = Files.Files.readString(self.source)
        return self

    def config (self, lang_code_id_iso639_2='ger') -> 'Text':
        self.lang = lang_code_id_iso639_2
        return self

    def parse (self) -> 'Text':
        '''
          replace self.text with list of sentences in self
        '''
        # get sentences from nltk for german
        nltk_lang = Toolkit().iso_639_2_to_nltk_language(self.lang, 'german')

        allSentences = nltk.sent_tokenize(self.text,language=nltk_lang)
        self.sentences = []
        for nextSentence in allSentences :
            oneSentence = Sentence.Sentence()
            self.sentences.append(oneSentence)
            oneSentence.parse(nextSentence, language=self.lang)

        return self

    def run (self, lang_code_id_iso639_2='ger') -> 'Text':
        '''
        run is a collection-method for
        *    load
        *    config
        *    parse
        '''
        self.load() \
            .config(lang_code_id_iso639_2) \
            .parse()
        return self

    def toString (self) -> str:
        result = ""
        for nextSentence in self.sentences :
            result +=nextSentence.toString() + " "
        return result.strip()


__all__ = (
    "StatService",
)

from .service import Service

class StatService(Service):
    _fields = ['base_url', 'http_headers']

    def __init__(self, *args, **kargs):
        super(StatService, self).__init__(*args, **kargs)

    def disable_ssl_cert_validation(self):
        self.restclient.disable_ssl_cert_validation()

    def _append_locales(self, ext, locales):
        for locale in locales:
            ext += '&locale=%s' % locale
        return ext

    def get_project_stats(
            self, project_id, project_version, word=False, locales=None
    ):
        ext = "?detail=true&word=true" if word else "?detail=true&word=false"
        if isinstance(locales, list) and len(locales) > 0:
            ext = self._append_locales(ext, locales)
        res, content = self.restclient.process_request(
            'proj_trans_stats', project_id, project_version,
            headers=self.http_headers, extension=ext
        )
        return self.messages(res, content)

    def get_doc_stats(
            self, project_id, project_version, doc_id, word=False, locales=None
    ):
        ext = "?detail=true&word=true" if word else "?detail=true&word=false"
        if isinstance(locales, list) and len(locales) > 0:
            ext = self._append_locales(ext, locales)
        res, content = self.restclient.process_request(
            'doc_trans_stats', project_id, project_version, doc_id,
            headers=self.http_headers, extension=ext
        )
        return self.messages(res, content)


# -*- coding:utf-8 -*-

#title = 'Der Konzernabschluss | Buch | 9783791037301';
#title = "Der Konzernabschluss | Buch | 978-37910373-01";
#title = 'Sterne: die Suche nach den Wurzeln von England, Michael Holz - 9780563205005';
#title = 'Rondo 7/8, Sch�lerbuch Karl-Heinz Keller, Mildenberger, ISBN 3-619-77700-4';
#title = 'Rondo 7/8, Sch�lerbuch Karl-Heinz Keller, Mildenberger,  3-619-77700-4';
title = 'Artikelmerkmale\nArtikelzustand:\nGut:\nHinweise des Verkäufers:\n“Wir haben diesen Artikel sorgfältig für Sie geprüft!”\nISBN:\n9783464571446\nEAN:\n9783464571446\nReihe:\nDas Große Tafelwerk Interaktiv\nFach:\nPhysik, Chemie, Mathematik\nProduktart:\nLehrbuch\nFormat:\nGebundene Ausgabe\nErscheinungsjahr:\n2003\nAnzahl der Seiten:\n168\nAutor:\nMatthias Felsch, Rolf Winter, Wolfgang Pfeil, Wolfgang Kricke, Karlheinz Martin\nVerlag:\nCornelsen Verlag Gmbh\nPublikationsname:\nDas große Tafelwerk interaktiv - Allgemeine Ausgabe / Tafelwerk Mathematik, Informatik, Astronomie, Physik, Chemie, Biologie von Karlheinz Martin, Matthias Felsch, Rolf Winter, Wolfgang Kricke und Wolfgang Pfeil (2003, Gebundene Ausgabe)\nSprache:\nDeutsch\nGewicht:\n518g'

#title = 'Artikelmerkmale\nArtikelzustand:\nGut:\nHinweise des Verkäufers:\n“Wir haben diesen (2003, Gebundene Ausgabe)\nSprache:\nDeutsch\nGewicht:\n518g'

ISBN = [];

title2  = title.replace('\n',' ');
title2  = title2.replace('-','');
title2  = title2.replace('/','');
title2  = title2.split(" ",-1);

#i = 0;
#while i < len(title2) :
#
#
#    title2[i]=title2[i].replace('-','');
#    title2[i]=title2[i].replace('/','');
#
#    i= i+1;

i   =  0;
while i < len(title2):
    print(i)

    if '978' in title2[i]:
        ISBN = title2[i];
        print('ISBN contains 978');

        break;
    elif 'ISBN' in title2[i]:
        if i + 1 < len(title2) :
            ISBN = title2[i+1];
            print('text has ISBN word');
            break;

    else :
        if title2[i].isnumeric() :
            ctr = 0;
            for ch in  title2[i]:
                if ch.isdigit():
                    ctr = ctr + 1;
                else :
                    print('String has non-numeric characters');
                    break;

            if ctr >= 10:
                print('Text has ASIN/EAN number');
                ISBN = title2[i];
                break;
    i = i+1;

print('ISBN: {}'.format(ISBN));


#!/usr/bin/env python3
import numpy as np
from scipy import interpolate
from matplotlib import pyplot as plt

from tqdm import tqdm
import argparse
import sys
import os

import NSencounter as NE
import perturbations as PB
import mass_function
import params
import dirs
import tools
from tools import r_AS

import MilkyWay
import Andromeda

if not os.path.exists(dirs.data_dir + "distributions/"):
    os.makedirs(dirs.data_dir + "distributions/")

# constants
hbar = 6.582e-16  # GeV/GHz
Tage = 4.26e17
RSun = 8.33e3  # pc
pc = 3.086e16  # pc in m
cs = 3.0e8  # speed of light in m/s
vrel0 = 1.0e-3
vrel = vrel0 * cs / pc  # relative velocity in pc/s
u_dispersion = 1.0e-11  # velocity dispersion in pc/s

M_cut = 1.0e-29

## Neutron Star characteristics
MNS = 1.4  # MSun
RNS = 3.24e-13  # pc -- This corresponds to 10km

Nbins_mass = 300

# ---------------------------------
# -------- OPTIONS ----------------
# ---------------------------------

def sample_encounters(Ne, m_a, profile,  AMC_MF, galaxyID = "MW", circular=False, AScut = False, unperturbed=False, IDstr=""):

    Ne = int(Ne)

    file_suffix_in = tools.generate_suffix(profile, AMC_MF, circular=circular,
                                        AScut=False, unperturbed=unperturbed, IDstr=IDstr, verbose=False)

    #We need different labels for the input and output files, because
    #we may also need to label the outputs with "AScut" if we use that option
    file_suffix_out = tools.generate_suffix(profile, AMC_MF, circular=circular,
                                        AScut=AScut, unperturbed=unperturbed, IDstr=IDstr, verbose=True)

    #Set up the mass function
    M0 = AMC_MF.M0

    if (galaxyID == "MW"):
        Galaxy = MilkyWay
    elif (galaxyID == "M31"):
        Galaxy = Andromeda
    else:
        raise ValueError("Invalid galaxyID.")

    plt_path = dirs.plot_dir
    dist_path = dirs.data_dir + "distributions/"

    #Select which columns we want, depending on whether we're including the AS cut
    if (AScut):
        col = 2
    else:
        col = 1

    # Load in survival probabilities
    R_surv_file = dirs.data_dir + "SurvivalProbability_R_" + file_suffix_in + ".txt"# List of survival probabilities
    R_list, psurv_R_list = np.loadtxt(R_surv_file, delimiter=",", dtype="f8", usecols=(0, col), unpack=True)

    # Load in encounter rates
    encounter_file = dirs.data_dir + "EncounterRate_" + file_suffix_in + ".txt"
    R_list, dGammadR_list = np.loadtxt(encounter_file, delimiter=",", dtype="f8", usecols=(0, col), unpack=True)

    # Generate some interpolation functions
    # psurv_a     = interpolate.interp1d(a_list, psurv_a_list) # survival probability (as a function of a)
    # psurv_R     = interpolate.interp1d(R_list, psurv_R_list) # survival probability (as a function of R)
    dGammadR = interpolate.interp1d(R_list, dGammadR_list)  # PDF of the galactic radius

    #Set up dictionaries, which will hold the M, R, rho distributions of AMCs at each radius
    dist_r_list = []
    dist_rho_list = []

    dict_interp_r = dict()
    dict_interp_r_corr = dict()
    dict_interp_rho = dict()
    dict_interp_z = dict()
    dict_interp_mass = dict()

    # --------------------- First we prepare the sampling distributions and total interactions

    if unperturbed:
        dist_r, dist_Pr, dist_Pr_sigu = np.loadtxt( dist_path + "distribution_radius_" + file_suffix_in + ".txt", delimiter=", ", dtype="f8", usecols=(0, 1, 2), unpack=True)

        interp_r = interpolate.interp1d(dist_r, dist_Pr)
        interp_r_corr = interpolate.interp1d(dist_r, dist_Pr_sigu)

    else:

        # Loop through distances
        for i, R in enumerate(R_list):
            #R_kpc = R / 1e3

            # distRX is AMC radius
            # distRY is the PDF dP/dR (normalised as int dP/dR dR = 1) where R is the AMC radius
            # distRC is the PDF <sigma u>*dP/dR where R is the AMC radius
            # distDX is the list of densities for dPdrho
            # distDY is the dPdrho

            try:
                #print("Loading file for radius %.4f " % (R,))
                fname = dist_path + f"distribution_radius_{R:.4f}_{file_suffix_out}.txt"
                #print(fname)
                distRX, distRY, distRC = np.loadtxt(
                    fname,
                    delimiter=", ",
                    dtype="f8",
                    usecols=(0, 1, 2),
                    unpack=True,
                )
                # distDX, distDY = np.loadtxt(dist_path + 'distribution_rho_%.2f_%s.txt'%(R_kpc, PROFILE), delimiter =', ', dtype='f8', usecols=(0,1), unpack=True)

                dist_r_list.append(distRX)
                dict_interp_r[i] = interpolate.interp1d(
                    distRX, distRY
                )  # Radius distribution
                dict_interp_r_corr[i] = interpolate.interp1d(
                    distRX, distRC
                )  # Corrected radius distr

                fname = dist_path + f"distribution_mass_{R:.4f}_{file_suffix_out}.txt"
                #print(fname)
                distMX, distMY = np.loadtxt(
                    fname,
                    delimiter=", ",
                    unpack=True,
                )
                dict_interp_mass[i] = interpolate.interp1d(
                    distMX, distMY, bounds_error=False, fill_value=0.0
                )

            except:
                #print("Warning: File for radius %.4f does not exist" % (R,))
                dist_r_list.append(None)
                dist_rho_list.append(None)
                dict_interp_r[i] = None
                dict_interp_r_corr[i] = None
                # dict_interp_rho[i] = None

                # if (PROFILE == "NFW"):
                dict_interp_mass[i] = None

                continue

    GammaTot = np.trapz(dGammadR_list, R_list)  # s^-1
    #print("Gamma = ", GammaTot, "s^-1 [for f_AMC = 1]")

    # --------------------- Below we are computing the Signal

    # BJK: This still needs lots of cleaning up! A lot of this may be left-over useless stuff.

    #BJK: GOT TO HERE!!!
    Interactions = []

    R_sample = tools.inverse_transform_sampling(
        dGammadR, [np.min(R_list[psurv_R_list > 2e-6]), np.max(R_list)], n_samples=Ne, logarithmic=True
    )  # Galactocentric radius of interaction in pc
    R_sample = np.array(R_sample)

    # Draw the radii and densities of the axion MCs
    Z_gal = np.zeros(Ne)
    MC_r = np.zeros(Ne)
    MC_rho = np.zeros(Ne)

    for l, R in enumerate(tqdm(R_sample, desc="> Sampling encounters")):

        # Draw a value of z for the encounter
        dpdz = lambda Z: Galaxy.dPdZ(Z)
        Z_gal[l] = tools.inverse_transform_sampling(dpdz, np.array([-R, R]), n_samples=1, nbins=1000)

        Pr_check = 0
        if not unperturbed:
            # Find the GC radius that is closer to what is drawn from distribution
            abs_val = np.abs(R_list - R)
            smallest = abs_val.argmin()
            while (Pr_check <= 0):
                R_orb = R_list[smallest]  # Closest orbit to the r drawn
                dist_r = dist_r_list[smallest]  # List of MC radii
                # dist_rho  = dist_rho_list[smallest] # List of MC densities
                # interp_rho = dict_interp_rho[smallest]
                interp_r_corr = dict_interp_r_corr[smallest]

                # if (PROFILE == "NFW"):
                interp_M = dict_interp_mass[smallest]
                Pr_check = np.sum(interp_r_corr(dist_r))
                if (Pr_check <= 0):
                    #print("Trying next radius...")
                    smallest -= 1

        alpha_AS = r_AS(1.0, m_a)
        k_AMC = (3 / (4 * np.pi)) ** (1 / 3)

        # radius in pc
        MC_r[l] = tools.inverse_transform_sampling(
            interp_r_corr, dist_r, n_samples=1, nbins=10000, logarithmic=True
        )

        rho_max = 3 * AMC_MF.mmax / (4 * np.pi * MC_r[l] ** 3)

        if AScut:
            rho_min_AS = (alpha_AS * k_AMC / MC_r[l] ** 2) ** 3
        else:
            rho_min_AS = 1e-30

        # Sample AMC density rho given the AMC radius R
        if unperturbed:

            dPdM = lambda x: AMC_MF.dPdlogM(x) / x
            # (M_f/rho)*P(M_f)
            P_rho_given_r = lambda rho: ((4 * np.pi / 3) * MC_r[l] ** 3) * dPdM(
                (4 * np.pi / 3) * rho * MC_r[l] ** 3
            )
            rho_min = np.maximum(3 * AMC_MF.mmin / (4 * np.pi * MC_r[l] ** 3), rho_min_AS)

        else:
            P_rho_given_r = lambda rho: ((4 * np.pi / 3) * MC_r[l] ** 3) * interp_M(
                (4 * np.pi / 3) * rho * MC_r[l] ** 3
            )
            #rho_min = np.maximum(
            #    3 * (1e-6 * AMC_MF.mmin) / (4 * np.pi * MC_r[l] ** 3), rho_min_AS
            #)
            rho_min = np.maximum(
                3 * (1e-10 * M0) / (4 * np.pi * MC_r[l] ** 3), rho_min_AS
            )
        # P_rho_given_r = lambda rho: NE.P_r_given_rho(MCrad[l], rho, mmin, mmax, gg)*dict_interp_rho[smallst](rho)/dict_interp_rad[smallst](MCrad[l])

        MC_rho[l] = tools.inverse_transform_sampling(
            P_rho_given_r, [1e-6*rho_min, rho_max], n_samples=1, nbins=100000, logarithmic=True
        )

        if (np.isnan(MC_rho[l])):
            print(R, R_orb, MC_r[l], MC_rho[l])
            print(dist_r)
            print(interp_r_corr(dist_r))

        #if (1 < 2):
            plt.figure()
            rho_list = np.geomspace(rho_min, rho_max, 100000)
            plt.loglog(rho_list, P_rho_given_r(rho_list))
            plt.show()

        # print(rho_min, MC_rho[l])

    psi = np.random.uniform(-np.pi, np.pi, Ne)  # Longitude
    xi = np.arcsin(Z_gal / R_sample)  # Latitude

    z0 = Z_gal
    x0 = R_sample * np.cos(xi) * np.cos(psi)
    y0 = R_sample * np.cos(xi) * np.sin(psi)
    R_spherical = np.sqrt(x0 ** 2 + y0 ** 2 + z0 ** 2)

    s0 = np.sqrt((RSun + x0) ** 2 + y0 ** 2 + z0 ** 2)  # Distance from events in pc
    bG = (
        np.arctan(z0 / np.sqrt((RSun + x0) ** 2 + y0 ** 2)) * 180.0 / np.pi
    )  # Galactic Latitude
    lG = np.arctan2(y0, (RSun + x0)) * 180.0 / np.pi  # Galactic Longitude

    # Relative velocity between NS & AMC
    vrel = np.sqrt(2) * Galaxy.sigma(R_sample) * 3.24078e-14
    ux = np.random.normal(0, vrel, Ne)  # pc/s
    uy = np.random.normal(0, vrel, Ne)  # pc/s
    uz = np.random.normal(0, vrel, Ne)  # pc/s
    # ut    = np.abs(np.sqrt((ux + VNSX)**2 + (uy + VNSY)**2 + uz**2)) # pc/s
    ut = np.sqrt(ux ** 2 + uy ** 2 + uz ** 2)  # pc/s

    # BJK: Here, we need to make sure that we implement the same cut on the
    # impact parameter as we did for the radii of very diffuse AMCs
    rho_loc = Galaxy.rhoNFW(R_sample)
    rho_crit = rho_loc * params.min_enhancement

    if profile == "PL":
        r_cut = MC_r * np.minimum(
            (MC_rho / (4 * rho_crit)) ** (4 / 9), np.ones_like(MC_rho)
        )

    elif profile == "NFW":
        c = 100
        rho_s = MC_rho * (
            c ** 3 / (3 * NE.f_NFW(c))
        )  # Convert mean density rhoi to AMC density
        r_cut = MC_r * np.minimum(NE.x_of_rho(rho_crit / rho_s), np.ones_like(MC_rho))

    # Make sure everything is fine
    r_cut[r_cut < 0.0] = 0.0 * r_cut[r_cut < 0.0] + MC_r[r_cut < 0.0]
    r_cut[np.isnan(r_cut)] = MC_r[np.isnan(r_cut)]
    r_cut[np.isinf(r_cut)] = MC_r[np.isinf(r_cut)]

    # Impact parameter
    b = np.sqrt(np.random.uniform(0.0, r_cut ** 2, Ne))

    MC_M = (4*np.pi*MC_r**3/3)*MC_rho

    lablist = ["x0", "y0", "z0", "rho", "r", "M"]

    for k in range(Ne):
        #Select from either the GC population or the CMZ population
        if (R_spherical[k] > 10):
            NS_dict = Galaxy.CMZ_dict
            len_NS = Galaxy.len_CMZ
        else:
            NS_dict = Galaxy.GC_dict
            len_NS = Galaxy.len_GC

        iNS = np.random.randint(0, len_NS)

        # SJW version
        interaction_params = [
            NS_dict["B0"][iNS],
            NS_dict["T"][iNS],
            NS_dict["theta"][iNS],
            NS_dict["t"][iNS],
            x0[k],
            y0[k],
            z0[k],
            MC_rho[k],
            MC_r[k],
            MC_M[k],
            b[k],
            ut[k]
        ]
        Interactions.append(interaction_params)

    #Interactions = np.array(Interactions)
    Interactions = np.array(Interactions)
    # print(Interactions, Interactions.shape)

    int_file = dirs.data_dir + "Interaction_params_" + file_suffix_out + ".txt.gz"

    print("> Outputting encounters to file:", int_file)

    np.savetxt(
        int_file,
        Interactions,
        header="B0 [G], Period [s], Misalignment angle [radians], NS Age [Myr], x [pc], y [pc], z [pc], AMC density [Msun/pc^3], AMC radius [pc], AMC mass [Msun],  Impact parameter [pc], Relative velocity [pc/s]",
        fmt="%.5e",
    )
    return b/ut

def getOptions(args=sys.argv[1:]):
    # Parse the arguments!
    parser = argparse.ArgumentParser(description="...")

    parser.add_argument("-Ne", "--Ne", help="Number of encounters to sample", type=int, default=1000)
    parser.add_argument("-profile", "--profile", help="Density profile for AMCs - `NFW` or `PL`", type=str, default="PL")
    parser.add_argument("-m_a", "--m_a", type=float, help="Axion mass in eV", default = 50e-6)
    parser.add_argument("-unperturbed", "--unperturbed", help="Calculate for unperturbed profiles?", type=bool, default=False)
    parser.add_argument("-galaxyID", "--galaxyID", type=str, help="ID of galaxy - 'MW' or 'M31'", default="MW")
    parser.add_argument("-circ", "--circular", dest="circular", action="store_true", help="Use the circular flag to force e = 0 for all orbits.")
    parser.add_argument("-AScut", "--AScut", dest="AScut", action="store_true", help="Include an axion star cut on the AMC properties.")
    parser.add_argument("-MF_ID", "--mass_function_ID", help="...", type=str, default="delta_c")
    parser.add_argument("-IDstr", "--IDstr", type=str, help = "ID string to label the output files.", default="")
    parser.set_defaults(circular=False)
    parser.set_defaults(AScut=False)

    options = parser.parse_args(args)
    return options

if __name__ == '__main__':
    opts = getOptions(sys.argv[1:])

    #Create a mass function based on the input "mass function ID"
    AMC_MF = get_mass_function(opts.MF_ID, opts.m_a, opts.profile)
    AMC_MF.label = opts.MF_ID

    sample_encounters(opts.Ne, opts.m_a, opts.profile, AMC_MF,  opts.galaxyID,  opts.circular, opts.AScut, opts.unperturbed, opts.IDstr)


#!/usr/bin/env python
# coding=utf-8
""""""

collapse_iupac = {
    ('-',): '-',
    ('A',): 'A',
    ('G',): 'G',
    ('C',): 'C',
    ('T',): 'T',
    ('C', 'T'): 'y',
    ('A', 'G'): 'r',
    ('A', 'T'): 'w',
    ('C', 'G'): 's',
    ('G', 'T'): 'k',
    ('A', 'C'): 'm',
    ('A', 'G', 'T'): 'd',
    ('A', 'C', 'G'): 'v',
    ('A', 'C', 'T'): 'h',
    ('C', 'G', 'T'): 'b',
    ('A', 'C', 'G', 'T'): 'n',
}

expand_iupac = {value.upper(): set(key) for key, value in collapse_iupac.items()}

def degenerate_consensus(aligned_seq_list):
    consensus = ""
    length = len(aligned_seq_list[0])
    for seq in aligned_seq_list:
        assert len(seq) == length
    for loc in range(length):
        base_set = set.union(*[expand_iupac[seq[loc].upper()] for seq in aligned_seq_list])
        if base_set == {"_"}:
            consensus += ""
        elif '-' in base_set:
            consensus += '-'
        else:
            consensus += collapse_iupac[tuple(sorted(base_set))]
    return consensus


import datetime
import hashlib
import hmac
import json
import platform
from learnosity_sdk._version import __version__

from learnosity_sdk.exceptions import ValidationException

def format_utc_time():
    "Get the current UTC time, formatted for a security timestamp"
    now = datetime.datetime.utcnow()
    return now.strftime("%Y%m%d-%H%M")

class Init(object):
    """
    This class is used to generate the necessary security and request data (in
    the correct format) to integrate with any of the Learnosity API services.
    """

    services = [
        'annotations', 'assess', 'author', 'data', 'events', 'items', 'questions',
        'reports'
    ]

    security_keys = [
        'consumer_key', 'domain', 'timestamp', 'expires', 'user_id'
    ]

    __telemetry_enabled = True

    def __init__(
            self, service, security, secret,
            request=None, action=None):
        self.service = service
        self.security = security.copy()
        self.secret = secret
        self.request = request
        if hasattr(request, 'copy'):
            self.request = request.copy()
        self.action = action

        self.validate()
        self.add_telemetry_data()
        self.request_string = self.generate_request_string()
        self.sign_request_data = True
        self.set_service_options()
        self.security['signature'] = self.generate_signature()

    def is_telemetry_enabled(self):
        return self.__telemetry_enabled

    def generate(self, encode=True):
        """
        Generate the data necessary to make a request to one of the Learnosity
        products/services.

        If encode is True, the result is a JSON string. Otherwise, it's a
        dictionary. If self.service == data, encode is ignored.
        """
        output = {}

        if self.service == 'questions':
            # Add the security packet to the root of output
            output.update(self.security)

            # Remove the domain key from the security packet
            if 'domain' in output.keys():
                del output['domain']

            # Stringify the request packet if necessary
            if self.request is not None:
                output.update(self.request)

        elif self.service == 'events':
            output['security'] = self.security
            output['config'] = self.request
        elif self.service == 'assess':
            if self.request is not None:
                output.update(self.request)
        elif self.service == 'data':
            # We ignore the encode param for data API
            output['security'] = json.dumps(self.security)

            if self.request_string is not None:
                output['request'] = self.request_string

            if self.action is not None:
                output['action'] = self.action

            return output
        else:
            output['security'] = self.security

            if self.request is not None:
                output['request'] = self.request

            if self.action is not None:
                output['action'] = self.action

        if encode or self.request_passed_as_string:
            return json.dumps(output)
        else:
            return output

    def get_sdk_meta(self):
        return {
            'version': self.get_sdk_version(),
            'lang': 'python',
            'lang_version': platform.python_version(),
            'platform': platform.system(),
            'platform_version': platform.release()
        }

    def get_sdk_version(self):
        return __version__

    def generate_request_string(self):
        if self.request is None:
            return None
        return json.dumps(self.request, separators=(',', ':'), ensure_ascii=False)

    def generate_signature(self):

        vals = []

        # Add each valid security field.
        # The order is signifcant.
        for key in self.security_keys:
            if key in self.security:
                vals.append(self.security[key])

        # Add the request if necessary
        if self.sign_request_data and self.request_string is not None:
            vals.append(self.request_string)

        if self.action is not None:
            vals.append(self.action)

        return self.hash_list(vals)

    def validate(self):
        # Parse the security packet if the user provided it as a string
        if isinstance(self.security, str):
            self.security = json.loads(self.security)

        # Parse the request packet if the user provided it as a string
        self.request_passed_as_string = False
        if isinstance(self.request, str):
            self.request = json.loads(self.request)
            self.request_passed_as_string = True

        # Validate field lengths and types
        if len(self.service) == 0:
            raise ValidationException(
                "The `service` argument wasn't found or was empty")

        if len(self.secret) == 0:
            raise ValidationException(
                "The `secret` argument wasn't found or  was empty")

        if self.action is not None and not isinstance(self.action, str):
            raise ValidationException("The action parameter must be a string")

        # Check that service is valid
        if self.service not in self.services:
            raise ValidationException(
                "Service not valid: {}".format(self.service))

        # Check that security keys are valid
        for k in self.security.keys():
            if k not in self.security_keys:
                raise ValidationException(
                    "Invalid key found in the security packet: {}".format(k))

        # Add timestamp if missing
        if 'timestamp' not in self.security:
            self.security['timestamp'] = format_utc_time()

        # Special case for Questions API
        if self.service == 'questions' and \
                'user_id' not in self.security:
            raise ValidationException("questions API requires a user id")

    def set_service_options(self):
        if self.service == 'questions':
            self.sign_request_data = False
        elif self.service == 'assess':
            self.sign_request_data = False

            if 'questionsApiActivity' in self.request:
                questionsApi = self.request['questionsApiActivity']

                if 'domain' in self.security:
                    domain = self.security['domain']
                elif 'domain' in questionsApi:
                    domain = questionsApi['domain']
                else:
                    domain = 'assess.learnosity.com'

                self.request['questionsApiActivity'] = {
                    'consumer_key': self.security['consumer_key'],
                    'timestamp': self.security['timestamp'],
                    'user_id': self.security['user_id'],
                    'signature': self.hash_list({
                        'consumer_key': self.security['consumer_key'],
                        'domain': domain,
                        'timestamp': self.security['timestamp'],
                        'user_id': self.security['user_id'],
                        'secret': self.secret
                    }.values())
                }

                for key in [ 'consumer_key', 'domain', 'timestamp', 'user_id', 'signature']:
                    if key in questionsApi.keys():
                        del questionsApi[key]

                self.request['questionsApiActivity'].update(questionsApi)

        elif self.service == 'items' or self.service == 'reports':
            if 'user_id' not in self.security and \
                    'user_id' in self.request:
                self.security['user_id'] = self.request['user_id']

        elif self.service == 'events':
            self.sign_request_data = False
            hashed_users = {}
            for user in self.request.get('users', []):
                concat = "{}{}".format(user, self.secret)
                hashed_users[user] = hashlib.sha256(concat.encode('utf-8')).hexdigest()

            if len(hashed_users) > 0:
                self.security['users'] = hashed_users

    def hash_list(self, l):
        "Hash a list by concatenating values with an underscore"
        concatValues = "_".join(l)
        signature =  hmac.new(bytes(str(self.secret),'utf_8'), msg = bytes(str(concatValues) , 'utf-8'), digestmod = hashlib.sha256).hexdigest()
        return '$02$' + signature

    def add_telemetry_data(self):
        if self.__telemetry_enabled:
            if 'meta' in self.request:
                self.request['meta']['sdk'] = self.get_sdk_meta()
            else:
                self.request['meta'] = {
                    'sdk': self.get_sdk_meta()
                }

    """
    We use telemetry to enable better support and feature planning. It is
    however not advised to disable it, and it will not interfere with any usage.
    """

    @classmethod
    def disable_telemetry(cls):
        cls.__telemetry_enabled = False

    @classmethod
    def enable_telemetry(cls):
        cls.__telemetry_enabled = True


import logging
import time
import unittest
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support.wait import WebDriverWait
from TestBaseSetUp.Variable import ConifigVariable
from TestPages.SettingPage import Setting
from UserTestData.UserTestDataVar import TestData

class DropBoxConnTest(unittest.TestCase):
    logging.basicConfig(filename="../LoggerReport/report.log",
                        format='%(asctime)s - %(message)s',
                        level=logging.DEBUG,
                        filemode='w')

    def setUp(self):
        self.driver = webdriver.Chrome(ConifigVariable.chromePathWin)
        print("Chrome driver opened for testing dropBox conn test")
        self.driver.maximize_window()
        logging.info("Browser launched and maximize")
        print("Chrome browser launched and maximized")
        self.driver.get(ConifigVariable.StaggingURL)
        logging.info("I2chain Application Opened")
        print("i2chain application running in backend")

    def testDropBoxCon(self):
        setting = Setting(self.driver)
        logging.info("=========Setting page initialized=====")
        print("setting page initialised")
        window_before = self.driver.window_handles[0]
        setting.clickOnSetting()
        print("clicked on setting")
        time.sleep(5)
        setting.clickDropBoxButton()
        time.sleep(10)
        window_After = self.driver.window_handles[1]

        if window_After != window_before:
            self.driver.switch_to_window(window_After)
            logging.info("getting window details" + window_After)
            print("new window found for dropBox connection user need to provide credentials")
            setting.clickOnSignWithGoogle()
            self.driver.implicitly_wait(40)
            setting.enterDropBoxEmailID(TestData.setUserName)
            time.sleep(10)
            logging.info("==entering gmail user name ==")
            print("entering drop box user name")
            setting.clickOnDropBoxUserNextButton()
            time.sleep(10)
            logging.info("==Clicking on next button==")
            print("==clicking on next button==")
            setting.enterDropBoxPassword(TestData.dropBoxPass)
            logging.info("==Entering gmail password==")
            print("entering gmail password")
            time.sleep(5)
            setting.clickOnDropBoxPassNextButton()
            time.sleep(15)

            self.driver.switch_to_window(window_before)

            dropBoxSucMessage = WebDriverWait(self.driver, 30).until(
                expected_conditions.presence_of_element_located(
                    (By.XPATH, "//div[@class='alert alert-success alert-dismissible fade show']")))

            logging.info(dropBoxSucMessage.text)

            text = "Successfully connected to dropbox!"

            if text in dropBoxSucMessage.text:
                logging.info("==dropBox drive connected successfully==")
                time.sleep(15)
                setting.clickDropBoxButton()
                logging.info("==clicking on gdrive button for logout connected drive==")
                print("==clicking on gdrive button for logout connected drive==")
                time.sleep(10)
                setting.clickOnDropBoxLogOutButton()
                logging.info("==clicking on logout button to log our==")
                print("==clicking on logout button to log our==")
                time.sleep(10)

                setting.clickOnGdriveLogOutyesButton()
                logging.info("==clicking on yes button==")
                print("==clicking on yes button ==")
                time.sleep(10)

                logoutSuccessMessage = WebDriverWait(self.driver, 30).until(
                    expected_conditions.presence_of_element_located(
                        (By.XPATH, "//div[@class='alert alert-success alert-dismissible fade show']")))

                logging.info(logoutSuccessMessage.text)

                text = "Successfully logout from dropbox!!!"
                if text in logoutSuccessMessage.text:
                    print("==Log Out Successfully=="+text)
                    logging.info("====logout successfully====")

            else:
                logging.info("==========alert Message DoesNot Match drive not connected============")

    def tearDown(self):
        self.driver.close()

if __name__ == '__main__':
    unittest.main()


import pandas as pd
import nltk
import os
PROXY = ""
os.environ["HTTP_PROXY"]  = PROXY
os.environ["HTTPS_PROXY"] = PROXY

# redact proper names and company names using nltk
def names(df):
    nltk.set_proxy(None)
    #nltk.download()

    # print('\nTag each row sentences\n')
    tagged_sentences = []
    for text in range(len(df)):
        sentence = df.iloc[text]
        # print('sentence\n', sentence)
        tagged_sentences.append(nltk.tag.pos_tag(sentence.split()))
        # print('tagged_sentences\n', tagged_sentences[text])

    # print('\nEdit each row sentences\n')
    edited_sentences = []
    for sentence in range(len(tagged_sentences)):
        edited_sentence = []
        for word, tag in tagged_sentences[sentence]:
            if tag == 'NNP' or tag == 'NNPS' or tag == 'NNS':
                edited_sentence.append('xxx')
            else:
                edited_sentence.append(word)

        edited_sentences.append(str(' '.join(edited_sentence)))
        # print('edited_sentences\n', edited_sentences)

    return pd.Series(edited_sentences)

######################### Test
##############################

# Texts = ["they work at Microsoft, and my name is Sami", "Google CEO is Sunder Pichai"]
# data = pd.Series(Texts)
# print(names(data))


# 必要なライブラリをインポートします。
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.metrics import r2_score
from sklearn.preprocessing import StandardScaler
import lightgbm as lgb

# データの読み込みと前処理を行う関数を定義します。
def load_and_preprocess_data():
    # CSVファイルからデータを読み込みます。
    df_mc = pd.read_csv("out/mc_append.csv", header=None)
    df_hc = pd.read_csv("out/hc_append.csv", header=None)

    # データを標準化します。
    scaler_hc = StandardScaler()
    scaler_mc = StandardScaler()
    scaler_hc.fit(df_hc)
    scaler_mc.fit(df_mc)
    df_hc = scaler_hc.transform(df_hc)
    X = scaler_mc.transform(df_mc)
    y = df_hc[:, 3]

    # データを学習用と評価用に分割します。
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)

    return X_train, X_test, y_train, y_test

# モデルの学習と評価を行う関数を定義します。
def train_and_evaluate_model(X_train, X_test, y_train, y_test):
    # LightGBMのモデルを作成するためのパラメータを設定します。
    params = {
        "boosting_type": "gbdt",
        "objective": "binary",
        "metric": "auc",
        "num_leaves": 16,
        "learning_rate": 0.1,
        "n_estimators": 100000,
        "random_state": 0,
    }
    model = lgb.LGBMRegressor(**params)

    # モデルを学習させます。学習中に評価用データでのAUCを計算し、早期停止の条件を設定します。
    model.fit(
        X_train,
        y_train,
        eval_set=[(X_train, y_train), (X_test, y_test)],
        early_stopping_rounds=100,
    )

    # モデルの最適なイテレーション数を表示します。
    print(model.best_iteration_)

    # 評価用データでの予測値を生成します。
    y_pred = model.predict(X_test)

    # 予測値と真の値の決定係数（R2スコア）を計算します。
    print(r2_score(y_test, y_pred))

    return model, y_pred

# メインの処理を実行します。
if __name__ == "__main__":
    # データの読み込みと前処理を行います。
    X_train, X_test, y_train, y_test = load_and_preprocess_data()

    # モデルの学習と評価を行います。
    model, y_pred = train_and_evaluate_model(X_train, X_test, y_train, y_test)

    # 予測値と真の値の散布図をプロットします。
    plt.scatter(y_test, y_pred)
    plt.xlabel("True value")
    plt.ylabel("Predicted value")
    plt.show()


from django.shortcuts import render_to_response
from django.http import HttpResponseRedirect
from django.core.urlresolvers import reverse
from django.db import connection
from django.http import HttpResponse, HttpResponseNotFound
from django.core.exceptions import *
from django.core.servers.basehttp import FileWrapper
from appy.pod.renderer import Renderer
import os
from models import *

import django.utils.simplejson as simplejson
import datetime
class JSONEncoder(simplejson.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, datetime.datetime):
            return obj.strftime('%Y-%m-%d %H:%M:%S')
        elif isinstance(obj,datetime.date):
            return obj.strftime('%Y-%m-%d')
        else:
            try:
                res = simplejson.JSONEncoder.default(self, obj)
            except TypeError:
                res = 'ERROR'
        return res

json2 = JSONEncoder()

def add_fields_to_persona(persone, *args, **kwargs):
    for persona in persone:
        try:
            call = Chiamata.objects.filter(persona = persona).latest()
        except ObjectDoesNotExist:
            continue
        persona.data_ultima_chiamata = call.data
        for k,v in risposta_choices:
            if call.risposta == k:
                persona.risposta = v
                break
    return persone

def index(request):
    return render_to_response('callcenter_index.html')

def search(request, emitter_format):
        p = Persona.objects.all().order_by('cognome')
        if request.GET.get('nome',''):
            p = p.filter(nome__icontains=request.GET.get('nome',''))
        if request.GET.get('cognome',''):
            p = p.filter(cognome__icontains=request.GET.get('cognome',''))
        if request.GET.get('secondo_cognome',''):
            p = p.filter(secondo_cognome__icontains=request.GET.get('secondo_cognome',''))
        if request.GET.get('indirizzo',''):
            p = p.filter(indirizzo__icontains=request.GET.get('indirizzo',''))
        if request.GET.get('citta',''):
            p = p.filter(citta__icontains=request.GET.get('citta',''))
        if request.GET.get('provincia',''):
            p = p.filter(provincia__icontains=request.GET.get('provincia',''))
        if request.GET.get('cap',''):
            p = p.filter(cap__icontains=request.GET.get('cap',''))
        if request.GET.get('telefono',''):
            p = p.filter(telefono__icontains=request.GET.get('telefono',''))
        if request.GET.get('email',''):
            p = p.filter(email__icontains=request.GET.get('email',''))
        if request.GET.get('cliente') == 'on':
            p = p.filter(cliente=True)
        if request.GET.get('problemi_udito') == 'on':
            p = p.filter(problemi_udito=True)
        if request.GET.get('porta_apparecchio') == 'on':
            p = p.filter(porta_apparecchio=True)
        if request.GET.get('venuto') == 'on':
            p = p.filter(venuto=True)
        if request.GET.get('spontaneo') == 'on':
            p = p.filter(spontaneo=True)
        if request.GET.get('perdita_media',''):
            p = p.filter(perdita_media__icontains=request.GET.get('perdita_media',''))
        if request.GET.get('note',''):
            p = p.filter(note__icontains=request.GET.get('note',''))
        if request.GET.get('preventivo',''):
            p = p.filter(preventivo_media__icontains=request.GET.get('preventivo',''))

        start = int(request.GET.get('start', 0))
        limit = int(request.GET.get('limit', len(p)))

        result = p[start:start+limit]

        result = add_fields_to_persona(result)

        return HttpResponse(json2.encode({"totalCount": len(p),"root": [pe.__dict__ for pe in result]}))

def ultimePersoneChiamate(request):
    calls = Chiamata.objects.all()
    p = list()
    for c in calls:
        c.persona.data_ultima_chiamata = c.data
        p.append(c.persona)

    start = int(request.GET.get('start', 0))
    limit = int(request.GET.get('limit', len(p)))

    return HttpResponse(json2.encode({"totalCount": len(p),"root": [x.__dict__ for x in p[start:start+limit]]}))

def Chiamate(request, id):
    calls = Chiamata.objects.filter(persona = Persona.objects.get(pk = id)).order_by('-data')

    start = int(request.GET.get('start', 0))
    limit = int(request.GET.get('limit', len(calls)))

    for x in calls[start:start+limit]:
        x.risposta = x.get_risposta_display()

    return HttpResponse(json2.encode({"totalCount": len(calls),"root": [x.__dict__ for x in calls[start:start+limit]]}))

def LetterePersone(request):
    p = Lettera.objects.filter(data_invio=None)

    start = int(request.GET.get('start', 0))
    limit = int(request.GET.get('limit', len(p)))

    res = [dict(x.__dict__,**x.persona.flat()) for x in p[start:start+limit]]

    return HttpResponse(json2.encode({"totalCount": len(res),"root": res }))

def genEtichette(request):

    basepath=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'etichette')
    try:
        os.remove(os.path.join(basepath, 'etichette.odt'))
    except OSError:
        pass
    letters = Lettera.objects.filter(data_invio=None)
    p_list = list(set(x.persona for x in letters if x.persona.indirizzo != ''))
    p_list.sort(key=lambda x: x.cognome)
    renderer = Renderer(os.path.join(basepath,'EtichetteA4Cac.odt'), {'p_list':p_list}, os.path.join(basepath, 'etichette.odt'))
    renderer.run()
    response = HttpResponse(FileWrapper(open(os.path.join(basepath, 'etichette.odt'))), content_type='application/vnd.oasis.opendocument.text')
    response['Content-Disposition'] = 'attachment; filename=etichette.odt'
    response['Content-Length'] = os.path.getsize(os.path.join(basepath, 'etichette.odt'))
    return response

def sendLetter(request, id):
    try:
        p = Persona.objects.get(pk=int(id))
    except Persona.DoesNotExist:
        return HttpResponseNotFound()
    l = Lettera(persona=p, data=datetime.date.today(), data_invio=None)
    l.save()
    return HttpResponse(json2.encode({"totalCount": 1, 'root':l.__dict__}))

def lettersSent(request):
    for x in Lettera.objects.filter(data_invio=None):
        x.data_invio=datetime.date.today()
        x.save()
    return HttpResponse(json2.encode({"totalCount": 1, 'success':True}))

def listLettersSent(request, id):
    l_list = Lettera.objects.filter(persona__pk=id)

    start = int(request.GET.get('start', 0))
    limit = int(request.GET.get('limit', len(l_list)))

    return HttpResponse(json2.encode({"totalCount": len(l_list),"root": [x.__dict__ for x in l_list[start:start+limit]]}))


# -*- coding: utf-8 -*-
import scrapy
import sys
import time

sys.path.append(".")
import logging
from scrapy import Request
from ..items import EtHostInfo, ParseEtBusiInfo, EtNameStatus, ParseEtShareholderInfo, ParseEtForeignInvestmentInfo, \
    ParseEtBranchOfficeInfo, ParseWechatListInfo, ParseEtContaInerCopyrightInfo, ParseEtContaInerIcpInfo, \
    ParseEtTrademarkInfo, ParseEtRongziInfo

logger = logging.getLogger(__name__)
from . import hu_utils

"""
使用yield是可以的，猜测是刚开始默认使用start_requests函数，之后的函数调用
scrapy中有一些默认使用的函数

"""

class TianyanchaSpider(scrapy.Spider):
    name = 'tianyancha'

    def start_requests(self):
        """
        拼接初始url，解析初始页面
        :param et_names: tuple元组类型，包含公司的etid和全名etname
        :return:一个字典，包含公司etid，etname，二级页面url
        """
        conn = hu_utils.open_local_db()
        et_names = hu_utils.select_one(conn)
        logger.info("获取的企业数{}".format(len(et_names)))
        a = 0
        for et_name in et_names:
            a += 1
            logger.info("正在获取第{}条数据......".format(a))
            print("正在获取第{}条数据......".format(a))
            if et_name[1]:
                start_url = "https://www.tianyancha.com/search?searchType=company&key=%s" % et_name[1]
                request = Request(url=start_url, callback=self.parse_start_url,
                                  meta={"etid": et_name[0], "et_name": et_name[1]})
                yield request

    def parse_start_url(self, response):
        try:
            if "没有找到相关结果" not in response.text:
                url_two = response.xpath("//div[@class='header']/a/@href").extract_first()  # 获取二级页面url
                # logger.info("url_two{}".format(url_two))
            else:
                logger.info("没有搜索到结果")
                et_state = EtNameStatus()
                et_state["et_name"] = response.meta["et_name"]
                et_state['etid'] = response.meta["etid"]
                et_state["status"] = 1
                et_state["update_time"] = int(time.time())
                yield et_state
                return
            if url_two:  # 判断url是否存在，不存在表示查询没结果
                logger.info("存在二级url，正在获取数据......")
                yield Request(url=url_two, callback=self.parse,
                              meta={"etid": response.meta["etid"], "et_name": response.meta["et_name"]})
        except:
            logger.debug("出错")
            yield Request(url=response.url, callback=self.parse_start_url)

    def parse(self, response):
        et_state, item = self.parse_et_host_info(response)
        yield et_state
        yield item
        # logger.info("执行parse_et_busi_info---------")
        yield self.parse_et_busi_info(response)
        # logger.info("执行parse_et_shareholder_info---------")
        yield self.parse_et_shareholder_info(response)
        # logger.info("执行parse_et_foreign_investment_info---------")
        yield self.parse_et_foreign_investment_info(response)
        # logger.info("执行parse_et_branch_office_info---------")
        yield self.parse_et_branch_office_info(response)
        # logger.info("执行parse_wechat_list_info---------")
        yield self.parse_wechat_list_info(response)
        # logger.info("执行parse_et_container_copyright_info---------")
        yield self.parse_et_container_copyright_info(response)
        # logger.info("执行parse_et_container_icp_info---------")
        yield self.parse_et_container_icp_info(response)
        # logger.info("执行parse_et_trademark_info---------")
        yield self.parse_et_trademark_info(response)
        # logger.info("执行parse_et_rongzi_info---------")
        yield self.parse_et_rongzi_info(response)

    def parse_et_host_info(self, response):
        """
        --企业主要信息--
        :return:
        """
        item = EtHostInfo()
        et_state = EtNameStatus()
        et_website = response.xpath("//a[@class='company-link']/text()").extract()  # 官网
        et_address = response.xpath("//span[@class='address']/@title").extract()  # 地址
        et_detail = response.xpath("//script[@id='company_base_info_detail']/text()").extract()  # 简介
        et_legal_rp = response.xpath("//div[@class='humancompany']/div[@class='name']/a/text()").extract()  # 法定代表人
        et_states = response.xpath("//div[@class='num-opening']/text()").extract()  # 公司状态
        item["et_website"] = "".join(et_website)
        item["et_address"] = "".join(et_address)
        item["et_detail"] = "".join(et_detail)
        item["et_legal_rp"] = "".join(et_legal_rp)
        item["et_states"] = "".join(et_states)
        item["etid"] = response.meta["etid"]
        item["et_name"] = response.meta["et_name"]
        et_state["et_name"] = response.meta["et_name"]
        et_state['etid'] = response.meta["etid"]
        et_state["status"] = 1
        et_state["update_time"] = int(time.time())
        return et_state, item

    def parse_et_busi_info(self, response):
        """
        --工商信息解析部分--
        """
        busi_info = ParseEtBusiInfo()
        reg_address = ""
        busi_regi_num = ""
        orga_code = ""
        soci_cred_code = ""
        industry = ""
        oper_period = ""
        et_type = ""
        taxp_lden_num = ""
        approval_date = ""
        taxpayer_qual = ""
        staff_site = ""
        paid_capital = ""
        regis_auth = ""
        part_num = ""
        english_name = ""
        a = response.xpath("//table[contains(@class,'-striped-col')]/tbody/tr")
        for i in range(1, len(a)):
            if i == 8:
                for z in range(1, 2):
                    td = response.xpath("//table[contains(@class,'-striped-col')]/tbody/tr[%s]/td[%s]/text()" % (i, z))
                    td = "".join(td.extract())
                    if td == "注册地址":
                        zhuce = response.xpath(
                            "//table[contains(@class,'-striped-col')]/tbody/tr[%s]/td[%s]/text()" % (i, z + 1))
                        reg_address = "".join(zhuce.extract())
            else:
                for z in range(1, 5, 2):
                    td = response.xpath("//table[contains(@class,'-striped-col')]/tbody/tr[%s]/td[%s]/text()" % (i, z))
                    td = "".join(td.extract())
                    if td == "工商注册号":
                        zhuce = response.xpath(
                            "//table[contains(@class,'-striped-col')]/tbody/tr[%s]/td[%s]/text()" % (i, z + 1))
                        busi_regi_num = "".join(zhuce.extract())
                    if td == "组织机构代码":
                        zhuce = response.xpath(
                            "//table[contains(@class,'-striped-col')]/tbody/tr[%s]/td[%s]/text()" % (i, z + 1))
                        orga_code = "".join(zhuce.extract())
                    if td == "统一社会信用代码":
                        zhuce = response.xpath(
                            "//table[contains(@class,'-striped-col')]/tbody/tr[%s]/td[%s]/text()" % (i, z + 1))
                        soci_cred_code = "".join(zhuce.extract())
                    if td == "公司类型":
                        zhuce = response.xpath(
                            "//table[contains(@class,'-striped-col')]/tbody/tr[%s]/td[%s]/text()" % (i, z + 1))
                        et_type = "".join(zhuce.extract())
                    if td == "纳税人识别号":
                        zhuce = response.xpath(
                            "//table[contains(@class,'-striped-col')]/tbody/tr[%s]/td[%s]/text()" % (i, z + 1))
                        taxp_lden_num = "".join(zhuce.extract())
                    if td == "行业":
                        zhuce = response.xpath(
                            "//table[contains(@class,'-striped-col')]/tbody/tr[%s]/td[%s]/text()" % (i, z + 1))
                        industry = "".join(zhuce.extract())
                    if td == "营业期限":
                        zhuce = response.xpath(
                            "//table[contains(@class,'-striped-col')]/tbody/tr[%s]/td[%s]/span/text()" % (i, z + 1))
                        oper_period = "".join(zhuce.extract())
                    if td == "核准日期":
                        zhuce = response.xpath(
                            "//table[contains(@class,'-striped-col')]/tbody/tr[%s]/td[%s]/text/text()" % (i, z + 1))
                        approval_date = "".join(zhuce.extract())
                    if td == "纳税人资质":
                        zhuce = response.xpath(
                            "//table[contains(@class,'-striped-col')]/tbody/tr[%s]/td[%s]/text()" % (i, z + 1))
                        taxpayer_qual = "".join(zhuce.extract())
                    if td == "人员规模":
                        zhuce = response.xpath(
                            "//table[contains(@class,'-striped-col')]/tbody/tr[%s]/td[%s]/text()" % (i, z + 1))
                        staff_site = "".join(zhuce.extract())
                    if td == "实缴资本":
                        zhuce = response.xpath(
                            "//table[contains(@class,'-striped-col')]/tbody/tr[%s]/td[%s]/text()" % (i, z + 1))
                        paid_capital = "".join(zhuce.extract())
                    if td == "登记机关":
                        zhuce = response.xpath(
                            "//table[contains(@class,'-striped-col')]/tbody/tr[%s]/td[%s]/text()" % (i, z + 1))
                        regis_auth = "".join(zhuce.extract())
                    if td == "参保人数":
                        zhuce = response.xpath(
                            "//table[contains(@class,'-striped-col')]/tbody/tr[%s]/td[%s]/text()" % (i, z + 1))
                        part_num = "".join(zhuce.extract())
                    if td == "英文名称":
                        zhuce = response.xpath(
                            "//table[contains(@class,'-striped-col')]/tbody/tr[%s]/td[%s]/text()" % (i, z + 1))
                        english_name = "".join(zhuce.extract())
        if busi_regi_num:
            busi_info["reg_address"] = reg_address
            busi_info["busi_regi_num"] = busi_regi_num
            busi_info["orga_code"] = orga_code
            busi_info["soci_cred_code"] = soci_cred_code
            busi_info["industry"] = industry
            busi_info["oper_period"] = oper_period
            busi_info["et_type"] = et_type
            busi_info["taxp_lden_num"] = taxp_lden_num
            busi_info["approval_date"] = approval_date
            busi_info["taxpayer_qual"] = taxpayer_qual
            busi_info["staff_site"] = staff_site
            busi_info["paid_capital"] = paid_capital
            busi_info["regis_auth"] = regis_auth
            busi_info["part_num"] = part_num
            busi_info["english_name"] = english_name
            busi_info["etid"] = response.meta["etid"]
            busi_info["et_name"] = response.meta["et_name"]
            return busi_info
        else:
            logger.info("信息不存在，不进行存储")

    def parse_et_shareholder_info(self, response):
        """
        --股东信息解析部分--
        """
        container_holder = response.xpath("//div[@id='_container_holder']/table/tbody/tr").extract()
        con_num = len(container_holder)
        for i in range(1, con_num + 1):
            et_shareholder_info = ParseEtShareholderInfo()
            et_shareholder_info["shareholder"] = ""
            et_shareholder_info["funded_ratio"] = ""
            et_shareholder_info["funded_num"] = ""
            et_shareholder_info["funded_time"] = ""
            tr = response.xpath("//div[@id='_container_holder']/table/tbody/tr[%s]/td" % i).extract()
            # print "fd", len(tr)
            for z in range(2, len(tr) + 1):
                if z == 2:
                    td = response.xpath(
                        "//div[@id='_container_holder']/table/tbody/tr[%s]/td[%s]//div[@class='dagudong']/a/text()" % (
                            i, z)).extract()
                    shareholder = "".join(td)  # 股东
                if z == 3 or z == 4 or z == 5:
                    td = response.xpath(
                        "//div[@id='_container_holder']/table/tbody/tr[%s]/td[%s]//span/text()" % (i, z)).extract()
                    td_format = "".join(td)
                    if z == 3:
                        funded_ratio = td_format  # 出资比例
                    if z == 4:
                        funded_num = td_format  # 认缴出资
                    if z == 5:
                        funded_time = td_format  # 出资时间
            if shareholder != "":
                et_shareholder_info["shareholder"] = shareholder
                et_shareholder_info["funded_ratio"] = funded_ratio
                et_shareholder_info["funded_num"] = funded_num
                et_shareholder_info["funded_time"] = funded_time
                et_shareholder_info["etid"] = response.meta["etid"]
                et_shareholder_info["et_name"] = response.meta["et_name"]
                return et_shareholder_info
            else:
                logger.info("信息不存在，不进行存储")

    def parse_et_foreign_investment_info(self, response):
        """
        --对外投资解析部分--
        """
        container_invest = response.xpath("//div[@id='_container_invest']/table/tbody/tr").extract()
        con_num = len(container_invest)
        for i in range(1, con_num + 1):
            et_foreign_investment_info = ParseEtForeignInvestmentInfo()
            tr = response.xpath("//div[@id='_container_invest']/table/tbody/tr[%s]/td" % i).extract()
            for z in range(2, len(tr) + 1):
                td = response.xpath(
                    "//div[@id='_container_invest']/table/tbody/tr[%s]/td[%s]//text()" % (i, z)).extract()
                td_format = "".join(td)
                if z == 2:
                    invested_et_name = td_format  # 被投资公司名称
                if z == 3:
                    td = response.xpath(
                        "//div[@id='_container_invest']/table/tbody/tr[%s]/td[%s]/span/a/text()" % (i, z)).extract()
                    invested_legal_rp = "".join(td)  # 被投资法定代表人
                if z == 4:
                    regi_capt = td_format  # 注册资本
                if z == 5:
                    invest_ratio = td_format  # 投资占比
                if z == 6:
                    regi_time = td_format  # 注册时间
                if z == 7:
                    states = td_format  # 状态
            if invested_et_name:
                et_foreign_investment_info["invested_et_name"] = invested_et_name
                et_foreign_investment_info["invested_legal_rp"] = invested_legal_rp
                et_foreign_investment_info["regi_capt"] = regi_capt
                et_foreign_investment_info["invest_ratio"] = invest_ratio
                et_foreign_investment_info["regi_time"] = regi_time
                et_foreign_investment_info["states"] = states
                et_foreign_investment_info["etid"] = response.meta["etid"]
                et_foreign_investment_info["et_name"] = response.meta["et_name"]
                return et_foreign_investment_info
            else:
                logger.info("信息不存在，不进行存储")

    def parse_et_branch_office_info(self, response):
        """
        --分支机构解析部分--
        """
        container_branch = response.xpath("//div[@id='_container_branch']/table/tbody/tr").extract()
        con_num = len(container_branch)
        for i in range(1, con_num + 1):
            et_branch_office = ParseEtBranchOfficeInfo()
            tr = response.xpath("//div[@id='_container_branch']/table/tbody/tr[%s]/td" % i).extract()
            for z in range(2, len(tr) + 1):
                td = response.xpath(
                    "//div[@id='_container_branch']/table/tbody/tr[%s]/td[%s]//text()" % (i, z)).extract()
                td_format = "".join(td)
                if z == 2:
                    fz_et_name = td_format  # 企业名称
                if z == 3:
                    td = response.xpath(
                        "//div[@id='_container_invest']/table/tbody/tr[%s]/td[%s]/span/a/text()" % (i, z)).extract()
                    fz_principal = "".join(td)  # 负责人
                if z == 4:
                    fz_regi_time = td_format  # 注册时间
                if z == 5:
                    fz_status = td_format  # 状态
            if fz_et_name:
                et_branch_office["fz_et_name"] = fz_et_name
                et_branch_office["fz_principal"] = fz_principal
                et_branch_office["fz_regi_time"] = fz_regi_time
                et_branch_office["fz_status"] = fz_status
                et_branch_office["et_name"] = response.meta["et_name"]
                et_branch_office["etid"] = response.meta["etid"]
                return et_branch_office
            else:
                logger.info("信息不存在，不进行存储")

    def parse_wechat_list_info(self, response):
        """
        --微信公众号解析部分--
        """
        wechat_list = response.xpath("//div[@class='wechat-list']/div[@class='wechat']").extract()
        for i in range(1, len(wechat_list) + 1):
            wechat_list_info = ParseWechatListInfo()
            wechat = response.xpath(
                "//div[@class='wechat-list']/div[@class='wechat'][%s]/div[@class='content']/div" % i).extract()
            for z in range(1, len(wechat) + 1):
                zz = response.xpath(
                    "//div[@class='wechat-list']/div[@class='wechat'][%s]/div[@class='content']/div[%s]//text()" % (
                        i, z)).extract()
                if z == 1:
                    wechat_name = "".join(zz)  # 微信名
                if z == 2:
                    wechat_code = response.xpath(
                        "//div[@class='wechat-list']/div[@class='wechat'][%s]/div[@class='content']/div[%s]/span[2]/text()" % (
                            i, z)).extract_first()  # 微信号
                if z == 3:
                    weinfo = response.xpath("//div[@class='wechat-list']/div[@class='wechat'][%s]/div[@class='content']"
                                            "/div[%s]//script/text()" % (i, z)).extract()
                    weinfo = ''.join(weinfo)
                    weinfo = eval(weinfo)
                    wechat_info = weinfo["recommend"]  # 介绍
            if wechat_code:
                wechat_list_info["wechat_name"] = wechat_name
                wechat_list_info["wechat_code"] = wechat_code
                wechat_list_info["wechat_info"] = wechat_info
                wechat_list_info["etid"] = response.meta["etid"]
                wechat_list_info["et_name"] = response.meta["et_name"]
                return wechat_list_info
            else:
                logger.info("信息不存在，不进行存储")

    def parse_et_container_copyright_info(self, response):
        """
        --软件著作权解析部分-- _container_copyright
        """
        container_copyright = response.xpath("//div[@id='_container_copyright']/table/tbody/tr").extract()
        con_num = len(container_copyright)
        # print(con_num)
        for i in range(1, con_num + 1):
            container_copyright_info = ParseEtContaInerCopyrightInfo()
            tr = response.xpath("//div[@id='_container_copyright']/table/tbody/tr[%s]/td" % i).extract()
            for z in range(2, len(tr) + 1):
                td = response.xpath(
                    "//div[@id='_container_copyright']/table/tbody/tr[%s]/td[%s]//text()" % (i, z)).extract()
                td_forma = ''.join(td)
                if z == 2:
                    appro_date = td_forma  # 批准日期
                if z == 3:
                    software_name = td_forma  # 软件全称
                if z == 4:
                    software_shortname = td_forma  # 软件简称
                if z == 5:
                    rege_num = td_forma  # 登记号
                if z == 6:
                    type_num = td_forma  # 类别号
                if z == 7:
                    version_num = td_forma  # 版本号
            if software_name:
                container_copyright_info["appro_date"] = appro_date
                container_copyright_info["software_name"] = software_name
                container_copyright_info["software_shortname"] = software_shortname
                container_copyright_info["rege_num"] = rege_num
                container_copyright_info["type_num"] = type_num
                container_copyright_info["version_num"] = version_num
                container_copyright_info["etid"] = response.meta["etid"]
                container_copyright_info["et_name"] = response.meta["et_name"]
                return container_copyright_info
            else:
                logger.info("信息不存在，不进行存储")

    def parse_et_container_icp_info(self, response):
        """
        --网站备案解析部分-- _container_icp
        """
        container_icp = response.xpath("//div[@id='_container_icp']/table/tbody/tr").extract()
        con_num = len(container_icp)
        # print(con_num)
        for i in range(1, con_num + 1):
            et_container_icp_info = ParseEtContaInerIcpInfo()
            tr = response.xpath("//div[@id='_container_icp']/table/tbody/tr[%s]/td" % i).extract()
            for z in range(2, len(tr) + 1):
                td = response.xpath("//div[@id='_container_icp']/table/tbody/tr[%s]/td[%s]//text()" % (i, z)).extract()
                td_form = ''.join(td)
                if z == 2:
                    review_time = td_form  # 审核时间
                if z == 3:
                    website_name = td_form  # 网站名称
                if z == 4:
                    website_url = td_form  # 网站首页
                if z == 5:
                    domain_name = td_form  # 域名
                if z == 6:
                    case_num = td_form  # 备案号
                if z == 7:
                    status = td_form  # 状态
                if z == 8:
                    unit_nature = td_form  # 单位性质
            if website_url:
                et_container_icp_info["review_time"] = review_time
                et_container_icp_info["website_name"] = website_name
                et_container_icp_info["website_url"] = website_url
                et_container_icp_info["domain_name"] = domain_name
                et_container_icp_info["case_num"] = case_num
                et_container_icp_info["status"] = status
                et_container_icp_info["unit_nature"] = unit_nature
                et_container_icp_info["et_name"] = response.meta["et_name"]
                et_container_icp_info["etid"] = response.meta["etid"]
                return et_container_icp_info
            else:
                logger.info("信息不存在，不进行存储")

    def parse_et_trademark_info(self, response):

        """
        --商标解析部分-- _container_icp
        """
        container_tmInfo = response.xpath("//div[@id='_container_tmInfo']//table/tbody/tr").extract()
        con_num = len(container_tmInfo)
        for i in range(1, con_num + 1):
            et_trademark_info = ParseEtTrademarkInfo()
            tr = response.xpath("//div[@id='_container_tmInfo']//table/tbody/tr[%s]/td" % i).extract()
            for z in range(2, len(tr) + 1):
                td = response.xpath(
                    "//div[@id='_container_tmInfo']//table/tbody/tr[%s]/td[%s]//text()" % (i, z)).extract()
                td_form = ''.join(td)
                if z == 2:
                    apply_time = td_form  # 申请时间
                if z == 3:
                    a = response.xpath(
                        "//div[@id='_container_tmInfo']//table/tbody/tr[%s]/td[%s]/div/div/img/@data-src" % (
                            i, z)).extract()  # 商标url
                    trademark_url = "".join(a)
                if z == 4:
                    trademark_name = td_form  # 商标名字
                if z == 5:
                    reg_num = td_form  # 注册号
                if z == 6:
                    apply_type = td_form  # 类别
                if z == 7:
                    process_status = td_form  # 流程状态
            if reg_num:
                et_trademark_info["apply_time"] = apply_time
                et_trademark_info["trademark_url"] = trademark_url
                et_trademark_info["trademark_name"] = trademark_name
                et_trademark_info["reg_num"] = reg_num
                et_trademark_info["apply_type"] = apply_type
                et_trademark_info["process_status"] = process_status
                et_trademark_info["etid"] = response.meta["etid"]
                et_trademark_info["et_name"] = response.meta["et_name"]
                return et_trademark_info
            else:
                logger.info("信息不存在，不进行存储")

    def parse_et_rongzi_info(self, response):
        """
        --融资信息解析部分-- _container_icp
        """
        container_rongzi = response.xpath("//div[@id='_container_rongzi']/table/tbody/tr").extract()
        con_num = len(container_rongzi)
        # print(con_num)
        for i in range(1, con_num + 1):
            et_rongzi_info = ParseEtRongziInfo()
            tr = response.xpath("//div[@id='_container_rongzi']//table/tbody/tr[%s]/td" % i).extract()
            # print "fd", len(tr)
            for z in range(2, len(tr) + 1):
                td = response.xpath(
                    "//div[@id='_container_rongzi']//table/tbody/tr[%s]/td[%s]//text()" % (i, z)).extract()
                td_form = ''.join(td)
                if z == 2:
                    rongzi_time = td_form  # 融资时间
                if z == 3:
                    rounds = td_form  # 轮次
                if z == 4:
                    valuation = td_form  # 估值
                if z == 5:
                    amount = td_form  # 金额
                if z == 6:
                    proportion = td_form  # 比例
                if z == 7:
                    investor = td_form  # 投资方
                if z == 8:
                    news_source = td_form  # 新闻来源
            if investor:
                et_rongzi_info["rongzi_time"] = rongzi_time
                et_rongzi_info["rounds"] = rounds
                et_rongzi_info["valuation"] = valuation
                et_rongzi_info["amount"] = amount
                et_rongzi_info["proportion"] = proportion
                et_rongzi_info["investor"] = investor
                et_rongzi_info["news_source"] = news_source
                et_rongzi_info["et_name"] = response.meta["et_name"]
                et_rongzi_info["etid"] = response.meta["etid"]
                return et_rongzi_info
            else:
                logger.info("信息不存在，不进行存储")


import json
import os
import socket
import sys
import textwrap
import numpy as np
import random
import time
import numpy.random as npyrnd
from numpy import dtype as npydtype
from coordinator_udp_sender import Smart_UDP_Sender_Socket
from keras import models, layers
from typing import List
# from coordinator_udp_receiver import AlwaysOpenUDP_Receiver
# import paramiko
import select
import zipfile
import tensorflow as tf
import csv
import pathlib
import cv2
import pickle
import communication
import srudp

useful_characters = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f",  "g", "h", "i", "j", "k", "l", "m", "n", "o", "p",  "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "{", "}", "[", "]", ":", "-", ",", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", " ", "\"", "-", ".", ":"]

rand_string = ""
test_string = "test_string"
rand_text_length = 4500
string_length_per_packet = 1500

for i in range(rand_text_length):
    rand_string += "1"
rand_packets = textwrap.wrap(rand_string, string_length_per_packet)

def read_json_file():
    some_file_handle = open("results_22.json", "r")
    my_json_obj = json.load(some_file_handle)
    print(my_json_obj)

def read_already_open_file():
    some_file = open("test_file.txt", "r")
    some_file_copy = open("test_file.txt", "r")
    print(some_file)
    print(some_file_copy)
    print("Done")

def change_directory_test():
    curr_dir = os.getcwd()
    print(curr_dir)
    os.chdir("../images/")
    curr_dir = os.getcwd()
    print(curr_dir)
    print(os.curdir)

def create_numpy_array():
    some_numpy_ndarray = np.zeros(shape=(1, 10), dtype=float)
    some_numpy_ndarray_converted_to_list = some_numpy_ndarray.tolist()
    print(some_numpy_ndarray_converted_to_list)

def add_two_numpy_arrays():
    first_numpy_array = np.zeros(shape=(1, 10), dtype=float)
    second_numpy_array = np.arange(10)
    print(first_numpy_array + second_numpy_array)

def get_multidimensional_array_length():
    some_dumb_array = list(list([]))
    some_dumb_array.append([1, 2, 3, 4])
    some_dumb_array.append([3, 4, 5, 6, 7])
    print(some_dumb_array[0])
    print(len(some_dumb_array[0]))

def generate_random_numbers():
    nanoseconds_time = time.perf_counter_ns()
    random.seed(nanoseconds_time)
    print(random.random(), random.random(), random.random())
    random.seed(nanoseconds_time)
    print(random.random(), random.random(), random.random())
    print(random.randint(15, 20))
    print(random.sample([1, 2, 3, 4, 5, 6, 7, 8], 3))
    a = [1, 2, 3, 4, 5, 6, 7]
    random.shuffle(a)
    print(a)
    print("Betavariate: ", random.betavariate(2, 2))
    print("Gauss: ", random.gauss(0, 1))
    npyrnd.seed(15)
    print(npyrnd.rand(3))
    npyrnd.seed(15)
    print(npyrnd.rand(3))
    print(npyrnd.randint(low=55, high=100, dtype=int))

def send_custom_payload():
    remote_port = 5757
    remote_ip = "192.168.1.41"
    remote_address_tuple = (remote_ip, remote_port)
    custom_payload = "Some payload"
    packet_data = custom_payload.encode()
    some_socket = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM)
    expected_byte_length = 1500
    packet = bytearray()
    packet.extend(bytearray(packet_data))
    while True:
        try:
            for i in range(expected_byte_length - len(packet_data) - 8):
                packet.extend(str(0).encode())
            some_socket.sendto(packet, remote_address_tuple)
        except socket.timeout:
            continue
        except KeyboardInterrupt:
            break

def send_reliable_data_router(remote_addr: str = "192.168.1.41", remote_port: int = 4040, pkt_rate=50, max_bytes_per_packet=1500, local_port_to_send_from: int = 5757, socket_arg: socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM) = None):

    PktPeriod = 1.0 / pkt_rate  # Period of packet generation
    sleeptime = PktPeriod  # time to sleep between packets

    print("Test source to MK1 on %s:%d" % (remote_addr, remote_port))
    rxsock = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM)
    if socket_arg is None:
        rxsock = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM)
    else:
        rxsock = socket_arg
    rxsock.bind(('', local_port_to_send_from))
    rxsock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
    Target = (remote_addr, remote_port)  # the target address and port
    new_time = time.time()
    start_pkt_num = 0
    last_pkt_num = 0
    pkt_count = 0
    last_update_time = time.time()
    start_time = last_update_time
    UpdatePeriod = 1.0
    PktRatePeriod = 1.0
    all_packets_sent_reliably = False
    try:
        # Main loop
        while not all_packets_sent_reliably:
            pkt_generation_time = time.time()
            pktbuf = bytearray()
            some_string = "abc"
            for i in range(len(some_string)):
                pktbuf.append(ord(some_string[i]) & 0xff)
                print(chr(ord(some_string[i])))
            for i in range(0, max_bytes_per_packet - len(some_string)):
                pktbuf.append(0 & 0xff)

            # print_byte_string(pktbuf)
            print('Total packets transmitted: %d' % (pkt_count + 1))
            # print ('')

            # Transmit the packet to the MK1 via the socket
            rxsock.sendto(pktbuf, Target)

            # Increment packet numbers
            pkt_count = pkt_count + 1

            if new_time - last_update_time > UpdatePeriod:
                tx_pkt_rate = ((pkt_count - last_pkt_num) /
                               (new_time - last_update_time))
                last_pkt_num = pkt_count
                last_update_time = new_time

            last_time = new_time
            new_time = time.time()

            timediff = new_time - last_time

            errtime = (PktPeriod * (pkt_count - start_pkt_num) -
                       (new_time - start_time))
            sleeptime = PktPeriod + errtime

            if new_time - start_time > PktRatePeriod:
                start_time = new_time
                start_pkt_num = pkt_count

            if sleeptime < 0:
                sleeptime = 0
            time.sleep(sleeptime)

        print("Transmitted %d packets" % pkt_count)
        return pkt_count
    except KeyboardInterrupt:
        rxsock.close()
        print('User CTRL-C, stopping packet source')
        sys.exit(1)

    except:
        rxsock.close()
        print("Got exception:", sys.exc_info()[0])
        raise

def send_reliable_data_loopback():
    reliable_packet_index = 0
    sender = Smart_UDP_Sender_Socket(udp_local_address_arg=("127.0.0.1", 5000), udp_target_address_arg=("127.0.0.1", 5250))
    sender.bind_to_local_address()
    sender.initiate_handshake()

def ord_long_string(some_long_string: str = "") -> bytearray:
    returned_list = bytearray()
    for i in range(len(some_long_string)):
        returned_list.append(ord(some_long_string[i]) & 0xff)
    return returned_list

def chr_long_string(some_byte_array_arg: bytearray) -> str:
    temp_str = ""
    some_bytes_object = bytes(some_byte_array_arg)
    some_bytes_object_decoded = some_bytes_object.decode()
    print()
    return temp_str

def ack_router_test():
    sender = Smart_UDP_Sender_Socket(udp_local_address_arg=("192.168.1.57", 5000),
                                     udp_target_address_arg=("192.168.1.31", 5757))
    sender.bind_to_local_address()
    sender.send_ACK_to_router(0)

def send_test_string():
    sender = Smart_UDP_Sender_Socket(udp_local_address_arg=("192.168.1.57", 4002),
                                     udp_target_address_arg=("192.168.1.31", 4000))
    sender.bind_to_local_address()
    sender.send_single_packet_to_router(curr_packet=test_string)

def keep_useful_characters(str_to_parse: str = ""):
    str_to_parse_len = len(str_to_parse)
    index = 0
    while index < str_to_parse_len:
        if str_to_parse[index] in useful_characters:
            index += 1
            continue
        else:
            str_to_parse = str_to_parse.replace(str_to_parse[index], "")
            str_to_parse_len = len(str_to_parse)
    return str_to_parse

def generate_list_of_floats_of_certain_length(length: int = 100) -> list:
    some_list = []
    for i in range(length):
        some_list.append(random.random())
    return some_list

def receive_test_string():
    some_socket = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM)
    some_socket.bind(("192.168.1.57", 4002))
    recvd_bytes, recvd_from = some_socket.recvfrom(1500)
    print(recvd_bytes)
    print(recvd_from)
    some_socket.close()
    some_socket = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM)
    some_socket.bind(("192.168.1.57", 4002))
    recvd_bytes, recvd_from = some_socket.recvfrom(1500)
    print(recvd_bytes)
    print(recvd_from)
    some_socket.close()

def remove_indicator_string(main_string: str = "mystr_some_mystr", indicator_string: str = "mystr"):
    main_string = main_string.replace(indicator_string, "")
    return main_string

def strip_away_indicators(input_string: str = "", indicator_string: str = "^||") -> str:
    indicator_found = input_string.find(indicator_string)
    input_string = input_string[indicator_found:]
    indicator_found = input_string.rfind(indicator_string)
    input_string = input_string[:indicator_found]
    indicator_found = input_string.find(indicator_string)
    indicator_found_reverse = input_string.rfind(indicator_string)
    while (indicator_found != indicator_found_reverse) and (indicator_found >= 0) and (indicator_found_reverse >= 0):
        input_string = input_string[indicator_found+len(indicator_string):]
        input_string = input_string[:indicator_found_reverse]
        indicator_found = input_string.find(indicator_string)
        indicator_found_reverse = input_string.rfind(indicator_string)
    input_string = input_string.replace(indicator_string, "")
    return input_string

def receive_long_string():
    test_socket = socket.socket()
    test_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    test_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
    test_socket.bind(("127.0.0.1", 5000))
    test_socket.listen()
    received_packets = []
    print(remove_indicator_string())
    while True:
        try:
            test_socket_connection, test_received_from = test_socket.accept()
            print("Connection complete")
            break
        except:
            print("Exception raised...")
            sys.exit(0)
    while True:
        try:
            some_string = test_socket_connection.recv(2500)
            if type(some_string) is not bytes:
                print(type(some_string))
            if some_string is not None:
                if "mystr" in some_string.decode():
                    print("Received a string of length: ", len(some_string.decode().replace("mystr", "")))
                    print(some_string.decode().replace("mystr", ""))
                    received_packets.append(some_string.decode().replace("mystr", ""))
                    print("Received packet count: ", len(received_packets))
                    print("Received packets cumulative length: ", len("".join(received_packets)))

        except KeyboardInterrupt:
            sys.exit(0)
        except TypeError:
            continue
        except:
            print(sys.exc_info())
            time.sleep(1.0)
            continue

def neural_net():
    x_train_new = np.load("images50_percent_training_data.npy")
    y_train = np.load("labels50_percent_training_data.npy")
    x_test_new = np.load("images50_percent_testing_data.npy")
    y_test = np.load("labels50_percent_testing_data.npy")
    model = models.Sequential()
    model.add(layers.Conv2D(16, (3, 3), activation='relu', input_shape=(200, 200, 1)))
    model.add(layers.MaxPooling2D((2, 2)))
    model.add(layers.Conv2D(32, (3, 3), activation='relu'))
    model.add(layers.MaxPooling2D((2, 2)))
    model.add(layers.Flatten())
    model.add(layers.Dense(128, activation='relu'))
    model.add(layers.Dense(2))
    model.summary()
    model.compile(optimizer='adam', loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
                  metrics=['accuracy'])
    history = model.fit(x_train_new, y_train, epochs=1, validation_data=(x_test_new, y_test))
    model_json = model.to_json()
    print(model_json)
    print(type(model_json))
    with open("model.json", "w") as json_file:
        json_file.write(model_json)
    # serialize weights to HDF5
    model.save_weights("model.h5")
    print("Saved model to disk")
    json_file = open('model.json', 'r')
    loaded_model_json = json_file.read()
    json_file.close()
    loaded_model = models.model_from_json(loaded_model_json)
    # load weights into new model
    loaded_model.load_weights("model.h5")
    print("Loaded model from disk")
    print(loaded_model)
    print(type(loaded_model))
    print(loaded_model.weights)
    print(loaded_model.layers)
    list_of_files_to_zip = ["model.h5", "model.json"]
    with zipfile.ZipFile('model_files.zip', 'w') as zipMe:
        for file in list_of_files_to_zip:
            zipMe.write(file, compress_type=zipfile.ZIP_DEFLATED)
    some_file = open("model_files.zip", "rb")
    my_bytes = some_file.read()
    print(len(my_bytes))
    sending_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sending_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    sending_socket.bind(("127.0.0.1", 5000))
    sending_socket.connect(("127.0.0.1",5001))
    print("Connected")
    sending_socket.sendall(my_bytes)

def change_float_type():
    some_np_arr = np.random.random((15, 1))
    some_np_arr_f16 = some_np_arr.astype(dtype=np.float16)
    print(some_np_arr)
    print(some_np_arr_f16)
    some_np_arr_f16_f32 = some_np_arr_f16.astype(dtype=np.float32)
    print(some_np_arr_f16_f32)

def edge_detection():
    print(pathlib.Path.home())
    some_image = cv2.imread("photos_common_directory/no20220719-124848-000001fframe0.jpg")
    print(some_image.shape)
    some_image = cv2.resize(some_image, (200,200))
    print(some_image.shape)
    some_input = np.ndarray(shape=(1,200,200,3), dtype=np.float32)
    some_input[0] = some_image
    some_tensor = tf.convert_to_tensor(some_image)
    # output = tf.keras.layers.Conv2D(3, 3, activation='relu', input_shape=some_input.shape)(some_input)
    # print(output.shape)
    img_gray = cv2.cvtColor(some_image, cv2.COLOR_BGR2GRAY)
    print(type(img_gray[0][0]))
    img_blur = cv2.GaussianBlur(img_gray, (3,3), 0)
    edges = cv2.Canny(image=img_blur, threshold1=100, threshold2=200)
    cv2.imshow('Canny Edge Detection', edges)
    cv2.waitKey(0)
    cv2.destroyAllWindows()
    print(edges.shape)
    print(type(edges[0][0]))
    print(edges)

def mini_batch(input_images: np.ndarray, image_labels: np.ndarray, batch_size: int):
    zip_obj = zip(input_images, image_labels)
    samples = random.sample(list(zip_obj), batch_size)
    output_images = np.ndarray(shape=(1, input_images.shape[1], input_images.shape[2]))
    output_labels = np.ndarray(shape=(1, image_labels.shape[1]))
    temp_image = np.ndarray(shape=(1, input_images.shape[1], input_images.shape[2]))
    temp_label = np.ndarray(shape=(1, image_labels.shape[1]))
    output_images[0] = samples[0][0]
    output_labels[0] = samples[0][1]
    for i in range(1, len(samples)):
        temp_image[0] = samples[i][0]
        temp_label[0] = samples[i][1]
        output_images = np.concatenate([output_images, temp_image])
        output_labels = np.concatenate([output_labels, temp_label])
    return output_images, output_labels

def view_images_and_labels(images: np.ndarray, labels: np.ndarray):
    for image, label in zip(images,labels):
        cv2.imshow("Current image", image)
        print("Label is: ", label)
        pressed_key = cv2.waitKey(0)
        if pressed_key == ord("q"):
            cv2.destroyAllWindows()
            break
        cv2.destroyAllWindows()

def receive_file_size(some_socket: socket.socket, file_length_keyword: str):
    while True:
        print("Started receival")
        try:
            bytez = some_socket.recv(1500)
            try:
                decoded_bytes = bytez.decode()
            except:
                continue
            print(decoded_bytes)
            if file_length_keyword in decoded_bytes:
                print("Prefix found")
                print(type(decoded_bytes))
                file_len = int(decoded_bytes.replace(file_length_keyword, ""))
                print(file_len)
                return file_len
            else:
                print("Did not get file size but got: ", decoded_bytes)
                continue
        except:
            pass

def receive_file(some_socket:socket.socket, file_length: int, packet_size: int = 1500):
    bytes_buffer = bytearray()
    while len(bytes_buffer) < file_length:
        recvd_bytes = some_socket.recv(packet_size)
        bytes_buffer.extend(recvd_bytes)
        print("Received bytes of length: ", len(recvd_bytes))
    return bytes_buffer

def receive_ID(some_socket: socket.socket, ID_keyword: str):
    recvd_bytes = some_socket.recv(1500)
    return int(recvd_bytes.decode().replace(ID_keyword, ""))

if __name__ == "__main__":
    my_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    my_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    my_socket.bind(("127.0.0.1", 2500))
    print("Socket bound")
    file_length = receive_file_size(my_socket, "file_len_")
    my_file_bytes = receive_file(my_socket, file_length)
    pass


from datetime import datetime

from setuptools._vendor import more_itertools

class Apriori:
    def __init__(self, transactions, min_support, min_confidence, len_max):
        self.transactions = transactions
        self.min_support = min_support
        self.min_confidence = min_confidence
        self.len_max = len_max
        self.sum_tran = len(transactions)
        self.list_frequent_itemsets = []
        self.l_conf = []
        self.temp = []
        self.l_support = []

    def time(self, s1, s2):
        s1, s2 = str(s1), str(s2)
        hour1 = int(s1[0:2])
        hour2 = int(s2[0:2])
        min1 = int(s1[3:5])
        min2 = int(s2[3:5])
        second1 = int(s1[6:8])
        second2 = int(s2[6:8])
        return (hour2 - hour1) * 3600 + (min2 - min1) * 60 + (second2 - second1)

    #toileanh
    def init_itemsets(self):
        itemsets = set()
        for i in self.transactions:
            for item in i:
                itemsets.add(item)
        results = []
        for i in list(itemsets):
            results.append([i])
        return results

    def create_new_candidate(self, itemsets):
        len_itemset = len(itemsets[0]) + 1
        items = set()
        for i in itemsets:
            for item in i:
                items.add(item)
        items = list(items)
        sinhtohop = more_itertools.distinct_combinations(items, len_itemset)
        candidate_itemsets = []
        for i in sinhtohop:
            candidate_itemsets.append(list(i))
        return candidate_itemsets

    def frequent_itemset(self, candidate_itemsets):
        frequent_itemsets = []
        for i in candidate_itemsets:
            if self.support_itemset(i) >= self.min_support:
                frequent_itemsets.append(i)
        return frequent_itemsets

    def create_frequent_itemsets(self):
        if (self.len_max < 1):
            print("Số item trong Frequent set phải lớn hơn 1 :) ")
        else:
            time1 = datetime.now().time()
            frequent_itemset = []
            itemset = self.init_itemsets()
            frequent_itemsets = self.frequent_itemset(itemset)
            for i in range(self.len_max - 1):
                new_candidates = self.create_new_candidate(
                    frequent_itemsets)
                frequent_itemsets = self.frequent_itemset(
                    new_candidates)
                if (len(frequent_itemsets) == 0):
                    print("Độ dài tối đa của  : %d" % (i + 1))
                    time2 = datetime.now().time()
                    print("Thời gian supported_itemsets len %d: " % (i + 2), self.time(time1, time2))
                    return frequent_itemset
                    break
                else:
                    frequent_itemset.append(frequent_itemsets)
                    time2 = datetime.now().time()
                    print("Time supported_itemsets len %d: " % (i + 2), self.time(time1, time2))
            return frequent_itemset

    def create_frequent_itemset_max(self):
        itemset = self.init_itemsets()
        frequent_itemsets = self.frequent_itemset(itemset)
        print("Len new_item 1 :", len(itemset))
        print("new_items 1: ", itemset)
        print("Len supported_itemset 1: ", len(frequent_itemsets))
        print("frequent_itemset 1: ", frequent_itemsets)
        i = 0
        while (True):
            new_candidates = self.create_new_candidate(frequent_itemsets)
            frequent_itemsets = self.frequent_itemset(new_candidates)
            if len(frequent_itemsets) > 0:
                print(
                    "================================================================================================")
                print("Len new_item %d :" % (i + 2), len(new_candidates))
                print("new_items %d : " % (i + 2), new_candidates)
                print("Len supported_itemset %d : " % (i + 2), len(frequent_itemsets))
                print("frequent_itemset %d : " % (i + 2), frequent_itemsets)
                i += 1
            else:
                return i + 1, frequent_itemsets

    def support_itemset(self, itemset):
        dem = 0
        for i in self.transactions:
            if all(itemm in i for itemm in itemset):
                dem += 1
        return dem / self.sum_tran

    def confident_itemset(self, X, Y):
        XY = []
        XY.extend(X)
        XY.extend(Y)
        return round(float(self.support_itemset(XY) / self.support_itemset(X)), 3)

    def confident_supported_itemset(self, left, right):
        self.rules(left, right)
        if len(left) == 1:
            pass
        else:
            for i in more_itertools.distinct_combinations(left, 1):
                x, y = left.copy(), right.copy()
                y.append(list(i)[0])
                x.remove(list(i)[0])
                if self.confident_itemset(x, y) >= self.min_confidence:
                    self.confident_supported_itemset(x, y)

    def rules(self, left, right):
        if len(right) > 0:
            for i in self.temp:
                if all(itemm in i for itemm in left) and len(left) == len(i):
                    return
            self.list_frequent_itemsets.append([left, right])
            self.l_conf.append(self.confident_itemset(left, right))
            c = []
            c.extend(left)
            c.extend(right)
            self.l_support.append(self.support_itemset(c))
            self.temp.append(left)

    def gen_rules(self, frequent_itemsets):
        for itemset in frequent_itemsets:
            self.temp = []
            left = itemset.copy()
            right = []
            self.confident_supported_itemset(left, right)
        return self.list_frequent_itemsets, self.l_support, self.l_conf