#the method follows Eqn. 7 in Zhou's paper
#the variance of the chi-square minimization fitting of the antisymmetric cross power spectrum dipole

import numpy as np
from scipy import interpolate, integrate
from scipy.stats import chi2
from scipy.special import erfinv, erfcinv
#*****************************#
## Cosmological Parameters ##
#*****************************#
HubblePara = 0.678
## no unit
HubbleLength = 3000
## in unit Mpc/h

OMega_m = 0.306
OMega_r = 8.6e-5
OMega_lambda = 1.0-OMega_m-OMega_r
## Constants of LambdaCDM, no units

nu_21 = 1420.405752
nu_CO = 115270
## Frequency (MHz) of emission lines at rest

lambda_21 = 0.2110611405
lambda_CO = 0.00261
## Wavelength (meters) of emission lines at rest

def E(z):
    """
    The scale factor which relates the current Hubble parameter H0 with the Hubble parameter at redshift z H(z) by H(z) = E(z) H0.
    """
    return np.sqrt( OMega_m*(1+z)**3 + OMega_r*(1+z)**4 + OMega_lambda )

def X(z):
    """
    The radial co-moving distance (Mpc/h) at redshift z.
    """
    integral = integrate.quad(lambda x: 1./E(x), 0, z)[0]
    integral *= HubbleLength
    return integral

def Y(z, nu_rest):
    """
    Y = d r_parallel (Mpc/h) / d v (MHz) = Hubblelength * (1+z)^2/E(z)/nu_rest.

    """
    return HubbleLength * (1.+z) **2. / E(z) / nu_rest

#*****************************#
## Main Functions ##
#*****************************#

def significance_level(signal_noise_ratio_square_sum, free_degree):
    """
    Calculate the significance level for claiming a detection.
    Assume the square sum of signal-noise-ratio from multiple measurements follows a chi-square distribution,
    then get the p-value and convert it to the equivalent significance level of a normal distribution.
    """
    p = chi2.cdf(signal_noise_ratio_square_sum, free_degree)
    significance = erfinv(p)
    if significance == float('inf'):
        p_comp = integrate.quad(lambda x: chi2.pdf(x, free_degree), signal_noise_ratio_square_sum, np.inf, epsrel = 1e-5)[0]
        significance = erfcinv(p_comp)
    return significance

def reliable_k_range(z, Omega_beam_CO, Omega_patch, delta_nu_CO, delta_D, L_min_21, L_max_21, delta_nu_21):
    """
    Omega_beam_CO : [radian]^2, the instant solid-angle on the sky subtented by one beam of CO detection
    Omega_patch : [radian]^2, the solid-angle on the sky subtented by a box
    delta_nu_CO : MHz, the width of single frequency channel of CO detection
    delta_D : Mpc/h, the length of the observation box along the LoS direction
    L_min_21: meters, the minimum baseline length of 21cm observation
    L_max_21: meters, the maximum baseline length of 21cm observation
    delta_nu_21 : MHz, the width of single frequency channel of 21 detection
    """
    k_perp_max_CO = 2.*np.pi / (X(z) * np.sqrt(Omega_beam_CO))
    k_perp_min_CO = 2.*np.pi / (X(z) * np.sqrt(Omega_patch))
    k_para_max_CO = 2.*np.pi / (Y(z, nu_CO) * delta_nu_CO)
    k_perp_max_21 = 2. * np.pi * L_max_21 / lambda_21 / (1 + z) / X(z);
    k_perp_min_21 = 2. * np.pi * L_min_21 / lambda_21 / (1 + z) / X(z);
    k_para_max_21 = 2. * np.pi / (Y(z, nu_21) * delta_nu_21)

    k_perp_max = min(k_perp_max_CO, k_perp_max_21);     k_perp_min = max(k_perp_min_CO, k_perp_min_21)
    k_para_max = min(k_para_max_CO, k_para_max_21);     k_para_min = 2.*np.pi / delta_D
    return [k_perp_min, k_perp_max, k_para_min, k_para_max] # in the unit of h/Mpc

def W(k_perp, k_para, z, nu_rest, Omega_beam, delta_nu):
    """
    The modified factor due to limited resolution
    -----------
    Parameters:
    -----------
    k_perp : h/Mpc, k perpendicular to the LoS direction
    k_para : h/Mpc, k parallel to the LoS direction
    z : redshift
    nu_rest : MHz, the frequency of the emission line at its rest frame
    Omega_beam : [radian]^2, the instant solid-angle on the sky subtented by one beam
    delta_nu : MHz, the width of single frequency channel
    """
    sigma_perp = X(z)*np.sqrt(Omega_beam) / (2 * np.pi)
    sigma_para = Y(z, nu_rest)*delta_nu / (2 * np.pi)
    modification = np.exp(sigma_perp**2 * k_perp**2 + sigma_para**2 * k_para**2)
    return modification

def error_CO_auto(k, delta_k, theta, delta_theta, P_k, z, Tsys, Omega_patch, NUM_PATCH, delta_D, Omega_beam, delta_nu, Nfeeds, t_int):
    """
    Calculate the cosmic variance on CO auto power spectrum at certain (k, theta)
    -----------
    Parameters:
    -----------
    k : h/Mpc
    delta_k : h/Mpc
    theta: radians
    delta_theta: radians
    P_k : [muK]^2[Mpc/h]^3, the auto power spectrum of CO line
    z : redshift
    Tsys : muK
    Omega_patch : [radian]^2, the solid-angle on the sky subtented by a box
    NUM_PATCH : number of patches in a survey
    delta_D : Mpc/h, the length of the observation box along the LoS direction
    Omega_beam : [radian]^2, the instant solid-angle on the sky subtented by one beam
    delta_nu : MHz, the width of single frequency channel
    Nfeeds : the number of independent feeds, like different beams, polarizations ...
    t_int : second, the total integration time
    --------
    Returns:
    --------
    sigma_error: [muK]^2[Mpc/h]^3
    sigma_cosmic_variance : [muK]^2[Mpc/h]^3
    sigma_noise : [muK]^2[Mpc/h]^3
    Nmodes :
    """

    Omega_survey = NUM_PATCH * Omega_patch
    V_survey = X(z)**2*Omega_survey*delta_D
    t_int *= 1e6

    k_perp = k * np.sin(theta); k_para = k * np.cos(theta)
    N_modes = V_survey / (4. * np.pi**2) * k**2 * delta_k * np.sin(theta) * delta_theta
    sigma_noise = W(k_perp, k_para, z, nu_CO, Omega_beam, delta_nu) * X(z)**2*Y(z, nu_CO)*Tsys**2*Omega_survey/Nfeeds/t_int/N_modes**0.5
    sigma_cosmic_variance = P_k / N_modes**0.5
    sigma_error = sigma_cosmic_variance + sigma_noise

    return sigma_error, sigma_cosmic_variance, sigma_noise, N_modes

def n_u(u, wavelength):
    """
    The number density of redundant baselines in uv space, assuming a SKA-low core configuration.
    u : no unit, L/wavelength
    wavelength : m
    """
    baseline_number_density = [0.0609823,0.05809261,0.05520293,0.05231325,0.04942356,\
    0.04673317,0.04404277,0.04135238,0.03866198,0.03597158,0.03328119,0.03079008,\
    0.02829897,0.02580787,0.02331676,0.02102494,0.01873312,0.01644131,0.01414949,\
    0.01205696,0.00996443,0.00807119,0.00617795,0.00468328,0.00318862,0.00209253,0.00099644,0.00049822,0.]
    baseline_length = [ 35, 70, 105,  140,  175,  210,  245,  280,  315,  350,  385, 420,  455,  \
    490,  525,  560,  595,  630,  665,  700,  735,  770, 805,  840,  875,  910,  945,  980, 1015 ]
    f = interpolate.interp1d(baseline_length, baseline_number_density, kind = "cubic", fill_value="extrapolate")
    return f(u*wavelength)*wavelength**2

def error_21_auto(k, delta_k, theta, delta_theta, P_k, z, Tsys, Omega_patch, NUM_PATCH, Omega_beam, delta_D, Nfeeds, t_int):
    """
    Calculate the error bar on 21cm auto power spectrum at certain (k, theta)
    -----------
    Parameters:
    -----------
    k : h/Mpc
    delta_k : h/Mpc, the width of the k-shell
    theta: radians
    delta_theta: radians
    P_k : [muK]^2[Mpc/h]^3
    z : redshift
    Tsys : muK
    delta_k : h/Mpc, the width of the k-shell
    Omega_patch : [radian]^2, the solid-angle on the sky subtented by a box
    NUM_PATCH : number of patches in a survey
    Omega_beam : [radian]^2, the instant solid-angle on the sky subtented by one beam
    delta_D : Mpc/h, the length of the observation box along the LoS direction
    Nfeeds : the number of independent feeds, like different beams, polarizations ...
    t_int : second, the total integration time
    --------
    Returns:
    --------
    sigma_error: [muK]^2[Mpc/h]^3
    sigma_cosmic_variance : [muK]^2[Mpc/h]^3
    sigma_noise : [muK]^2[Mpc/h]^3
    Nmodes :
    """
    Omega_survey = NUM_PATCH * Omega_patch
    V_survey = X(z)**2*Omega_survey*delta_D
    t_int *= 1e6

    k_perp = k * np.sin(theta); k_para = k * np.cos(theta)
    u_perp = k_perp*X(z)/2/np.pi
    N_modes = V_survey / (4. * np.pi**2) * k**2 * delta_k * np.sin(theta) * delta_theta
    sigma_noise = X(z)**2*Y(z, nu_21)*Tsys**2*Omega_patch*NUM_PATCH*Omega_beam/t_int/Nfeeds/n_u(u_perp, lambda_21*(1+z)) / N_modes**0.5
    sigma_cosmic_variance = P_k / N_modes**0.5
    sigma_error = sigma_cosmic_variance + sigma_noise

    return sigma_error, sigma_cosmic_variance, sigma_noise, N_modes

def error_21CO_SymCross(k, delta_k, theta, delta_theta, PS_k, z, Omega_patch, NUM_PATCH, delta_D):
    """
    -----------
    Parameters:
    -----------
    k : h/Mpc
    delta_k : h/Mpc, the width of the k-shell
    theta: radians
    delta_theta: radians
    PS_k : [muK]^2[Mpc/h]^3, the symmetric cross power spectrum between HI and CO lines
    z : redshift
    Omega_patch : [radian]^2, the solid-angle on the sky subtented by a box
    NUM_PATCH : number of patches in a survey
    delta_D : Mpc/h, the length of the observation box along the LoS direction
    --------
    Returns:
    --------
    sigma_error: [muK]^2[Mpc/h]^3
    Nmodes :
    """
    V_survey = X(z)**2 * Omega_patch * NUM_PATCH * delta_D

    N_modes = V_survey / (4. * np.pi**2) * k**2 * delta_k * np.sin(theta) * delta_theta
    sigma_error = PS_k / N_modes**0.5

    return abs(sigma_error), N_modes

def error_21CO_AntiCross(k, delta_k, theta, delta_theta, PA_k, z, Omega_patch, NUM_PATCH, delta_D):
    """
    -----------
    Parameters:
    -----------
    k : h/Mpc
    delta_k : h/Mpc, the width of the k-shell
    theta: radians
    delta_theta: radians
    PA_k : [muK]^2[Mpc/h]^3, the anti-symmetric cross power spectrum between HI and CO lines
    z : redshift
    Omega_patch : [radian]^2, the solid-angle on the sky subtented by a box
    NUM_PATCH : number of patches in a survey
    delta_D : Mpc/h, the length of the observation box along the LoS direction
    --------
    Returns:
    --------
    sigma_error: [muK]^2[Mpc/h]^3
    Nmodes :
    """
    V_survey = X(z)**2 * Omega_patch * NUM_PATCH * delta_D
    PI_k = PA_k * (3 / 4 / np.pi)**0.5 * np.cos(theta)

    N_modes = V_survey / (4. * np.pi**2) * k**2 * delta_k * np.sin(theta) * delta_theta
    sigma_error = PI_k / N_modes**0.5

    return sigma_error, N_modes

def VAR_PA_k(k, delta_k, z, Omega_patch, NUM_PATCH, Tsys_CO, Tsys_21, Omega_beam_CO, Omega_beam_21, delta_nu_CO, delta_nu_21, Nfeeds_CO, Nfeeds_21, t_int_CO, t_int_21, delta_D, L_min_21, L_max_21, P_k_CO, P_k_21, PS_k, PA_k):
    """
    Calculate the variance on the dipole of the CO auto antisymmetric power spectrum at certain k
    -----------
    Parameters:
    -----------
    k : h/Mpc
    delta_k : h/Mpc, the width of the k-shell
    z : the redshift
    z : redshift

    Omega_patch : [radian]^2, the solid-angle on the sky subtented by a box
    NUM_PATCH : number of patches in a survey
    Tsys_CO, Tsys_21 : muK, the system temperature of the two lines detction respectively
    Omega_beam_CO, Omega_beam_21 : radians^2, the solid angle of CO and 21cm detection
    delta_nu_CO, delta_nu_21 : MHz, the width of single frequency channel of CO and 21cm lines
    N_feeds_CO, N_feeds_21 : the number of independent feeds, like different beams, polarizations ...
    t_int_CO, t_int_21 : second, the total integration time
    delta_D : Mpc/h, the length of the observation box along the LoS direction
    L_min_21, L_max_21: meters, the minimum and the maximum baseline length of 21cm observation
    P_k_CO, P_k_21, PS_k, PA_k : [muK]^2[Mpc/h]^3, the auto power spectrum of CO line, the auto power spectrum of 21cm line, the symmetric cross power spectrum and the antisymmetric power spectru,
    --------
    Returns:
    --------
    sigma_error: [muK]^2[Mpc/h]^3, the total error the dipole detection
    sigma_cosmic_variance : [muK]^2[Mpc/h]^3, the cosmic variance of the dipole detection
    """

    #the limits are around 0.033, 0.56, 0.024, 143 h/Mpc
    k_perp_min, k_perp_max, k_para_min, k_para_max = reliable_k_range(z, Omega_beam_CO, Omega_patch, delta_nu_CO, delta_D, L_min_21, L_max_21, delta_nu_21)
    if k**2 < k_perp_min**2 + k_para_min**2 or k**2 > k_perp_max**2 + k_para_max**2:
        return 'ERROR: k is out of the reliable range'
    delta_theta = k_para_min / k
    sin_max = k_perp_max / k if k_perp_max / k < 1 else 0.999
    theta_max = np.arcsin(sin_max)
    theta, VAR_PA_k, VAR_PA_k_cv = 0.01, 0, 0
    while theta < theta_max:
        #the total error and the cosmic variance
        sigma_P_k_CO, sigma_P_k_CO_cv, _, _ = error_CO_auto(k, delta_k, theta, delta_theta, P_k_CO, z, Tsys_CO, Omega_patch, NUM_PATCH, delta_D, Omega_beam_CO, delta_nu_CO, Nfeeds_CO, t_int_CO)
        sigma_P_k_21, sigma_P_k_21_cv, _, _ = error_21_auto(k, delta_k, theta, delta_theta, P_k_21, z, Tsys_21, Omega_patch, NUM_PATCH, Omega_beam_21, delta_D, Nfeeds_21, t_int_21)
        sigma_PS_k, _ = error_21CO_SymCross(k, delta_k, theta, delta_theta, PS_k, z, Omega_patch, NUM_PATCH, delta_D)
        sigma_PI_k, _ = error_21CO_AntiCross(k, delta_k, theta, delta_theta, PA_k, z, Omega_patch, NUM_PATCH, delta_D)
        #print(sigma_P_k_CO, sigma_P_k_21, sigma_PS_k, sigma_PI_k)
        VAR_PI_k_theta_cv = sigma_P_k_CO_cv * sigma_P_k_21_cv - sigma_PS_k**2 + sigma_PI_k**2
        VAR_PI_k_theta = sigma_P_k_CO * sigma_P_k_21 - sigma_PS_k**2 + sigma_PI_k**2
        VAR_PA_k_cv += ((3 / 4 / np.pi * np.cos(theta)**2) / VAR_PI_k_theta_cv)
        VAR_PA_k += ((3 / 4 / np.pi * np.cos(theta)**2) / VAR_PI_k_theta) #eqn 8 in Zhou et al. 2021
        theta += delta_theta
    VAR_PA_k = 1 / VAR_PA_k; VAR_PA_k_cv = 1 / VAR_PA_k_cv
    return VAR_PA_k**0.5, VAR_PA_k_cv**0.5 #return the sigma of error


"""
This solution uses the same thought to solve the problem, but used a recursive
function to do it.
"""

class TreeNode:
    def __init__(self, val=0, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right

class Solution:
    def lowest_common_ancestor(
        self,
        root: "TreeNode",
        p: "TreeNode",
        q: "TreeNode",
    ) -> "TreeNode":
        # proceed to the left subtree if the nodes are smaller than the nodes
        if root.val > max(p.val, q.val):
            return self.lowest_common_ancestor(root.left, p, q)
        # proceed to the right subtree if the nodes are greater than the nodes
        if root.val < min(p.val, q.val):
            return self.lowest_common_ancestor(root.right, p, q)

        return root


from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.resources import SERVICE_NAME, Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
#from common import get_tracer
import os
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
#import uptrace
import grpc
from traces.vendor_specific.uptrace.dsn_helper import parse_dsn, DSN

def get_otlp_exporter():
    dsn = "https://lHBrBFFwBfqMei7yNieoWA@uptrace.dev/861"
    dsn = parse_dsn(dsn)
    print(dsn.str)
    print(dsn.otlp_grpc_addr)
    credentials = grpc.ssl_channel_credentials()

    exporter = OTLPSpanExporter(
        endpoint=dsn.otlp_grpc_addr,
        # credentials=credentials,
        headers=(("uptrace-dsn", dsn.str),),
        timeout=5,
        compression=grpc.Compression.Gzip,
    )

    return exporter

def get_tracer():
    span_exporter = get_otlp_exporter()

    provider = TracerProvider()
    if not os.environ.get("OTEL_RESOURCE_ATTRIBUTES"):
        # Service name is required for most backends
        resource = Resource(attributes={
            SERVICE_NAME: "test-py-manual-otlp",
        })
        #log_level = "DEBUG",
        provider = TracerProvider(resource=resource)
        print("Using default service name")

    processor = BatchSpanProcessor(span_exporter)
    provider.add_span_processor(processor)
    trace.set_tracer_provider(provider)

    return trace.get_tracer(__name__)

tracer = get_tracer()
url = "http://"
with tracer.start_as_current_span("client operation"):
    try:
        carrier = {}
        TraceContextTextMapPropagator().inject(carrier)
        header = {"traceparent": carrier["traceparent"]}
        #res = requests.get(url, headers=header)
        print(f"Request to {url}")
    except Exception as e:
        print(f"Request to {url} failed {e}")
        pass


class Solution:
    def maxProfit(self, prices):
        """
        :type prices: List[int]
        :rtype: int
        """
        n = len(prices)
        if n <= 1: return 0
        p1, p2 = [0] * n, [0] * n

        minV, maxV = prices[0], prices[-1]
        for i in range(1, n):
            minV = min(prices[i], minV)
            p1[i] = max(p1[i-1], prices[i] - minV)

        for i in range(n-2, -1, -1):
            maxV = max(prices[i], maxV)
            p2[i] = max(p2[i+1], maxV - prices[i])

        res = 0
        for i in range(n):
            res = max(res, p1[i] + p2[i])
        return res


from flask import Blueprint, request
from big_data.db import get_db, close_db
import json, os
from big_data.spark import get_spark, get_client

import logging
LOG_FORMAT = "/********** %(asctime)s - %(levelname)s - %(message)s **********/"
logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)

bp = Blueprint('dataset', __name__, url_prefix='/dataset')

class Dataset:
    def __init__(self, username, project_id, name, description, separator, columns, file_obj):
        self.username = username
        self.project_id = project_id
        self.name = name
        self.description = description
        self.separator = separator
        self.columns = columns
        self.file_obj = file_obj
        self.local_dir = os.getcwd() + "/"
        self.hdfs_dir = "/user/" + username + "/" + project_id + "/"

    def logging(self):
        logging.info("ds 的内容如下:")
        logging.info(self.username)
        logging.info(self.project_id)
        logging.info(self.name)
        logging.info(self.description)
        logging.info(self.separator)
        logging.info(self.columns)
        logging.info(self.file_obj)

    # 上传时的进度回调函数
    def uploadCallback(self, filename, size):
        print(filename, "one chunk uploaded, ", "current size: ", size)
        if size == -1:
            print("file uploaded")

    # 临时地把文件保存到本地
    def saveFileToLocal(self):
        self.file_obj.save(self.local_dir + self.file_obj.filename)

    # 删除本地临时文件
    def deleteLocalFile(self):
        os.remove(self.local_dir + self.file_obj.filename)

    # 把本地文件上传到 Hadoop
    def uploadToHadoop(self):
        get_client().upload(
            hdfs_path = self.hdfs_dir + self.file_obj.filename,
            local_path = self.local_dir + self.file_obj.filename,
            chunk_size = 2 << 19,
            progress = self.uploadCallback
        )

    # 删除 Hadoop 中该项目下的所有数据集
    def deleteAllFromHadoop(self):
        get_client().delete(self.hdfs_dir, recursive=True)

    # 删除 Hadoop 中该数据集的原始文件
    def deleteRawFromHadoop(self):
        get_client().delete(self.hdfs_dir + self.file_obj.filename)

    # 数据处理
    def dataProcess(self):
        # 数据初始化
        sc = get_spark().sparkContext
        separator = self.separator
        columns = self.columns

        # 读取数据集文件
        lines = sc.textFile(self.hdfs_dir + self.file_obj.filename)

        # 用分割符对行进行分割
        splits = lines.map(lambda x: x.split(separator))

        # 数据类型转换
        transformed = splits.map(lambda x: typeTransform(x, columns))

        # 创建 dataframe
        col_name = []
        for column in columns:
            col_name.append(column["col_name"])
        logging.info(col_name)
        df = get_spark().createDataFrame(transformed, col_name)

        # 这里改命名 parquet 文件的规则
        df.select("*").write.save(self.hdfs_dir + self.file_obj.filename.split(".")[0] + ".parquet")

# 数据类型转换
def typeTransform(x, columns):
    for i, column in enumerate(columns):
        if column["data_type"] == "1":
            x[i] = int(x[i])
        if column["data_type"] == "2":
            x[i] = float(x[i])
    return x

# 上传文件
@bp.route('upload/', methods = ['POST'])
def info():
    # 变量初始化 & 初始化数据集对象
    status = 0
    ds = Dataset(
        request.form['username'],
        request.form['project_id'],
        request.form['name'],
        request.form['description'],
        request.form['separator'],
        json.loads(request.form['columns']),
        request.files['file']
    )

    # logging
    ds.logging()

    #
    ds.saveFileToLocal()
    ds.uploadToHadoop()
    ds.deleteLocalFile()
    ds.dataProcess()
    ds.deleteRawFromHadoop()
    # ds.deleteAllFromHadoop()

    # 插入数据库
    db = get_db()
    c = db.cursor()
    t = (int(ds.project_id), ds.name, ds.file_obj.filename.split(".")[0], ds.description)
    c.execute("INSERT INTO dataset (project_id, name, filename, description) VALUES (?, ?, ?, ?)", t)
    db.commit()
    status = 1
    close_db()

    # 接口约定:
    # 状态码:
    # 0: 添加失败  1: 添加成功
    return {"status": status}

# 获取所有数据集
@bp.route('get-all/', methods = ['POST'])
def getAllDatasets():
    # 变量初始化 & 接收请求数据
    status = 0
    total = 0
    datasets = []
    received_data = request.get_json(silent=True)

    # logging
    logging.info("received_data: " + json.dumps(received_data))

    # 应该接收到有:
    # project_id, num, start[not index]
    project_id = received_data['project_id']
    num = received_data['num']
    start = received_data['start']

    # 获取 total[not index]
    db = get_db()
    c = db.cursor()
    t = (project_id,)
    c.execute("SELECT COUNT(*) FROM dataset WHERE project_id=?", t)
    res = c.fetchone()
    total = tuple(res)[0]

    # 如果 start > total
    #   返回状态码为 0
    # 如果 start <= total
    #   返回 num 个项目信息
    if start > total:
        status = 0
    else:
        status = 1
        t = (project_id, start - 1, num)
        key = 0
        tmp = {
            "id": 0,
            "key": "",
            "name": "",
            "description": ""
        }
        for dataset in c.execute("SELECT * FROM dataset WHERE project_id=? LIMIT ?, ?", t):
            logging.info(tuple(dataset))
            tmp["id"] = tuple(dataset)[0]
            key = key + 1
            tmp["key"] = str(key)
            tmp["name"] = tuple(dataset)[2]
            tmp_description = tuple(dataset)[4]
            if tmp_description == "":
                tmp_description = "无"
            tmp["description"] = tmp_description
            datasets.append(tmp.copy())
        logging.info(datasets)

    close_db()

    # 接口约定:
    # status: 状态码
    # total: 总项目数
    # datasets: 数据集数组
    return {
        "status": status,
        "total": total,
        "datasets": datasets
    }

# 获取数据集名称
@bp.route('get-name/', methods = ['POST'])
def getName():
    # 变量初始化 & 接收请求数据
    status = 0
    name = ""
    received_data = request.get_json(silent=True)

    # logging
    logging.info("received_data: " + json.dumps(received_data))

    # 接收到的数据有:
    # id
    id = received_data['id']

    # 在数据库查找
    db = get_db()
    c = db.cursor()
    t = (id,)
    c.execute("SELECT name FROM dataset WHERE id=?", t)
    res = c.fetchone()
    name = tuple(res)[0]
    status = 1
    close_db()

    # 接口约定:
    return {
        "status": status,
        "name": name
    }

# 获取数据集数据
@bp.route('get-data/', methods = ['POST'])
def getData():
    # 变量初始化 & 接收请求数据
    status = 0
    total = 0
    data = []
    columns = []
    received_data = request.get_json(silent=True)

    # logging
    logging.info("received_data: " + json.dumps(received_data))

    # 应该接收到有:
    # id, num, start[not index]
    dataset_id = received_data['id']
    num = received_data['num']
    start = received_data['start']

    db = get_db()
    c = db.cursor()

    # 通过 dataset_id 获取 project_id 和 filename
    t = (dataset_id,)
    c.execute("SELECT project_id, filename FROM dataset WHERE id=?", t)
    res = c.fetchone()
    project_id = tuple(res)[0]
    filename = tuple(res)[1]

    # 通过 project_id 获取 username
    t = (project_id,)
    c.execute("SELECT username FROM project WHERE id=?", t)
    res = c.fetchone()
    username = tuple(res)[0]

    # 拼接出 hdfs_path，获得数据集 df
    hdfs_path = "/user/" + username + "/" + str(project_id) + "/" + filename + ".parquet"
    df = get_spark().read.load(hdfs_path)

    # 获取 total
    total = df.count()

    # 获取 columns
    tmp = {
        "title": "",
        "dataIndex": "",
    }
    tmp["title"] = "id"
    tmp["dataIndex"] = "id"
    columns.append(tmp.copy())
    for column in df.columns:
        tmp["title"] = column
        tmp["dataIndex"] = column
        columns.append(tmp.copy())

    # 获取 data
    tmp = {}
    key = 0
    for row in df.collect()[start - 1: start + num - 1]:
        tmp["id"] = str(start + key)
        key = key + 1
        tmp["key"] = str(key)
        for column in df.columns:
            tmp[column] = row[column]
        data.append(tmp.copy())

    status = 1

    # 接口约定:
    # status: 状态码
    # total: 数据项数
    # columns: 列名
    # data: 数据
    return {
        "status": status,
        "total": total,
        "columns": columns,
        "data": data
    }


#ans1
a=int(input('Number 1:'))
b=int(input('Number 2:'))
c=int(input('Number 3:'))
x=a+b+c/3
print(x)

#ans2
a=int(input('Please enter your Gross Income:-'))
print(a)
b=int(input('No. of dependents:-'))
print(b)
S=10000
D=3000
x=D*b
c=S+x
T=a-c
t=T*0.2
print(t)

#ans3
a=int(input("Number of Seconds:-"))
print("The time in minutes and seconds is: ",(a//60),"Minutes and",(a%60),"Seconds")

#ans4
x = int('25')
result = 25 + x + 25.0
result_str = str(result)
print(result_str)
result = round(25 + x + 25.0)
result_str = str(result)
print(result_str)


from . import slc
from . import telopt
from . import accessories

CR, LF, NUL = '\r\n\x00'

__all__ = ('telnet_server_shell',)

async def telnet_server_shell(stream):
    """
    A default telnet shell, appropriate for use with asynctelnet.create_server.

    This shell provides a very simple REPL, allowing introspection and state
    toggling of the connected client session.

    """
    stream = RLTextStream(stream)
    await stream.send("Ready." + CR + LF)

    command = None
    while True:
        if command:
            await writer.send(CR + LF)
        await writer.send('tel:sh> ')
        command = await stream.readline()
        if not command:
            return
        await stream.send(CR + LF)
        if command == 'quit':
            await stream.send('Goodbye.' + CR + LF)
            break
        elif command == 'help':
            await stream.send('quit, stream, slc, toggle [option|all], '
                         '… and that\'s it.')
        elif command == 'stream':
            await stream.send(repr(stream))
        elif command == 'version':
            await stream.send(accessories.get_version())
        elif command == 'slc':
            await stream.send(get_slcdata(stream))
        elif command.startswith('toggle'):
            option = command[len('toggle '):] or None
            await stream.send(await do_toggle(stream, option))
        elif command:
            await stream.send(f'no such command: {command!r}.')
    await stream.aclose()

def get_slcdata(writer):
    """Display Special Line Editing (SLC) characters."""
    _slcs = sorted([
        '{:>15}: {}'.format(slc.name_slc_command(slc_func), slc_def)
        for (slc_func, slc_def) in sorted(writer.slctab.items())
        if not (slc_def.nosupport or slc_def.val == slc.theNULL)])
    _unset = sorted([
        slc.name_slc_command(slc_func)
        for (slc_func, slc_def) in sorted(writer.slctab.items())
        if slc_def.val == slc.theNULL])
    _nosupport = sorted([
        slc.name_slc_command(slc_func)
        for (slc_func, slc_def) in sorted(
            writer.slctab.items())
        if slc_def.nosupport])

    return ('Special Line Characters:\r\n' +
            '\r\n'.join(_slcs) +
            '\r\nUnset by client: ' +
            ', '.join(_unset) +
            '\r\nNot supported by server: ' +
            ', '.join(_nosupport))

async def do_toggle(writer, option):
    """Display or toggle telnet session parameters."""
    tbl_opt = {
        'echo': writer.local_option.enabled(telopt.ECHO),
        'goahead': not writer.local_option.enabled(telopt.SGA),
        'outbinary': writer.outbinary,
        'inbinary': writer.inbinary,
        'binary': writer.outbinary and writer.inbinary,
        'xon-any': writer.xon_any,
        'lflow': writer.lflow,
    }

    if not option:
        return ('\r\n'.join('{0} {1}'.format(
            opt, 'ON' if enabled else 'off')
            for opt, enabled in sorted(tbl_opt.items())))

    msgs = []
    if option in ('echo', 'all'):
        cmd = (telopt.WONT if tbl_opt['echo'] else telopt.WILL)
        await writer.iac(cmd, telopt.ECHO)
        msgs.append('{} echo.'.format(
            telopt.name_command(cmd).lower()))

    if option in ('goahead', 'all'):
        cmd = (telopt.WILL if tbl_opt['goahead'] else telopt.WONT)
        await writer.iac(cmd, telopt.SGA)
        msgs.append('{} suppress go-ahead.'.format(
            telopt.name_command(cmd).lower()))

    if option in ('outbinary', 'binary', 'all'):
        cmd = (telopt.WONT if tbl_opt['outbinary'] else telopt.WILL)
        await writer.iac(cmd, telopt.BINARY)
        msgs.append('{} outbinary.'.format(
            telopt.name_command(cmd).lower()))

    if option in ('inbinary', 'binary', 'all'):
        cmd = (telopt.DONT if tbl_opt['inbinary'] else telopt.DO)
        await writer.iac(cmd, telopt.BINARY)
        msgs.append('{} inbinary.'.format(
            telopt.name_command(cmd).lower()))

    if option in ('xon-any', 'all'):
        writer.xon_any = not tbl_opt['xon-any']
        await writer.send_lineflow_mode()
        msgs.append('xon-any {}abled.'.format(
            'en' if writer.xon_any else 'dis'))

    if option in ('lflow', 'all'):
        writer.lflow = not tbl_opt['lflow']
        await writer.send_lineflow_mode()
        msgs.append('lineflow {}abled.'.format(
            'en' if writer.lflow else 'dis'))

    if option not in tbl_opt and option != 'all':
        msgs.append('toggle: not an option.')

    return '\r\n'.join(msgs)

class ReadLine:
    """
    A very primitive readline wrapper.

    Usage::
        rl = ReadLine(stream)
        while True:
            line = await rl()
    """
    def __init__(self, stream):
        self._stream = stream
        self._buf = ''
        self._echo = self.extra_attributes[EchoAttr]
        self._last_inp = ''

    async def __call__(self):
        buf = self._buf
        command = ""
        while True:
            if buf == "":
                buf = await self._stream.receive()
            for i,inp in enumerate(buf):
                if inp in (LF, NUL) and self._last_inp == CR:
                    self._buf = buf[i+1:]
                    return command

                elif inp in (CR, LF):
                    # first CR or LF yields command
                    self._buf = buf[i+1:]
                    return command

                elif inp in ('\b', '\x7f'):
                    # backspace over input
                    if command:
                        command = command[:-1]
                        await self.send('\b \b')

                else:
                    # buffer and echo input
                    command += inp
                    await self._stream.echo(inp)
                    self._last_inp = inp


from . import bp as api
from app.blueprints.auth.models import User
from .models import Item, Category
from app.blueprints.auth.auth import token_auth
from flask import request, make_response, g

##########
## CATEGORY API CALLS
##########

@api.get('/category')
@token_auth.login_required()
def get_categories():
    cats = Category.query.all()
    cats_dicts=[cat.to_dict() for cat in cats]
    return make_response({"categories":cats_dicts},200)

# create a new category
@api.post('/category')
@token_auth.login_required()
def post_category():
    if not g.current_user.is_admin:
        return make_response(f"You are not an admin",403)
    cat_name=request.get_json().get('name')
    cat=Category(name=cat_name)
    cat.save()
    return make_response(f"category {cat.id} with name {cat.name} created",200)

# change a category
@api.patch('/category')
@token_auth.login_required()
def patch_category():
    if not g.current_user.is_admin:
        return make_response(f"You are not an admin",403)
    cat_name=request.get_json().get('name')
    cat_id = request.get_json().get('id')
    if not cat_id or not cat_name:
        return make_response("Invalid Request payload",400)
    cat=Category.query.get(cat_id)
    if not cat:
        return make_response("Invalid category id",400)
    cat.name = cat_name
    cat.save()
    return make_response(f"category {cat.id} has a new name {cat.name}",200)

# Delete a category
@api.delete('/category')
@token_auth.login_required()
def delete_category():
    if not g.current_user.is_admin:
        return make_response(f"You are not an admin",403)
    cat_id = request.get_json().get('id')
    if not cat_id:
        return make_response("Invalid Request payload",400)
    cat=Category.query.get(cat_id)
    if not cat:
        return make_response("Invalid category id",400)
    cat_id=cat.id
    cat.delete()
    return make_response(f"category {cat_id} has been deleted",200)

##########
## ITEM API CALLS
##########
# Get a list of all the items in our shop
@api.get('/all_items')
@token_auth.login_required()
def get_all_items():
    all_items=Item.query.all()
    items = [item.to_dict() for item in all_items]
    return make_response({"items":items},200)

# Get a list of all items in a category
@api.get('/items_by_category_id')
@token_auth.login_required()
def get_items_by_category():
    id = request.get_json().get('id')
    if not id:
        return make_response("Invalid Request payload",400)
    all_items=Item.query.filter_by(category_id=id).all()
    items = [item.to_dict() for item in all_items]
    return make_response({"items":items},200)

# look up a specific item by its id
@api.get('/item')
@token_auth.login_required()
def get_item():
    id = request.get_json().get('id')
    if not id:
        return make_response("Invalid Request payload",400)
    item=Item.query.filter_by(id=id).first()
    return make_response(item.to_dict(),200)

# create a new item by sending a payload with all the needed info
                                                    # name
                                                    # description
                                                    # price
                                                    # img
                                                    # category_id
@api.post('/item')
@token_auth.login_required()
def post_item():
    if not g.current_user.is_admin:
        return make_response(f"You are not an admin",403)
    item_dict = request.get_json()
    print(item_dict)
    if 'name'  not in item_dict or 'description'  not in item_dict or 'price'  not in item_dict or 'img'  not in item_dict or 'category_id'  not in item_dict:
        return make_response("Invalid Request payload",400)
    item=Item(**item_dict)
    item.save()
    return make_response(f"Item {item.name} was created with the id {item.id}",201)

# This will Alter an item by looking up item fom an item_id
@api.patch('/item')
@token_auth.login_required()
def patch_item():
    if not g.current_user.is_admin:
        return make_response(f"You are not an admin",403)
    item_dict = request.get_json()
    if not item_dict.get('id'):
        return make_response("Invalid Request payload",400)
    item = Item.query.get(item_dict['id'])
    item.from_dict(item_dict)
    item.save()
    return make_response(f"Item {item.id} was edited",200)

# This will delete an item by its id
@api.delete('/item')
@token_auth.login_required()
def delete_item():
    if not g.current_user.is_admin:
        return make_response(f"You are not an admin",403)
    id = request.get_json().get('id')
    if not id:
        return make_response("Invalid Request payload",400)
    item_to_del=Item.query.get(id)
    item_to_del.delete()
    return make_response(f"Item id {id} has been deleted", 201)


# -*- coding: utf-8 -*-

def check_ddd(input):
  DDDS = {
    61: 'Brasilia',
    71: 'Salvador',
    11: 'Sao Paulo',
    21: 'Rio de Janeiro',
    32: 'Juiz de Fora',
    19: 'Campinas',
    27: 'Vitoria',
    31: 'Belo Horizonte'
    # 51: 'Porto Alegre'
  }

  for ddd in DDDS:
    if input == ddd:
      return print(DDDS[ddd])
  return print('DDD nao cadastrado')

check_ddd(int(input()))


# https://www.hackerrank.com/challenges/alphabet-rangoli/problem
# Score: 20

def print_rangoli(size):
    # your code goes here

    alphabet_list = list(map(chr, range(97, 123)))
    alphabet = "".join(alphabet_list)
    result = []
    for i in range(size):
        s = "-".join(alphabet[i:size])
        result.append((s[::-1]+s[1:]).center(4*size-3, "-"))

    print("\n".join(result[:0:-1]+result))

if __name__ == '__main__':
    n = int(input())
    print_rangoli(n)

Hackerrank/03 - Strings/011 - Alphabet Rangoli.py


import torch as t
import torchvision as tv
import torchvision.transforms as transforms
from torchvision.transforms import ToPILImage
show = ToPILImage() # 可以把Tensor转成Image，方便可视化
import torch.nn as nn
import torch.nn.functional as F
from torch import optim

# http://www.cs.toronto.edu/~kriz/cifar.html
# root：Root directory of dataset where directory ``cifar-10-batches-py`` exists or will be saved to if download is set to True.

## 数据加载
# 定义对数据的预处理
transform = transforms.Compose([
        transforms.ToTensor(), # 转为Tensor
        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), # 归一化
                             ])

# 训练集
# 下载数据或者提取数据
trainset = tv.datasets.CIFAR10(
                    root='D:/Machine_learning/1.Dataset/1.CIFAR10',# windows
#                     root = '/Users/iker/Downloads',# mac
                    train = True,   # True 则作为训练集，否则为测试集
                    download=True,
                    transform=transform)

# 定义一个数据迭代器
trainloader = t.utils.data.DataLoader(
                    trainset,
                    batch_size=4,
                    shuffle=True,  # 设置为True时会在每个epoch重新打乱数据，训练集打乱，测试集不打乱
                    num_workers=2)

# 测试集
testset = tv.datasets.CIFAR10(
                    root = 'D:/Machine_learning/1.Dataset/1.CIFAR10',# windows
#                     root = '/Users/iker/Downloads',# mac
                    train=False,
                    download=True,
                    transform=transform)

testloader = t.utils.data.DataLoader(
                    testset,
                    batch_size=4,
                    shuffle=False,
                    num_workers=2)

classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')

## 网络搭建
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1   = nn.Linear(16*5*5, 120)
        self.fc2   = nn.Linear(120, 84)
        self.fc3   = nn.Linear(84, 10)

    def forward(self, x):
        x = F.max_pool2d(F.relu(self.conv1(x)), (2, 2)) # (2,2)相当于把图像变成原来一半
        x = F.max_pool2d(F.relu(self.conv2(x)), 2)
        x = x.view(x.size()[0], -1)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

# 损失函数和参数更新算法
criterion = nn.CrossEntropyLoss() # 交叉熵损失函数
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)  # 优化更新算法

## 参数训练
t.set_num_threads(8)
for epoch in range(2):

    running_loss = 0.0
    for i, data in enumerate(trainloader, 0): # enumerate可以同时返回数据下标和数据

        # 输入数据
        inputs, labels = data

        # 梯度清零
        optimizer.zero_grad()

        # forward + backward
        outputs = net(inputs)
        loss = criterion(outputs, labels)
        loss.backward()

        # 更新参数
        optimizer.step()

        # 打印log信息
        # loss 是一个scalar,需要使用loss.item()来获取数值，不能使用loss[0]
        running_loss += loss.item()
        if i % 2000 == 1999: # 每2000个batch打印一下训练状态
            print('[%d, %5d] loss: %.3f'% (epoch+1, i+1, running_loss / 2000))
            running_loss = 0.0
print('Finished Training')

## 测试集检验
correct  =  0  # 预测正确的图片数# 预测正确的图
total = 0 # 总共的图片数

# 由于测试的时候不需要求导，可以暂时关闭autograd，提高速度，节约内存
with t.no_grad():
    for data in testloader:
        images, labels = data
        outputs = net(images)
        _, predicted = t.max(outputs, 1)
        total += labels.size(0)
        correct += (predicted == labels).sum()

print('10000张测试集中的准确率为: %d %%' % (100 * correct / total))


#!/usr/bin/python3
# -*- coding: utf-8 -*-

from gnr.web.gnrbaseclasses import BaseComponent
from gnr.core.gnrdecorator import public_method
from datetime import datetime
import zipfile
import os
from os.path import basename
from pathlib import Path

class View(BaseComponent):

    def th_struct(self,struct):
        r = struct.view().rows()
        r.fieldcell('port')
        r.fieldcell('imbarcazione_id', width='30em')
        r.fieldcell('email_to')
        r.fieldcell('email_cc')
        r.fieldcell('prearr_descr')
        r.fieldcell('__ins_ts')
        r.fieldcell('data_invio')

    def th_order(self):
        return '__ins_ts:d'

    def th_query(self):
        return dict(column='port', op='contains', val='')

class Form(BaseComponent):

    def th_form(self, form):
        pane = form.record
        fb = pane.formbuilder(cols=2, border_spacing='4px', fld_width='30em')
        fb.field('port', colspan=2)
        fb.field('imbarcazione_id', colspan=2, hasDownArrow=True)
        fb.div("!![en]Insert the emails separate by commas",font_weight='bold', margin_top='10px')
        fb.br()
        fb.field('email_to', colspan=2,tag='textarea', height='5em')
        fb.div("!![en]Insert the emails separate by commas",font_weight='bold', margin_top='10px')
        fb.br()
        fb.field('email_cc',colspan=2,tag='textarea', height='5em')
        fb.field('prearr_descr',tag='dbSelect', hasDownArrow=True, colspan=2)
        fb.field('data_invio', readOnly=True, disabled=True)

    def th_bottom_custom(self, bottom):
        bar = bottom.slotBar('10,email_prearr,*,10')
        btn_email_prearr=bar.email_prearr.button('!![en]Email Pre-Arrivals')
        btn_email_prearr.dataRpc('nome_temp', self.email_prearrival,record='=#FORM.record', email_template_id='prearrivals',
                                    _ask=dict(title='!![en]Select if you want a zipped file',
                                    fields=[dict(name='type_atc',lbl='!![en]Type atc',tag='filteringSelect',values='zip:zip,unzip:non compresso')]),
                                    _onResult="this.form.save();this.form.reload();")
        bar.dataController("""if(msgspec=='prearrivals') {genro.publish("floating_message",{message:msg_txt, messageType:"message"});}""",
                            msgspec='^nome_temp', msg_txt = 'Email ready to be sent')

    def th_options(self):
        return dict(dialog_height='400px', dialog_width='600px')

    @public_method
    def email_prearrival(self, record,email_template_id=None, **kwargs):
        #print(x)
        id_prearr_atc=record['prearr_descr']
        record_id=record['id']
        if not record_id:
            return
        #leggiamo nei kwargs la tipologia dell'allegato se zip oppure non compresso
        type_atc = None
        for chiavi in kwargs.keys():
            if chiavi=='type_atc':
                if kwargs['type_atc']:
                    type_atc=kwargs['type_atc']

        tbl_att =  self.db.table('shipsteps.prearrivals_default_atc')
        fileurl = tbl_att.query(columns='$fileurl',
                  where='$maintable_id=:mt_id',mt_id=id_prearr_atc).fetch()
        ln = len(fileurl)
        attcmt=[]
        attcmt_name=[]

        for r in range(ln):
            file_url = fileurl[r][0]
            file_path = file_url.replace('/home','site')
            fileSn = self.site.storageNode(file_path)
            attcmt.append(fileSn.internal_path)
            attcmt_name.append(Path(fileSn.internal_path).name)

        if email_template_id=='prearrivals':

            pdfpath = self.site.storageNode('home:shipsteps_prearrivals_default')
            pdfpath=pdfpath.internal_path
            with zipfile.ZipFile(pdfpath + "/prearrivals.zip", mode="w") as archive:

                for filenamedir,filename in zip(attcmt,attcmt_name):
                     archive.write(filenamedir,filename)

        if type_atc == 'zip':
            attcmt=archive.filename
        elif type_atc == 'unzip':
            attcmt
        else:
            attcmt

        # Lettura degli account email predefiniti all'interno di Agency e Staff
        tbl_staff =  self.db.table('shipsteps.staff')
        account_email = tbl_staff.readColumns(columns='$email_account_id',
                  where='$agency_id=:ag_id',
                    ag_id=self.db.currentEnv.get('current_agency_id'))
        tbl_agency =  self.db.table('shipsteps.agency')
        account_emailpec = tbl_agency.readColumns(columns='$emailpec_account_id',
                  where='$id=:ag_id',
                    ag_id=self.db.currentEnv.get('current_agency_id'))

        email_to = record['email_to']
        email_cc = record['email_cc']
        email_bcc = ''

        if (email_to) is not None:
            self.db.table('email.message').newMessageFromUserTemplate(
                                                          record_id=record_id,
                                                          table='shipsteps.prearrivals',
                                                          account_id = account_email,
                                                          to_address=email_to,
                                                          cc_address=email_cc,
                                                          bcc_address=email_bcc,
                                                          attachments=attcmt,
                                                          template_code=email_template_id)
            self.db.commit()

        if (email_to) is not None:
            #avendo preso il valore services nei kwargs ossia il consignee dell'email sevices andiamo a copiarlo nel record della tasklist nome_servizio
            data_corrente = datetime.now()
            tbl_prearrivals = self.db.table('shipsteps.prearrivals')
            tbl_prearrivals.batchUpdate(dict(data_invio=data_corrente),
                                        where='$id=:id_prearr', id_prearr=record_id)
            self.db.commit()

          # if servizio == ['capitaneria']:
            nome_temp = 'prearrivals'
            return nome_temp


import csv

with open('Perceptron/DataSets/train.csv', newline='') as csvfile:
    train = list(csv.reader(csvfile))

with open('Perceptron/DataSets/test.csv', newline='') as csvfile:
    test = list(csv.reader(csvfile))

# Separate the nx5 array into a nx4 array and a nx1 array
trainX = [row[:4] for row in train]  # Extract the first 4 columns of each row
trainY = [row[4] for row in train]   # Extract the last column of each row

testX = [row[:4] for row in test]  # Extract the first 4 columns of each row
testY = [row[4] for row in test]   # Extract the last column of each row

# Convert strings to doubles and ints
trainX = [[float(s) for s in row] for row in trainX]
trainY = [int(s) for s in trainY]
testX = [[float(s) for s in row] for row in testX]
testY = [int(s) for s in testY]

# Replace 0 with -1 for y data
for i in range(len(trainY)):
    if trainY[i] == 0:
        trainY[i] = -1

for i in range(len(testY)):
    if testY[i] == 0:
        testY[i] = -1


from midi.MidiOutFile import MidiOutFile

major = [0, 2, 4, 5, 7, 9, 11]
minor = [0, 2, 3, 5, 7, 8, 10]

class Song:
    def __init__(self):
        self.tracks = []
        self.tempo = 140 # BPM
        self.beatsPerMeasure = 4
        self.key = 0x40
        self.mode = major

    def addTrack(self, track):
        self.tracks.append(track)

    def noteToPitch(self, note):
        """
        returns the midi pitch (int) for a note in the current key and mode
        """
        number = note.number - 1

        offsetIntoMode = number % 7
        octaveOffset = 12 * (number - offsetIntoMode) / 7
        return self.key + octaveOffset + self.mode[offsetIntoMode]

    def getEventTracks(self):
        """
        convert notes to on- and off-events
        """
        class Event:
            def __init__(self, note, type):
                self.note = note
                self.type = type
        eventTracks = []
        for track in self.tracks:
            events = {}
            def addEvent(pos, event):
                if not events.has_key(pos):
                    events[pos] = []
                events[pos].append(event)
            positions = track.notes.keys()
            positions.sort()
            for pos in positions:
                noteArray = track.notes[pos]
                for note in noteArray:
                    addEvent(pos, Event(note, "on")) # TODO on/off
                    addEvent(pos + note.duration, Event(note, "off")) # TODO on/off
            eventTracks.append(events)
        return eventTracks

    def writeToMidi(self, filepath, eventTracks=None):
        if eventTracks == None:
            eventTracks = self.getEventTracks()

        midi = MidiOutFile(filepath)
        midi.header(1, len(self.tracks), 480) # TODO 480?

        for i in range(len(self.tracks)):
            track = self.tracks[i]
            events = eventTracks[i]
            midi.start_of_track(i)
            midi.tempo(60000000 / self.tempo) # int(60,000,000.00 / BPM)
            midi.time_signature(self.beatsPerMeasure, 2, 24, 8) # TODO: ?

            midi.patch_change(i, track.instrument)

            lastPos = 0
            positions = events.keys()
            positions.sort()
            for pos in positions:
                eventArray = events[pos]
                deltaPos = pos - lastPos
                midiTime = int(float(deltaPos) * 300) # TODO: wtf
                for event in eventArray:
                    midi.update_time(midiTime)
                    pitch = self.noteToPitch(event.note)
                    if event.type == "on": # TODO on/off
                        midi.note_on(channel=i, note=pitch, velocity=event.note.velocity)
                    else:
                        midi.note_off(channel=i, note=pitch)
                    midiTime = 0 # remainder of events are simultaneous
                lastPos = pos
            midi.update_time(0)
            midi.end_of_track()
        midi.eof()

    def __str__(self):
        return ",".join([str(t) for t in tracks])

#from .track import Track
#from .note import Note

#s = Song()
#t = Track()
#s.addTrack(t)
#t.addNote(0, Note(1))
#t.addNote(1, Note(2))
#t.addNote(2, Note(3))

#eventTracks = s.writeToMidi("")

#print eventTracks


from random import randint
def sorteio(lista):
    print("Os 5 valores da lista são :")
    for i in range(0, 5):
        lista.append(randint(1,10))
    print(lista)

def soma(lista):
    total=0
    print(" ",end="")
    for v in lista:
        if v % 2==0:
            total+=v

    print(f"Somando os valores pares de o resultado vai ser {total}")

numeros=list()

sorteio(numeros)
soma(numeros)


'''
将一个正方形数组矩阵顺时针旋转90度
tips：
     从外层，一层一层旋转，只需搞定最外层即可同理
'''
import sys
def read_input():
    try:
        input_martix = []
        while True:
            line = sys.stdin.readline().strip()
            if line == '':
                break
            input_martix.append(list(map(int,line.split())))
    except:
        pass
    return input_martix
def rotate(matrix):
    l1 = 0
    l2 = 0
    r1 = len(matrix) - 1
    r2 = len(matrix[0]) - 1
    while l1 < r1:
        rotateedge(matrix,l1,l2,r1,r2)
        l1 += 1
        l2 += 1
        r1 -= 1
        r2 -= 1
    return matrix
def rotateedge(matrix,l1,l2,r1,r2):
    len_edge = r1 - l1
    for i in range(len_edge):
        tmp = matrix[l1][l2+i]  #想着各自的位置将由哪个位置的数字替换即可
        matrix[l1][l2 + i] = matrix[r1-i][l2]
        matrix[r1 - i][l2] = matrix[r1][r2-i]
        matrix[r1][r2 - i] = matrix[l1+i][r2]
        matrix[l1 + i][r2] = tmp
    return
if __name__ == '__main__':
    matrix = read_input()
    ans = rotate(matrix)
    print(ans)

NowCoder/二刷/顺时针旋转矩阵（正方形）.py

顺时针旋转矩阵（正方形）.py


import os
import sys

import numpy as np
from scipy.stats import ortho_group
import torch

ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__),'..','..'))
if ROOT_DIR not in sys.path: sys.path.append(ROOT_DIR)

import response_contour_analysis.utils.principal_curvature as curve_utils

import pytest

DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'

class QuadraticFunction(torch.nn.Module):
    def __init__(self, diag=None, hess=None):
        super().__init__()

        if hess is not None:
            self.hessian = torch.tensor(hess).to(DEVICE)
        else:
            if diag is None:
                diag = [1.0, 2.0]
            self.hessian = torch.diag(torch.tensor(diag)).to(DEVICE)

        self.hessian = torch.nn.Parameter(self.hessian, requires_grad=False)

    def forward(self, x):
        return torch.dot(x, torch.matmul(self.hessian, x))

def value_grad_hess(f, point, dtype):
    value = f(point)
    grad = torch.autograd.functional.jacobian(f, point)
    hess = torch.autograd.functional.hessian(f, point)
    return value.to(dtype), grad.to(dtype), hess.to(dtype)

#def hyperboloid_eq(x, a, c):
#    return (x[:-1] ** 2) / (a ** 2) - x[-1] ** 2 / c ** 2

@pytest.mark.parametrize('curve_func', ['moosavi', 'golden', 'poole', 'lee_graph'])
def test_hyperboloid(curve_func):
    if curve_func == 'moosavi' or curve_func=='poole':
        pytest.skip() # we know this will fail
    a = 1
    c = 3
    f = lambda x: curve_utils.hyperboloid_graph(x[0], x[1], a=a, c=c)
    x = 2.0
    y = 2.0
    point = torch.tensor([x, y]).to(DEVICE)
    z, grad, hess = value_grad_hess(f, point, dtype=torch.double)

    shape_operator, principal_curvatures, principal_directions = curve_utils.local_response_curvature_alternates(grad, hess, curve_func)

    u = z.cpu().numpy() / c
    analytic_gauss_curvature = -c**2 / (c**2 + (a**2 + c**2) * u**2)**2

    np.testing.assert_allclose(
        analytic_gauss_curvature,
        torch.det(shape_operator).cpu().numpy(),
        rtol=1e-6
    )

@pytest.mark.parametrize('curve_func', ['moosavi', 'poole', 'lee_level'])
@pytest.mark.parametrize('dimensions', [2, 3, 5, 10])
@pytest.mark.parametrize('radius', [0.1, 1, 2, 3, 10])
def test_sphere_pc_variance(dimensions, radius, curve_func):
    if curve_func == 'moosavi' or curve_func=='poole':
        pytest.skip() # we know this will fail
    f = QuadraticFunction(np.ones(dimensions)).to(DEVICE)
    #point = torch.tensor(np.hstack((np.zeros(dimensions-1), [radius]))).to(DEVICE)
    point = torch.tensor(np.ones(dimensions) / np.sqrt(dimensions) * radius).to(DEVICE)
    value, pt_grad, pt_hess = value_grad_hess(f, point, dtype=torch.double)
    iso_shape_operator, iso_curvatures, iso_directions = curve_utils.local_response_curvature_alternates(pt_grad, pt_hess, curve_func)
    iso_curvatures = iso_curvatures.detach().cpu().numpy()
    assert iso_curvatures.var() < 1e-16

@pytest.mark.parametrize('curve_func', ['moosavi', 'poole', 'lee_level'])
@pytest.mark.parametrize('dimensions', [2, 3, 5, 10])
@pytest.mark.parametrize('radius', [0.1, 1, 2, 3, 10])
def test_sphere_pcs(dimensions, radius, curve_func):
    if curve_func == 'moosavi' or curve_func=='poole':
        pytest.skip() # we know this will fail
    f = QuadraticFunction(np.ones(dimensions)).to(DEVICE)
    #point = torch.tensor(np.hstack((np.zeros(dimensions-1), [radius]))).to(DEVICE)
    point = torch.tensor(np.ones(dimensions) / np.sqrt(dimensions) * radius).to(DEVICE)
    value, pt_grad, pt_hess = value_grad_hess(f, point, dtype=torch.double)
    iso_shape_operator, iso_curvatures, iso_directions = curve_utils.local_response_curvature_alternates(pt_grad, pt_hess, curve_func)
    iso_curvatures = iso_curvatures.detach().cpu().numpy()
    expected_principal_curvature = 1 / radius
    np.testing.assert_allclose(np.abs(iso_curvatures), [expected_principal_curvature,]*(dimensions-1))

@pytest.mark.parametrize('curve_func', ['moosavi', 'poole', 'lee_level'])
@pytest.mark.parametrize('dimensions', [2, 3, 5, 10])
@pytest.mark.parametrize('radius', [0.1, 1, 2, 3, 10])
def test_sphere_gauss(dimensions, radius, curve_func):
    if curve_func == 'moosavi' or curve_func=='poole':
        pytest.skip() # we know this will fail
    f = QuadraticFunction(np.ones(dimensions)).to(DEVICE)
    #point = torch.tensor(np.hstack((np.zeros(dimensions-1), [radius]))).to(DEVICE)
    point = torch.tensor(np.ones(dimensions) / np.sqrt(dimensions) * radius).to(DEVICE)
    value, pt_grad, pt_hess = value_grad_hess(f, point, dtype=torch.double)
    iso_shape_operator, iso_curvatures, iso_directions = curve_utils.local_response_curvature_alternates(pt_grad, pt_hess, curve_func)
    iso_curvatures = iso_curvatures.detach().cpu().numpy()
    expected_principal_curvature = 1 / radius
    expected_gaussian_curvature = np.prod([expected_principal_curvature,]*(dimensions-1))
    np.testing.assert_allclose(np.abs(np.prod(iso_curvatures)), expected_gaussian_curvature)

@pytest.mark.parametrize('dimensions', [2, 3, 5])
@pytest.mark.parametrize('radius', [0.1, 1, 2])
def test_coordinate_transform_sphere(dimensions, radius):
    f = QuadraticFunction(np.ones(dimensions)).to(DEVICE)
    #point = torch.tensor(np.hstack((np.zeros(dimensions-1), [radius]))).to(DEVICE)
    point = torch.tensor(np.ones(dimensions) / np.sqrt(dimensions) * radius).to(DEVICE)

    value, pt_grad, pt_hess = value_grad_hess(f, point, dtype=torch.double)

    coordinate_transformations = [
        None,
        -torch.eye(len(pt_grad), dtype=torch.double).to(DEVICE),
        torch.tensor(ortho_group.rvs(len(point)), dtype=torch.double).to(DEVICE)]
    #coordinate_transformation = torch.tensor(ortho_group.rvs(len(point)), dtype=torch.double).to(DEVICE)
    for coordinate_transformation in coordinate_transformations:
        iso_shape_operator, iso_curvatures, iso_directions = curve_utils.local_response_curvature_level_set(
            pt_grad, pt_hess, coordinate_transformation=coordinate_transformation)

        iso_curvatures = iso_curvatures.detach().cpu().numpy()

        assert iso_curvatures.var() < 1e-16

        expected_gaussian_curvature = 1 / (radius ** (dimensions - 1))
        np.testing.assert_allclose(np.abs(np.prod(iso_curvatures)), expected_gaussian_curvature)

@pytest.mark.skip("Will usually fail because the transformation reults in bad implicit functions")
def test_coordinate_transform():
    f = QuadraticFunction(np.array([1.0, 2.0, 0.1, 0.5])).to(DEVICE)
    point = torch.tensor(np.hstack((np.zeros(3), [1]))).to(DEVICE)

    value, pt_grad, pt_hess = value_grad_hess(f, point, dtype=torch.double)

    iso_shape_operator1, iso_curvatures1, iso_directions1 = curve_utils.local_response_curvature_level_set(pt_grad, pt_hess)
    iso_curvatures1 = iso_curvatures1.detach().cpu().numpy()

    M = ortho_group.rvs(len(point))
    #M = np.eye(len(point))
    #_M = ortho_group.rvs(len(point) - 1)
    #M[:-1, :-1] = _M
    coordinate_transformation = torch.tensor(M, dtype=torch.double).to(DEVICE)
    iso_shape_operator2, iso_curvatures2, iso_directions2 = curve_utils.local_response_curvature_level_set(pt_grad, pt_hess, coordinate_transformation=coordinate_transformation)
    iso_curvatures2 = iso_curvatures2.detach().cpu().numpy()

    np.testing.assert_allclose(iso_curvatures1, iso_curvatures2)

@pytest.mark.parametrize('dimensions', [20, 40, 60, 500])
@pytest.mark.parametrize('radius', [0.1, 1, 2])
def test_bad_condition(dimensions, radius):
    f_3d = QuadraticFunction([3.0, 3.0, 1.0]+[0.5] * (dimensions - 3)).to(DEVICE)
    point = torch.tensor(
        [np.sqrt(0.5), np.sqrt(0.5), 0]+[0] * (dimensions - 3),
        dtype=torch.float
    ).to(DEVICE) * radius

    # first in original coordinates
    value_orig, pt_grad_orig, pt_hess_orig = value_grad_hess(f_3d, point, dtype=torch.double)
    with pytest.raises(ValueError):
        iso_shape_operator_orig, iso_curvatures_orig, iso_directions_orig = \
            curve_utils.local_response_curvature_level_set(
                pt_grad_orig,
                pt_hess_orig,
                coordinate_transformation=torch.eye(len(point)).to(DEVICE)
            )

    rst = None
    # This will in general result in very bad conditioning
    M = ortho_group.rvs(len(point), random_state=rst)
    coordinate_transformation = torch.tensor(M).to(DEVICE)

    def new_f(x):
        return f_3d(torch.matmul(coordinate_transformation.T.type(x.dtype), x))
    new_point = torch.matmul(
        coordinate_transformation.type(point.dtype),
        point
    ).detach().clone()

    value_new, pt_grad_new, pt_hess_new = value_grad_hess(new_f, new_point, dtype=torch.double)

    np.testing.assert_allclose(
        value_orig.cpu().numpy(),
        value_new.cpu().numpy(),
        rtol=1e-6
    )

    iso_shape_operator_new, iso_curvatures_new, iso_directions_new = \
        curve_utils.local_response_curvature_level_set(
            pt_grad_new, pt_hess_new,
        )

    expected_curvatures = np.array(
        [-1/6] * (dimensions - 3) + [-1/3, -1.0]
    ) / radius

    np.testing.assert_allclose(
        iso_curvatures_new.cpu().numpy(),
        expected_curvatures,
        rtol=1e-6,
    )


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import socketserver

class UDP(socketserver.DatagramRequestHandler):
    """
    Retrieves and Logs the UDP Datagram packets through local Broadcast to the InfluxDB instance.
    Extends the socketserver class.
    """

    # There's no prettier way.
    #: (str) Pattern of the
    COL_HEAD = "Timestamp,Accel_X,Accel_Y,Accel_Z,Roll,Pitch,Yaw,Quat.X,Quat.Y,Quat.Z,Quat.W,RM11,RM12,RM13,RM21,RM22,RM23,RM31,RM32,RM33,GravAcc_X,GravAcc_Y,GravAcc_Z,UserAcc_X,UserAcc_Y,UserAcc_Z,RotRate_X,RotRate_Y,RotRate_Z,MagHeading,TrueHeading,HeadingAccuracy,MagX,MagY,MagZ,Lat,Long,LocAccuracy,Course,Speed,Altitude".split(",")

    def _transform_dict(self, data):
        """
        Creates a Dictionary of the incoming UDP data string.
        """
        column_data = data.split(",")

        if len(column_data) == len(self.COL_HEAD):
            return {self.COL_HEAD[_]: float(column_data[_]) for _ in range(0, len(self.COL_HEAD))}

        return dict()

    def probe(self):
        """
        Translates the dict to standard probe data.
        """

        data = self._transform_dict(self.rfile.readline().rstrip().decode())

        acce = ['Accel_X', 'Accel_Y', 'Accel_Z']
        gyro = ['RotRate_X', 'RotRate_Y', 'RotRate_Z']
        magn = ['MagX', 'MagY', 'MagZ']
        ahrs = ['Roll', 'Pitch', 'Yaw']

        return {
            'accelerometer': [data[_] for _ in acce],
            'gyroscope':     [data[_] for _ in gyro],
            'magnetometer':  [data[_] for _ in magn],
            'ahrs':          [data[_] for _ in ahrs]
        }

    def handle(self):
        """
        This method is called on every UDP packets that are recieved.
        On every received data, the callable, `self.data_handler` is called with the data.
        """
        try:
            if not UDP.handler:
                raise
            else:
                UDP.handler(dat = self.probe())
        except ValueError:
            pass

    @staticmethod
    def handler(func):
        """
        Decorator function, registers data handler.
        """
        UDP.handler = func

    @staticmethod
    def start_routine(hostname, port):
        """
        Helper function that starts the routine.
        """
        c = socketserver.UDPServer((hostname, port), UDP)
        c.serve_forever()


# Authour: Jaimin Kirankumar Kerai, v1.0
# A program which counts the nummber of specified punctuation and words
# It will normalise this data by dividing each count by the number of sentences
# With the counts (and normalising) the program is able to 'compare' to texts by determining
# the distance between each
# The program accepts up to three parameters, in four different situations, the last arguement is assumed to be False unless stated
# The first being: (filename of the first text, 'listing'), this will list the counts of the punctuation or words
# The second being: (filename of first text, filename of second text), this will determine the 'distance' between the texts
# The third being: (filename of the first text, 'listing', True), this will list the normalised counts of the punctuation or words
# The fourth being: (filename of first text, filename of second text, True), this will determine the normalised 'distance' between the texts

import math

# This main function will complete all major concluding computations and outputs
def main(textfile1, arg2, normalize=False):
    # The determining the route according to the inputs
    if arg2 == 'listing':
        # Non-normalised path
        if normalize == False:
            # Calling the function to count the words and punctuation as well as the paragraph, sentence and word count
            profile1, words_per_sent, sent_per_para,_= word_punctuation_count(textfile1)
            profile1['words_per_sentence'] = words_per_sent
            profile1['sentences_per_par'] = sent_per_para
            # The print output of the dictionaries
            print("The pure count of the punctuation and words of the chosen file:")
            for key, value in sorted(profile1.items()):
                print('{}: {}'.format(key,format(value,'.4f')))
        # Normalised path
        elif normalize == True:
            profile1,_,_,_= word_punctuation_count(textfile1)
            profile1 = normalize_data(textfile1)
            # The print output of the dictionaries
            print("The normalized count of the punctuation and words of the chosen file:")
            for key, value in sorted(profile1.items()):
                print('{}: {}'.format(key,format(value,'.4f')))
        # The program will exit if an invalid third arguement is entered
        else:
            exit()
    # if the input is a file, or not 'listing', the following will compute
    else:
        # Non-normalised pathway
        if normalize == False:
            # Calling the functions to count the words and punctuation as well as the paragraph, sentence and word count
            profile1, words_per_sent1, sent_per_para1,_= word_punctuation_count(textfile1)
            profile2, words_per_sent2, sent_per_para2,_= word_punctuation_count(arg2)
            profile1['words_per_sentence'] = words_per_sent1
            profile1['sentences_per_par'] = sent_per_para1
            profile2['words_per_sentence'] = words_per_sent2
            profile2['sentences_per_par'] = sent_per_para2
            # Calling functions to determine the distance between two calculated profiles
            difference_sum = distance(profile1, profile2)
            # The print output of the difference sum
            print("The distance between the two texts is:", difference_sum)
        # Normalised pathway
        elif normalize  == True:
            # The two profiles will be calculated as normalised
            profile1 = normalize_data(textfile1)
            profile2 = normalize_data(arg2)
            # The difference of sum will be computed
            difference_sum = distance(profile1, profile2)
            # The print output of the difference sum
            print("The normalized distance between the two texts is:", difference_sum)
        # The program will exit if an invalid second arguement is entered
        else:
            exit()

# A function designed to take one input, the file, and calculate the count of the paragraphs, sentences and words.
# It will also count the number of specific punctuation and words, then add these into a dictionary labelled as the profile
def word_punctuation_count(file):
    # A list of the punctuation required to count
    punctuation_list = [',',';',"'",'-']
    # A list of the words required to count
    words_list = ["also", "although", "and", "as", "because", "before", "but", "for", "if", "nor", "of", "or", "since", "that","though", "until", "when", "whenever", "whereas", "which", "while", "yet"]
    # Initialisation of dictionaries and variables
    profile = {}
    words_in_text = 0
    # Opening the file and turning the text into a list seperated by each word and punctuation
    input_file = open(file,'r')
    # Removal of unnessary characters and addition of seperator characters
    count_file = input_file.read().replace('.',' ').replace('?',' ').replace('!',' ').replace('\n\n\n',' ').replace('--\n\n',' ').replace('\n\n',' ').replace('--',' ').replace('\n',' ').replace('-',' - ').replace(',',' ,').replace('; ',' ; ').replace('"','').replace(" '",' ').replace("' ",' ').replace("'"," ' ").replace(","," , ").lower().split(' ')
    # A loop to count each of the punctuation and add it to the profile
    for punctuation in punctuation_list:
        punct_count = count_file.count(punctuation)
        profile[punctuation] = punct_count
    # A loop to count each of the specific words and add it to the profile
    for words in words_list:
       word_count = count_file.count(words)
       profile[words] = word_count
    input_file.close()
    # Opening the file and turning the text into a list seperated by paragraphs
    input_file = open(file,'r')
    # Removal of unnessary characters and addition of seperator characters
    words_file = input_file.read().replace(';','').replace(',','').replace(':','').replace("'",'').replace('"','').replace('?','.').replace('!','.').replace('\n\n\n','<>').replace('--\n\n','<>').replace('\n\n','<>').replace('\n',' ').replace('--',' ').replace('-','').replace('@',' ').replace('$','').replace('%','').split('<>')
    # Counting of paragraphs and words
    for paragraph in words_file:
        paragraph_in_text = len(words_file)
        paragraph = paragraph.split('. ')
        for sentence in paragraph:
            sentence = sentence.split(' ')
            words_in_sentence = len(sentence)
            words_in_text += words_in_sentence
    input_file.close()
    # Opening the file and turning the text into a list seperated by sentences
    input_file = open(file,'r')
    # Removal of unnessary characters and addition of seperator characters
    sentence = input_file.read().replace(';','').replace(',','').replace(':','').replace("'",'').replace('"','').replace('?','.').replace('!','.').replace('\n\n\n',' ').replace('--\n\n',' ').replace('\n',' ').replace('--',' ').replace('-','')
    sentences_in_text = len(sentence.strip().split('. '))
    input_file.close()
    # Calculating final values and returning required the profile and variables
    words_per_sent = words_in_text/sentences_in_text
    sent_per_para = sentences_in_text/paragraph_in_text
    return profile, words_per_sent, sent_per_para, sentences_in_text

# A function designed to take a file as an input and run the above function.
# The profile and other returned values are used to normalise each data point of the dictionary
def normalize_data(file):
    # Aquiring the profile from running the word_punctuation_count function
    profile, words_per_sent, sent_per_para, sentences_in_text  = word_punctuation_count(file)
    # Updating each normalised data point in the profile
    for key, value in profile.items():
        profile[key] = value/sentences_in_text
    # Adding the non-normalised values into the profile
    profile['words_per_sentence'] = words_per_sent
    profile['sentences_per_par'] = sent_per_para
    # Returning completely filled/normalised profile
    return profile

# A function designed to take two profiles of two texts as the input
# Then determining the distance between these two profiles
def distance(profile1, profile2):
    # Initialising the variable for the sum
    difference_sum = 0
    # Looping through the values of the profiles and using such to determine the difference sum of each
    for key,_ in profile1.items():
        d = profile1[key] - profile2[key]
        difference_sum += d**2
    # Doing the final calculation and formating the float to 4 decimal places
    difference_sum = format(math.sqrt(difference_sum),'.4f')
    # Returning the difference sum
    return difference_sum


import pytest

from app.models.items import Item
from app.repositories.items import ItemRepository

@pytest.fixture()
def item_repository():
    repository = ItemRepository()
    yield repository
    repository.delete_all_items()

@pytest.fixture
def setup_test_data(item_repository):
    item1 = Item(
        id=1,
        name="Item 1",
        category="Stationary",
        price=5.5,
        created_dt="2022-01-01 10:00:00",
        last_updated_dt="2022-01-01 10:00:00",
    )
    item_repository.create_item(item1)

    item2 = Item(
        id=2,
        name="Item 2",
        category="Gift",
        price=18,
        created_dt="2022-01-01 10:00:00",
        last_updated_dt="2022-01-30 10:00:00",
    )
    item_repository.create_item(item2)

    item3 = Item(
        id=3,
        name="Item 3",
        category="Gift",
        price=2,
        created_dt="2022-01-01 10:00:00",
        last_updated_dt="2022-01-30 10:00:00",
    )
    item_repository.create_item(item3)


#!/usr/bin/env python
# coding: utf-8

# In[156]:

import streamlit as st
import pickle
import pandas as pd

# In[162]:

#links = pd.read_csv("/Users/hi/Desktop/data_science_wbs_school/Recommender_system/project/c1ba2d8cbaa22297e5d9b0b7a17fcb7awbsflix-dataset/ml-latest-small/links.csv")

#movies = pd.read_csv("/Users/hi/Desktop/data_science_wbs_school/Recommender_system/project/c1ba2d8cbaa22297e5d9b0b7a17fcb7awbsflix-dataset/ml-latest-small/movies.csv")
#ratings = pd.read_csv("/Users/hi/Desktop/data_science_wbs_school/Recommender_system/project/c1ba2d8cbaa22297e5d9b0b7a17fcb7awbsflix-dataset/ml-latest-small/ratings.csv")
#tags = pd.read_csv("/Users/hi/Desktop/data_science_wbs_school/Recommender_system/project/c1ba2d8cbaa22297e5d9b0b7a17fcb7awbsflix-dataset/ml-latest-small/tags.csv")

# In[163]:

movies_dict = pickle.load(open('movies_dict.pkl', 'rb'))
movies= pd.DataFrame(movies_dict)

rating_final = pickle.load(open('rating_final.pkl', 'rb'))
ratings = pickle.load(open('ratings.pkl', 'rb'))
rating_avg = pickle.load(open('rating_avg.pkl', 'rb'))
rating_count = pickle.load(open('rating_count.pkl', 'rb'))
users = pickle.load(open('users.pkl', 'rb'))
user_Id = pickle.load(open('user_Id.pkl', 'rb'))
movie_rating_tab = pickle.load(open('movie_rating_tab.pkl', 'rb'))
rating_final1 = pickle.load(open('rating_final1.pkl', 'rb'))

# In[180]:

st.set_page_config(page_title="WEBFLIX", layout="wide")
st.title('WBSFLIX')

#search_choices = ['Movie title', 'Genre', 'Rating', 'Tags']
#search_selected = st.sidebar.selectbox("Your choice please", search_choices)
movie_name = movies['title'].values

#dropdown_movie = st.selectbox('What would you like to watch today?', movie_name)

selected_movie_name= st.selectbox('What would you like to watch today?',movie_name)

st.write('You selected:', selected_movie_name)

def get_recommendations (movie_name,n=20):
    movie_rating_tab= pd.pivot_table(data=rating_final, values='rating', index='userId', columns='title')
      #movie_rating_tab = movie_rating_tab.fillna(0)
    rest_movie_ratings = movie_rating_tab[movie_name]
    similar_to_rest = movie_rating_tab.corrwith(rest_movie_ratings)
    corr_rest = pd.DataFrame(similar_to_rest, columns = ['Pearson_coeif'])
    corr_rest.dropna(inplace=True)
    corr_rest.drop(movie_name, inplace=True)
    similar = corr_rest.merge(rating_final, on = "title")
    similar = similar.drop(['rating', 'timestamp', 'userId'], axis=1)
    similar = similar.drop_duplicates("title")
    similar_final= similar[similar['rating_count']>=100].sort_values(by = 'Pearson_coeif', ascending = False).head(n)
    similar_final = similar_final.drop(["overall_rating","rating_count","Pearson_coeif"], axis=1)
    similar_final["rating_avg"] = round(similar_final["rating_avg"], 1)
    similar_final["Year"]= similar_final["title"].str.findall("\d{4}")
    similar_final["title"]= similar_final["title"].str.replace('(\(\d{4})\)', '')
    similar_final = similar_final[['movieId', 'title', 'Year', 'genres', 'rating_avg']]
    similar_final_new= similar_final.set_index("movieId")
    return(similar_final_new)

with st.container():
    st.subheader('Top picks for you based on the movie you selected :')
    buffer1, col1, buffer2 = st.columns([2.5, 3, 1])
    is_clicked = col1.button(label = 'Recommended')
    if is_clicked :
        recommendations = get_recommendations(selected_movie_name)
        columns = st.columns((1, 4, 1))
        columns[1].write(recommendations)

def popular_table (movie_count):

  rating_avg = pd.DataFrame(ratings.groupby('movieId')['rating'].mean())
  rating_avg['overall_rating'] = round((ratings.groupby('movieId')['rating'].mean()))
  rating_count = pd.DataFrame(ratings.groupby('movieId')['rating'].count())
  rating_final  = rating_avg.merge(rating_count, on = 'movieId').merge(movies, on = "movieId")
  rating_final.rename(columns = {'rating_x':'rating_avg', 'rating_y':'rating_count'}, inplace = True)
  rating_final = rating_final[rating_final['rating_count']>=100].sort_values("rating_count", ascending=False)
  rating_final = rating_final.drop_duplicates(["title"])
  top_movies = rating_final.head(movie_count)

  return(top_movies)

# list of top 15 popular movies in our database
popular_movies = pickle.load(open('popular_all.pkl', 'rb'))

#session.options = st.multiselect(label="Select Movies", options=popular_movies)

#dataframe = None

#session.slider_count = st.slider(label="movie_count", min_value=5, max_value=138)

#with st.container():
   # is_clicked = col1.button(label="Popular")
    #if is_clicked:
       # dataframe = popular_table(session.options, movie_count = session.slider_count, data= popular_movies)
   # if dataframe is not None:
        #st.table(dataframe)

with st.container():
    st.subheader('Top 20:')
    buffer1, col1, buffer2 = st.columns([2.8, 3, 1])
    is_clicked = col1.button(label = "Popular")
    if is_clicked:
        columns = st.columns((0.5, 0.8, 0.3))
        columns[1].write(popular_movies)

def special_picks_for_you(user_id, n=30):
    user_rating_tab = movie_rating_tab.T.fillna(0)
  # computing cosine similarities
    from sklearn.metrics.pairwise import cosine_similarity
    user_similarities = pd.DataFrame(cosine_similarity(user_rating_tab),
                                  columns=user_rating_tab.index,
                                  index=user_rating_tab.index)
  # computing the weights for the users
    weights = (
        user_similarities.query("userId!=@user_id")[user_id] / sum(user_similarities.query("userId!=@user_id")[user_id])
            )
  # select movies that the inputed user has not rated, # dataframe for the non rated movies by all the user
    not_rated_movies = user_rating_tab.loc[user_rating_tab.index!=user_id, user_rating_tab.loc[user_id,:]==0]

    weighted_averages = pd.DataFrame(not_rated_movies.T.dot(weights), columns=["predicted_rating"])
    recommendations = weighted_averages.merge(movies, left_index=True, right_on="title").merge(rating_final1, on ="movieId").merge(users, on ="movieId").drop_duplicates("title")
    special_picks = recommendations[recommendations['rating_count']>100].sort_values("predicted_rating", ascending=False).head(n)
    special_picks = special_picks.drop(["userId","overall_rating","rating_count"], axis=1)
    special_picks["rating_avg"] = round(special_picks["rating_avg"], 1)
    special_picks["predicted_rating"] = round(special_picks["predicted_rating"], 1)

    special_picks["Year"]= special_picks["title"].str.findall("\d{4}")
    special_picks["title"]= special_picks["title"].str.replace('(\(\d{4})\)', '')
    special_picks_new = special_picks[['movieId', 'title', 'Year', 'genres', 'rating_avg', 'predicted_rating']]

    special_picks_new = special_picks_new.set_index('movieId')
    return(special_picks_new)

with st.container():
    st.subheader('Search by user Id:')
    user_id = user_Id
    user_choices =st.selectbox('See what other user have liked by User Id', user_id)
    buffer1, col1, buffer2 = st.columns([3, 3, 1])
    is_clicked = col1.button(label = 'Show')
    if is_clicked :
        special_picks = special_picks_for_you(user_choices)
        columns = st.columns((1, 6.8, 1))
        columns[1].write(special_picks)

#import streamlit.components.v1 as components

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:


import connexion
import yaml
import logging
from connexion import NoContent
import os
import logging.config
import json
import datetime
from apscheduler.schedulers.background import BackgroundScheduler
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.sql.elements import and_
import datetime
import requests
from flask_cors import CORS, cross_origin

try:
    with open('/config/app_conf.yaml', 'r') as f:
        app_config = yaml.safe_load(f.read())
except:
    with open('app_conf.yaml', 'r') as f:
        app_config = yaml.safe_load(f.read())

STORE_SURGERY_INFO_REQUEST_URL = app_config['datastore']['hostname'] + ":8090/report/book_surgery"
STORE_XRAY_REPORT_REQUEST_URL = app_config['datastore']['hostname'] + ":8090/report/xRay"
HEADERS = {"content-type":"application/json"}

DB_ENGINE = 'mysql+pymysql://' + app_config['datastore']['user'] + ':' + app_config['datastore']['password'] + '@' + \
    app_config['datastore']['hostname'] + ':' + app_config['datastore']['port'] + '/' + app_config['datastore']['db']
#Base.metadata.bind = DB_ENGINE
DB_SESSION = sessionmaker(bind=DB_ENGINE)

def get_hospital_stats():
    """ Get surgery info from the data store """

    logger = logging.getLogger('basicLogger')
    logger.info("Request started")

    if os.path.exists(app_config['datastore']['filename']):
        with open(app_config['datastore']['filename'], 'r') as f:
            data = json.loads(f.read())
            data = data['stats']
            logger.debug(data)
            logger.info("Request completed")
            return data, 200

    else:
        print("File not found")
        return 404

        with open(app_config['datastore']['filename'], 'w') as outfile:
            json.dump(data, outfile)

    return NoContent, 201

def populate_stats():
    """ Periodically update stats """
    logger = logging.getLogger('basicLogger')
    logger.info("Start Periodic Processing")

    # logger.info(parsed_json)

    now = datetime.datetime.now()
    current_time = now.strftime("%Y-%m-%dT%H:%M:%S")

    if os.path.exists(app_config['datastore']['filename']):
        with open(app_config['datastore']['filename'], 'r') as f:
            data = json.loads(f.read())

    else:
        data = {}
        data['stats'] = []
        data['stats'].append({
            'numXrays': 0,
            'numSurgeries': 0,
            'timestamp': "2010-01-23T11:50:40"
        })
        with open(app_config['datastore']['filename'], 'w') as f:
            f.write(json.dumps(data))

    numXrays = data['stats'][0]["numXrays"]
    numSurgeries = data['stats'][0]["numSurgeries"]

    params = {
        "startDate": data['stats'][0]['timestamp'],
        "endDate": current_time
      }

    surgeryResponse = requests.get(STORE_SURGERY_INFO_REQUEST_URL, params=params)
    xrayResponse = requests.get(STORE_XRAY_REPORT_REQUEST_URL, params=params)

    if(surgeryResponse.status_code == 200 and xrayResponse.status_code == 200):
        logger.info("Events recieved - surgeries: %s" % len(surgeryResponse.json()))
        logger.info("Events recieved - xRays: %s" % len(xrayResponse.json()))

        numXrays += len(xrayResponse.json())
        numSurgeries += len(surgeryResponse.json())

        newData = {}
        newData['stats'] = []
        newData['stats'].append({
            'numXrays': numXrays,
            'numSurgeries': numSurgeries,
            'timestamp': current_time
        })

        logger.debug(newData)
        with open(app_config['datastore']['filename'], 'w') as f:
            f.write(json.dumps(newData))
        logger.debug("Period processing has ended")

    else:
        logger.error("200 status code not recieved")

def init_scheduler():
    sched = BackgroundScheduler(daemon=True)
    sched.add_job(populate_stats, 'interval',
                  seconds=app_config['scheduler']['period_sec'])
    sched.start()

app = connexion.FlaskApp(__name__, specification_dir='')
app.add_api("openapi.yaml")
CORS(app.app)
app.app.config['CORS_HEADERS'] = 'Content-Type'

with open('app_conf.yaml', 'r') as f:
    app_config = yaml.safe_load(f.read())

with open('log_conf.yaml', 'r') as f:
    log_config = yaml.safe_load(f.read())
    logging.config.dictConfig(log_config)

if __name__ == "__main__":
# run our standalone gevent server
    init_scheduler()
    app.run(port=8100, use_reloader=False)


import numpy as np
import tensorflow as tf
xy=np.loadtxt('d:/dl/data/zoo.csv',delimiter=",",dtype=np.float32)
xdata = xy[:,0:-1]
ydata = xy[:,[-1]]
# print(xdata.shape,ydata.shape)
nb_classes=7 #0~6
x=tf.placeholder(tf.float32,[None,16])
y=tf.placeholder(tf.int32,[None,1])

#y에는 0~6사이의 임의의 수 저장
#원핫 인코딩 해야함.
y_one_hot =tf.one_hot(y, nb_classes)
print("one hot 상태: ",y_one_hot)
#0 -> 1000000, 3 -> 0001000
#원핫 인코딩을 수행하면 차원이 1 증가
#예를 들어 y 가 (None,1) ->(None,1,7)이 됨
#[[0],[3]]->[[[1000000]],[[0001000]]]
y_one_hot=tf.reshape(y_one_hot,[-1,nb_classes]) #--> -1은 전체 데이터를 정의
print("reshape 결과: ",y_one_hot)

w=tf.Variable(tf.random_normal([16,nb_classes]))
b=tf.Variable(tf.random_normal([nb_classes]))
logits =tf.matmul(x,w)+b #logit=score
hf=tf.nn.softmax(logits)

cost_i =tf.nn.softmax_cross_entropy_with_logits(logits=logits,labels=y_one_hot)

cost = tf.reduce_mean(cost_i)
optimizer = tf.train.GradientDescentOptimizer(0.1).minimize(cost)

prediction=tf.argmax(hf,axis=1)
correct_prediction=tf.equal(prediction,tf.argmax(y_one_hot,axis=1))

accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    for step in range(2001):
        sess.run(optimizer, feed_dict={x:xdata,y:ydata})
        if step%100==0:
            cv,av=sess.run([cost,accuracy],feed_dict={x:xdata,y:ydata})
            print(step, cv, av)
    print(sess.run(prediction,feed_dict={x:[[0.,0.,1.,0.,0.,
                                       1.,1.,1.,1.,0.,
                                       0.,1.,0.,1.,0.,0.]]}))


import gc
import os
import cv2
import torch
import torch.nn.functional as F
import numpy as np
from tqdm import tqdm
from transformers import DistilBertTokenizer
import matplotlib.pyplot as plt
import argparse

import config as CFG
from dataset import get_transforms
from pretrain import build_loaders, make_train_valid_dfs
from CLIP import CLIPModel

def get_image_embeddings(valid_df, model_path):
    tokenizer = DistilBertTokenizer.from_pretrained(CFG.text_tokenizer)
    valid_loader = build_loaders(valid_df, tokenizer, mode="valid")

    model = CLIPModel().to(CFG.device)
    model.load_state_dict(torch.load(CFG.model_path, map_location=CFG.device))
    model.eval()

    valid_image_embeddings = []
    with torch.no_grad():
        for batch in tqdm(valid_loader):
            image_features = model.image_encoder(batch["image"].to(CFG.device))
            image_embeddings = model.image_projection(image_features)
            valid_image_embeddings.append(image_embeddings)
    return model, torch.cat(valid_image_embeddings)

def get_embeddings(text_file, image_file):
    model = CLIPModel().to(CFG.device)
    model.load_state_dict(torch.load(CFG.model_path, map_location=CFG.device))
    model.eval()

    # get text query
    f = open(text_file)
    query = f.readlines()
    query = list(map(lambda x:x.replace("\n", ""), query))

    tokenizer = DistilBertTokenizer.from_pretrained(CFG.text_tokenizer)
    encoded_query = tokenizer(query, padding=True, truncation=True, max_length=CFG.max_length)
    batch = {
        key: torch.tensor(values).to(CFG.device)
        for key, values in encoded_query.items()
    }

    # get image
    image = cv2.imread(image_file)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

    transforms = get_transforms()
    image = transforms(image=image)['image']
    image = torch.tensor(image).permute(2, 0, 1).float()
    image = torch.reshape(image, ((1,) + image.shape))

    # get text & image embedding
    with torch.no_grad():
        image_features = model.image_encoder(image.to(CFG.device))
        image_embeddings = model.image_projection(image_features)
        text_features = model.text_encoder(
            input_ids=batch["input_ids"], attention_mask=batch["attention_mask"]
        )
        text_embeddings = model.text_projection(text_features)

    # find match
    image_embeddings_n = F.normalize(image_embeddings, p=2, dim=-1)
    text_embeddings_n = F.normalize(text_embeddings, p=2, dim=-1)
    dot_similarity = text_embeddings_n @ image_embeddings_n.T
    dot_similarity = dot_similarity.T
    _, indices = torch.topk(dot_similarity.squeeze(0), 3)
    matches = [query[idx] for idx in indices]
    print(matches)

def find_matches(text_embeddings, image_embeddings, query):
    image_embeddings_n = F.normalize(image_embeddings, p=2, dim=-1)
    text_embeddings_n = F.normalize(text_embeddings, p=2, dim=-1)
    dot_similarity = text_embeddings_n @ image_embeddings_n.T

    _, indices = torch.topk(dot_similarity.squeeze(0), 3)
    matches = [image_filenames[idx] for idx in indices[::5]]

    _, axes = plt.subplots(3, 3, figsize=(10, 10))
    for match, ax in zip(matches, axes.flatten()):
        image = cv2.imread(f"{CFG.image_path}/{match}")
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        ax.imshow(image)
        ax.axis("off")

    plt.show()
    query_ = query.replace(" ","_")
    result_filename = query_ + "_" + CFG.model_name + "_" + CFG.dataset + "_result.png"
    plt.savefig(result_filename)

def main(args):
    if (args.query_text is None) or (args.query_image is None):
        raise Exception("Usage :[python3 inference_image_to_text.py --query_text={YOUR QUERY TEXT FILE} --query_image={YOUR QUERY IMAGE FILE}]")

    get_embeddings(args.query_text, args.query_image)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--query_text', type=str)
    parser.add_argument('--query_image', type=str)
    args = parser.parse_args()
    main(args)


# -*- coding: utf-8 -*-
"""Logging utilities."""
import sys
from datetime import datetime

from loguru import logger

def create_logger(script_path, subdir="logs"):
    """Create a logger using loguru."""
    logpath = script_path.parent / subdir
    logpath.mkdir(exist_ok=True)
    logger.configure(
        handlers=[
            {"sink": sys.stdout, "level": "INFO"},
            {
                "sink": logpath
                / f"log_{script_path.stem}_{datetime.now():%Y-%m-%d_%H%M%S}.log",
                "level": "DEBUG",
            },
        ]
    )
    return logger


def stringPermutation(word):
    try:
        n=int(word)
        print("inputan tidak valid")
    except:
        for i in range(len(word)):
            word = word[-1]+word[:-1]
            print(word,end="|")
input_word = input("Masukkan kata: ")
stringPermutation(input_word)


import pytz
import json
import hashlib
import pandas as pd
import quantutils.dataset.pipeline as ppl
import marketinsights.utils.http as http
from marketinsights.remote.models import MIModelServer
from marketinsights.remote.ibmcloud import CloudFunctions
from marketinsights.utils.auth import CredentialsStore

class MIAssembly():

    def __init__(self, modelSvr=None, credentials_store=None, secret=None):

        if not credentials_store:
            credentials_store = CredentialsStore()

        self.fun = CloudFunctions(credentials_store)
        if not secret:
            secret = "marketinsights-local-cred"
        self.credentials = credentials_store.getSecrets(secret)
        self.modelSvr = modelSvr

    def put_dataset(self, data, dataset_desc, market, debug=False):
        dataset = Dataset.csvtojson(data, dataset_desc, market)
        headers = {
            'X-IBM-Client-Id': self.credentials["clientId"],
            'X-IBM-Client-Secret': self.credentials["clientSecret"],
            'content-type': 'application/json'
        }
        url = "".join([self.credentials["mi-api-endpoint"], "/v1/datasets"])
        return http.put(url=url, headers=headers, data=dataset, debug=debug)

    def get_dataset(self, dataset_desc, market, debug=False):
        return self.get_dataset_by_id(Dataset.generateMarketId(dataset_desc, market), debug)

    def get_dataset_by_id(self, dataset_id, debug=False):
        headers = {
            'X-IBM-Client-Id': self.credentials["clientId"],
            'X-IBM-Client-Secret': self.credentials["clientSecret"],
            'accept': 'application/json'
        }

        # TODO eliminate id, no need.
        query = {
            'where': {
                'id': dataset_id,
            }
        }
        url = "".join([self.credentials["mi-api-endpoint"], "/v1/datasets?filter=", json.dumps(query)])
        resp = http.get(url=url, headers=headers, debug=debug)
        dataset = resp[0]
        return [Dataset.jsontocsv(dataset), dataset["dataset_desc"]]

    # TODO  (without getting data too)
    def get_dataset_desc(self):
        pass

    def put_model(self, data, debug=False):
        headers = {
            'X-IBM-Client-Id': self.credentials["clientId"],
            'X-IBM-Client-Secret': self.credentials["clientSecret"],
            'content-type': 'application/json'
        }
        url = "".join([self.credentials["mi-api-endpoint"], "/v1/models"])
        return http.put(url=url, headers=headers, data=json.dumps(data), debug=debug)

    def get_model(self, modelId, debug=False):
        headers = {
            'X-IBM-Client-Id': self.credentials["clientId"],
            'X-IBM-Client-Secret': self.credentials["clientSecret"],
            'accept': 'application/json'
        }
        url = "".join([self.credentials["mi-api-endpoint"], "/v1/models/", modelId])
        return http.get(url=url, headers=headers, debug=debug)

    def put_training_run(self, data, debug=False):
        headers = {
            'X-IBM-Client-Id': self.credentials["clientId"],
            'X-IBM-Client-Secret': self.credentials["clientSecret"],
            'content-type': 'application/json'
        }
        url = "".join([self.credentials["mi-api-endpoint"], "/v1/training_runs"])
        return http.put(url=url, headers=headers, data=json.dumps(data), debug=debug)

    def get_training_run(self, training_run_id, debug=False):
        headers = {
            'X-IBM-Client-Id': self.credentials["clientId"],
            'X-IBM-Client-Secret': self.credentials["clientSecret"],
            'accept': 'application/json'
        }
        url = "".join([self.credentials["mi-api-endpoint"], "/v1/training_runs/", training_run_id])
        return http.get(url=url, headers=headers, debug=debug)

    def get_predictions_with_dataset(self, dataset, training_run_id, labels=1, debug=False):

        if not self.modelSvr:
            raise Exception("No Model Server configured")

        features = dataset.iloc[:, :-labels]
        # Send the dataset features to the model and retrieve the scores (predictions)
        return pd.concat([dataset, self.modelSvr.getPredictions(features, training_run_id, debug=debug)], axis=1)

    def get_predictions_with_dataset_id(self, dataset_id, training_run_id, start=None, end=None, debug=False):

        # Get the dataset from storage, crop and strip out labels
        dataset, dataset_desc = self.get_dataset_by_id(dataset_id, debug)

        if debug:
            print(dataset)

        if dataset.empty:  # No predictions in this time range
            return None

        return self.get_predictions_with_dataset(dataset, training_run_id, labels=dataset_desc["labels"], debug=debug)

    def get_predictions_with_raw_data(self, data, training_id, debug=False):

        training_run = self.get_training_run(training_id)
        if debug:
            print("Training run : " + str(training_run))

        dataset_id = training_run["datasets"][0]
        _, dataset_desc = self.get_dataset_by_id(dataset_id)
        pipeline = dataset_desc["pipeline"]

        # Generate a dataset from the raw data through the given pipeline
        dataset = Dataset().createDataset(pipeline["id"], data, pipeline["pipeline_desc"], debug)

        if dataset.empty:
            return dataset

        #dataset = dataset.iloc[:, :-dataset_desc["labels"]]
        if debug:
            print("Sending feature vector : " + str(dataset))

        return self.get_predictions_with_dataset(dataset, training_id, labels=dataset_desc["labels"], debug=debug)

    @staticmethod
    def generateMarketId(dataset_desc, market):
        return hashlib.md5("".join([market, json.dumps(dataset_desc["pipeline"], sort_keys=True), str(dataset_desc["features"]), str(dataset_desc["labels"])]).encode('utf-8')).hexdigest()

    @staticmethod
    def generateTrainingId(dataset_desc, model_id, name=None):
        training_id = str(hashlib.md5(f'{str(dataset_desc)}{str(model_id)}'.encode('utf-8')).hexdigest())
        if name:
            training_id = f"{name}-{training_id}"
        return training_id

class Dataset:

    def __init__(self, fun=None, credentials_store=None):

        if not credentials_store:
            credentials_store = CredentialsStore()
        if not fun:
            fun = CloudFunctions(credentials_store)
        self.fun = fun

    def createDataset(self, pipeline, data, config, debug=False):

        if debug:
            print(f"Pipeline info - ID: {pipeline}, Config: {str(config)}")
            print(f"Sending data to pipeline : {str(data)}")

        data = {
            "data": json.loads(data.to_json(orient='split', date_format="iso")),
            "dataset": config
        }

        response = self.fun.call_function(pipeline, data, debug)

        if debug:
            print("Pipeline response : " + str(response))

        if response:
            dataset = pd.read_json(json.dumps(response), orient='split', dtype=False)
            if not dataset.empty:  # Check for empty dataset
                dataset.index.names = ["Date_Time"]  # Workaround for lack of index name in marshalling
                dataset = ppl.localize(dataset, "UTC", config["timezone"])  # Marshalling turns dates to UTC
            return dataset
        else:
            raise Exception("No response received from pipeline execution")

    @staticmethod
    def csvtojson(csv, dataset_desc, market, createId=True):
        obj = {}
        if (createId):
            obj["id"] = Dataset.generateId(dataset_desc, market)
        obj["dataset_desc"] = dataset_desc
        obj["market"] = market
        obj["data"] = csv.values.tolist()
        obj["tz"] = csv.index.tz.zone
        obj["index"] = [date.isoformat() for date in csv.index.tz_localize(None)]  # Remove locale
        return json.dumps(obj)

    @staticmethod
    def jsontocsv(jsonObj):
        return pd.DataFrame(jsonObj["data"], index=pd.DatetimeIndex(jsonObj["index"], name="Date_Time", tz=pytz.timezone(jsonObj["tz"])))