import json
import sys
from typing import List
import os
import secrets
import string
from requests import Response
from json import loads
import shutil

def clean_up():
    shutil.rmtree('temp/reads/')
    shutil.rmtree('temp/writes/')

def create_dirs(dirs: List[str] = ['temp/']):
    """creates directories

    Args:
        dirs (List[str], optional): list of dir to create. Defaults to ['temp/'].
    """
    for dir in dirs:
        if not os.path.isdir(dir):
            os.mkdir(dir)

def structure_result(response: Response) -> dict:
    """extrracts the results from a response object

    Args:
        response (Response): _description_

    Returns:
        dict: _description_
    """
    if not (response.content or response.text):
        content = None,
    else:
        content = loads(
            response.content if response.content else response.text)

    return {
        'method': response.request.method,
        'url': response.url,
        'rtt': response.elapsed.total_seconds(),
        'status_code': response.status_code,
        'content': content
    }

def generate_customer_object(id: str, size: int = 0) -> dict:
    """generates a customer object

    Args:
        id (str): customer id
        size (int): size of object in bytes

    Returns:
        dict: customer dict
    """
    customer = {
        "customer_id": id,
        "email": "someone@example.com",
        "name": id,
        "address": {
            "street": "123 Some Street",
            "city": "Somewhere"
        },
        'timestamp': '0.0'
    }

    customer_size = len(json.dumps(customer))

    if size > customer_size:
        customer['address']['street'] = customer['address']['street'] + \
            ''.join(['x' for _ in range(((size - customer_size) // 4))])
        customer['address']['city'] = customer['address']['city'] + \
            ''.join(['x' for _ in range(((size - customer_size) // 4))])
        customer['name'] = customer['name'] + \
            ''.join(['x' for _ in range(((size - customer_size) // 4))])
        customer['email'] = customer['email'] + \
            ''.join(['x' for _ in range(((size - customer_size) // 4))])

    return customer

def create_directory(name: str, path: str = ''):
    """_summary_

    Args:
        name (str): directory name
        path (str): directory path. defaults to '/'
    """

    os.mkdir(f'{path}/{name}')

def generate_random_id(length: int = 10) -> str:
    """generates a random id

    Args:
        length (int, optional): length of generated id. Defaults to 10.

    Returns:
        str: id
    """
    return ''.join(secrets.choice(
        string.ascii_letters + string.digits) for _ in range(length))

def generate_customer_ids(length: int = 10) -> List[str]:
    """generates a list of random id to use for an experiment

    Args:
        length (int, optional): number of ids. Defaults to 10.

    Returns:
        List[str]: list of random ids
    """

    return [generate_random_id() for _ in range(length)]


import pandas as pd
import os
from datetime import datetime
from pathlib import Path

PATH = Path(os.getcwd())
DATA_PATH = os.path.join(PATH, 'data')

def clean_column_names(df):
    """
        Splits the column names on the space (' ') and joins
        them with '_' in order to make it easier to use.
    """

    df.columns = ['_'.join(col.lower().split()) for col in df.columns]
    return df

def read_file(path, skiprows=0):
    """
        Function to simplify the reading of files. Uses the DATA_PATH
        in order to fetch the files from the right
        folder.
    """
    if path.endswith('.xlsx'):
        df = pd.read_excel(os.path.join(DATA_PATH, path),
                           skiprows=skiprows, engine='openpyxl')

    if path.endswith('.csv'):
        df = pd.read_csv(os.path.join(DATA_PATH, path))

    df = clean_column_names(df)
    return df

def str_to_datetime(date_str):
    """
       Converts string object to datetime object. In case null, returns None.
    """
    if date_str is None:
        return None

    return datetime.strptime(date_str, '%d-%m-%Y')

def categorise_duration(months):
    """
        Buckets float month value into string categories like '<2 months',
        '2-4 months', etc.
    """
    MAX_COURSE_DURATION = 45

    for i in range(0, MAX_COURSE_DURATION, 2):
        if (months > i) and (months <= i+2):
            return f'{i if i>=10 else "0" + str(i)}-{i+2 if i+2>=10 else "0" + str(i+2)} months'


import random
import numpy as np
import pandas as pd
import tensorflow as tf
from scipy import stats

from scripts import Slopes
import matplotlib.pyplot as plt
import seaborn as sns

def reset_seeds(seed=42):
    np.random.seed(seed)
    random.seed(seed)
    tf.random.set_seed(seed)

def joonista(m, X, y, bpoint_fn=Slopes.breakpoint_finder, max_delta=0.0001):
    X_pred = np.linspace(min(X)-0.2, max(X)+0.2, 1000)
    X_vahemik = X

    bpoints = bpoint_fn(m, X_pred, max_delta)

    if len(bpoints) != 0 and isinstance(bpoints[0][1], np.ndarray):
        bpoints = [bp[0] for bp in bpoints]
    print(f"model contains {len(bpoints)} breaks")

    _fig, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(15, 5))
    ax1.scatter(X, y, marker="+", color='red', label="target")
    ax1.plot(X_pred, m.predict(X_pred, batch_size=32768,), label="prediction",  linewidth=0.5)
    if len(bpoints) != 0:
        ax1.scatter(*zip(*bpoints), marker="x",
                    color='magenta', label="breaks")
        ax1.legend()
    ax2.scatter(X, y, marker="+", color='red', label="target")
    plt.show()
    return _fig, (ax1, ax2)

def joonista_temp(m, X, y, bpoint_fn=Slopes.breakpoint_finder, max_delta=0.0001):
    X_pred = np.linspace(-1.2, 1.2, 1000)
    X_vahemik = X

    bpoints = bpoint_fn(m, X_vahemik, max_delta)

    if len(bpoints) != 0 and isinstance(bpoints[0][1], np.ndarray):
        bpoints = [(bp[0][0], bp[0][1]/1000) for bp in bpoints]
    print(f"model contains {len(bpoints)} breaks")

    _fig, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(15, 5))
    ax1.scatter(X, y, marker="+", color='red', label="target")
    ax1.plot(X_pred, m.predict(X_pred)/1000,
             label="prediction",  linewidth=0.5)
    if len(bpoints) != 0:
        ax1.scatter(*zip(*bpoints), marker="x",
                    color='magenta', label="breaks")
        ax1.legend()
    ax2.scatter(X, y, marker="+", color='red', label="target")
    plt.show()
    return _fig, (ax1, ax2)

def joonista_variance(m: tf.keras.Sequential, X_test, X_train, y_train=None, bpoint_fn=Slopes.breakpoint_finder, *, ground_truth=False, xlim=None, ylim=None, return_fig=False, no_variance=False, n_variances=2, title_text=""):

    bpoints = bpoint_fn(m, X_test)

    _patterns = [bp[1] for bp in bpoints]
    bpoints = [bp[0] for bp in bpoints]

    start, end = xlim if xlim else (-1, 1)

    print(f"model contains {len(bpoints)} breaks")

    y_pred = m.predict(X_test, batch_size=32768)

    if no_variance:
        y_pred_mean = y_pred
    else:
        y_pred_mean, y_pred_logvar = y_pred[:, 0], y_pred[:, 1]
        y_pred_var = np.exp(y_pred_logvar)
        y_pred_sd = np.sqrt(y_pred_var)

    fig, ax = plt.subplots(figsize=(6, 6))
    plt.xlim(xlim)
    plt.ylim(ylim)

    # absoluutselt õige
    if ground_truth:
        plt.plot(np.linspace(start, end, 1000), ground_truth(np.linspace(
            start, end, 1000)), label="ground truth", alpha=0.5, color="orange")

    plt.plot(X_test, y_pred_mean, label="mean")
    if not no_variance:
        plt.fill_between(X_test, y_pred_mean - n_variances*y_pred_sd,
                         y_pred_mean + n_variances*y_pred_sd, alpha=0.2, label=f"{n_variances} standard hälvet")
    plt.scatter(X_train, y_train, marker='.', color="red", label="train")

    temp_bpoints = list(zip(*bpoints))
    bx, by = temp_bpoints[0], temp_bpoints[1]

    plt.scatter(bx, by, marker="o", color="green", label="breaks")
    plt.title(title_text)
    plt.legend()

    if return_fig:
        return fig

    plt.show()

def neg_log_likelihood(y_true, y_predicted):

    y_pred_mean, y_pred_var = y_predicted[:, 0:1], y_predicted[:, 1:]

    # y_pred_var ongi log variance
    # return tf.reduce_mean(tf.math.square(y_true - y_pred_var))
    # return tf.reduce_mean(y_pred_var + 1.7 / tf.math.exp(y_pred_var))
    return tf.reduce_mean(y_pred_var + tf.math.square(y_true - y_pred_mean) / tf.math.exp(y_pred_var))

def nll_gaussian(y_pred_mean, y_pred_sd, y_test):

    # element wise square
    # preserve the same shape as y_pred.shape
    square = tf.math.square(y_pred_mean - y_test)
    ms = tf.math.add(tf.math.divide(square, y_pred_sd), tf.math.log(y_pred_sd))
    # axis = -1 means that we take mean across the last dimension
    # the output keeps all but the last dimension
    ## ms = tf.reduce_mean(ms,axis=-1)
    # return scalar

    # nemad teevad square/sd + log(sd)

    # ehk nemad teevad square/var**0.5 + 0.5log(var)

    # meie teeme square/var + log(var)

    return tf.reduce_mean(ms)

def neg_log_likelihood2(ytrue, ypreds):
    n_dims = int(int(ypreds.shape[1])/2)
    mu = ypreds[:, 0:n_dims]
    logsigma = ypreds[:, n_dims:]

    mse = -0.5*tf.reduce_sum(tf.math.square((ytrue-mu) /
                             tf.math.exp(logsigma)), axis=1)
    sigma_trace = -tf.reduce_sum(logsigma, axis=1)
    log2pi = -0.5*n_dims*np.log(2*np.pi)

    log_likelihood = mse+sigma_trace+log2pi

    return tf.reduce_mean(-log_likelihood)

def train_data_maker(elements: list[tuple[float, float, float, float]], fn=lambda x: x**2, seed=0) -> tuple[np.array, np.array]:
    """Loob treeningandmed, võtab sisse listi, kus iga element on tuple:
        - mean
        - std
        - n_samples
        - noise_std

    :param elements: list of tuples, where each tuple is (mean, std, n_samples, noise_std)
    :param fn: f(x), funktsioon, defaults to lambdax:x**2
    :return: treeningandmed
    """
    reset_seeds(seed)

    X, y = [], []
    for el in elements:
        xi = np.random.normal(el[0], el[1], el[2])
        yi = fn(xi) + np.random.normal(0, el[3], size=el[2])
        X.extend(xi)
        y.extend(yi)
    X, y = np.array(X), np.array(y)
    return X, y

####################
#
# RMSE arvutamised
#
####################
def tegelik_myra(x, fn=lambda x: 0.09*x**2+0.09, reverse=False, end=10):
    """Tegelik myra, mis on funktsioonist arvutatud

    :param x: x
    :param fn: analüütiliselt võetud sd, defaults to lambda x: np.sqrt(0.09*x**2+0.09)
    :return: tegelik myra
    """
    # tegelik_myra_lambda = lambda x: np.sqrt(0.09*x**2+0.09)
    #x = 10-x
    if reverse:     # TODO: see töötab vaid siis kui uurimispiirkond on 0..10, vist töötab nüüd
        x = end-x
    return np.sqrt(fn(x))  # noqa: C3001

def rmse_from_model_variance_estimate(model, start, end, n_points=1000):
    """Joon nr2, mudeli poolt ennustatud variansi summast arvutatud RMSE,
    täpsemalt variancide keskmise ruutjuur

    :param model: _description_
    :param start: _description_
    :param end: _description_
    :param n_points: _description_, defaults to 1000
    :return: _description_
    """
    X = np.linspace(start, end, n_points)
    y_pred = model.predict(X, batch_size=32768, verbose=0)
    y_pred_variance = np.exp(y_pred[:, 1:])
    #assert abs(np.sum(y_pred_variance)/n_points - np.mean(y_pred_variance)) < 1e-5
    return np.sqrt(np.sum(y_pred_variance)/n_points)

def rmse_from_modelmean_2_samples(model, X_test, y_test):
    """Joon nr3, mudeli poolt ennustatud keskväärtuse ja punktide vaheline kaugus

    :param model: _description_
    :param X_test: _description_
    :param y_test: _description_
    :return: _description_
    """
    y_pred = model.predict(X_test, batch_size=32768, verbose=0)
    y_pred_mean = y_pred[:, 0]

    return np.sqrt(np.mean(
        (y_pred_mean-y_test)**2
    ))

def rmse_from_diff_means(model, start, end, fn, n_points=1000,):
    """Joon nr4, mudeli poolt ennustatud variansi summast arvutatud RMSE

    :param model: _description_
    :param start: _description_
    :param end: _description_
    :param n_points: _description_, defaults to 1000
    :return: _description_
    """
    X = np.linspace(start, end, n_points)
    y_pred = model.predict(X, batch_size=32768, verbose=0)
    y_pred_mean = y_pred[:, 0]
    y_true_mean = fn(X)
    return np.sqrt(np.mean(
        (y_true_mean-y_pred_mean)**2
    ))

def rmse_from_gtruth_2_samples(X_test, y_test, fn):
    """Joon nr5, testandmepunktide kaugus tõelisest funktsioonist

    :param X_test: testhulk
    :param y_test: testhulgale vastavate X-idele varem genetud y väärtused, et müra oleks kõikides katsetes sama
    :param fn: funktsioon mida ennustame
    :return: rmse v5
    """
    y_true_mean = fn(X_test)
    return np.sqrt(np.mean(
        (y_true_mean-y_test)**2
    ))

####################
#
# RMSE plotting
#
####################

def calculate_rmses(model, start=0, end=10, steps=1000, akna_laius=0.1,
                    fn=lambda x: x*np.sin(x), noise_fn=lambda x:0.3 * np.random.randn(len(x)) + 0.3 * x * np.random.randn(len(x)),
                    analyytiline_myra=lambda x: 0.09*x**2+0.09,
                    reverse_fn=False, reverse_noise=False):
    reset_seeds(100)
    X_test = np.linspace(start-2, end+2, 100_000) # TODO see on tõeline X_test, ei saa parameetriga ette anda.
    y_test = fn(X_test) + noise_fn(X_test)

    if reverse_fn and reverse_noise:
        y_test = fn(X_test[::-1]) + noise_fn(X_test[::-1])
    elif reverse_fn:
        y_test = fn(X_test[::-1]) + noise_fn(X_test)
    elif reverse_noise:
        y_test = fn(X_test) + noise_fn(X_test[::-1])
    else:
        y_test = fn(X_test) + noise_fn(X_test)

    y_pred = model.predict(X_test,batch_size=32768, verbose=0)
    y_pred_mean = y_pred[:, 0]
    y_pred_variance = np.exp(y_pred[:,1:])
    y_pred_sd = np.sqrt(y_pred_variance)

    y_true_mean = fn(X_test)

    uuringuruum, _ = np.linspace(start, end, steps, retstep=True)
    rmses2 = []
    for w_start in uuringuruum:

        w_end = w_start + akna_laius
        X_aknas = X_test[(X_test >= w_start) & (X_test < w_end)]
        y_aknas = y_test[(X_test >= w_start) & (X_test < w_end)]

        y_pred_aknas = y_pred[(X_test >= w_start) & (X_test < w_end)]
        y_pred_mean_aknas = y_pred_mean[(X_test >= w_start) & (X_test < w_end)]
        y_pred_variance_aknas = y_pred_variance[(X_test >= w_start) & (X_test < w_end)]
        y_pred_sd_aknas = y_pred_sd[(X_test >= w_start) & (X_test < w_end)]

        y_true_mean_aknas = y_true_mean[(X_test >= w_start) & (X_test < w_end)]

        rmse1 = tegelik_myra((w_start+w_end)/2, fn=analyytiline_myra, reverse=reverse_noise, end=end)
        rmse2 = np.sqrt(np.mean(y_pred_variance_aknas))
        rmse3 =  np.sqrt(np.mean(
            (y_pred_mean_aknas-y_aknas)**2
        ))
        rmse4 =  np.sqrt(np.mean(
            (y_true_mean_aknas-y_pred_mean_aknas)**2
        ))
        rmse5 =  np.sqrt(np.mean(
            (y_true_mean_aknas-y_aknas)**2
        ))
        rmses2.append([w_start, rmse1, rmse2, rmse3, rmse4, rmse5])
        #print(f"{w_start:.2f} {rmse1:.2f} {rmse2:.2f} {rmse3:.2f} {rmse4:.2f} {rmse5:.2f}")

    return np.array(rmses2).reshape(-1, 6)

def joonista_rmses5x(model, start=0, end=10, steps=1000, akna_laius=0.1, x_lim=None,y_lim=(0,30),
                     fn=lambda x: x*np.sin(x), noise_fn=lambda x:0.3 * np.random.randn(len(x)) + 0.3 * x * np.random.randn(len(x)),
                    analyytiline_myra=lambda x: 0.09*x**2+0.09,
                    reverse_fn=False, reverse_noise=False,
                    show_plt=True, title_text=""):

    rmses = calculate_rmses(model, start=start, end=end, steps=steps, akna_laius=akna_laius, fn=fn, noise_fn=noise_fn, analyytiline_myra=analyytiline_myra, reverse_fn=reverse_fn, reverse_noise=reverse_noise)
    sns.set_style('ticks')

    fig, ax = plt.subplots()

    c1, c2, c3, c4, c5 = sns.color_palette("tab10", 5)
    sns.lineplot(x=rmses[:, 0]+akna_laius/2, y=rmses[:, 1]**2, color=c1, label='müra valem (1)')
    sns.lineplot(x=rmses[:, 0]+akna_laius/2, y=rmses[:, 2]**2, color=c2, label='mudeli variance kaudu (2)') # mudel annab välja log-variance. Ehk tegelikult tuleks võtta hoopis selle ruutjuur - ei ole, see on juba sd sest me võtsime ruutjuure enne
    sns.lineplot(x=rmses[:, 0]+akna_laius/2, y=rmses[:, 3]**2, color=c3, label='mudeli keskväärtus vs samples (3)')
    sns.lineplot(x=rmses[:, 0]+akna_laius/2, y=rmses[:, 4]**2, color=c4, label='keskväärtuste erinevus (4)')
    sns.lineplot(x=rmses[:, 0]+akna_laius/2, y=rmses[:, 5]**2, color=c5, label='tõeline keskväärtus vs samples (5)') # esimesena plotitud et oleks kõige rohkem tagaplaanil.
    ax.legend()
    ax.set_ylabel('variance')

    ax.set_ylim(y_lim)

    plt.title(title_text+f", w={akna_laius}")

    if show_plt:
        plt.show()

    return rmses, fig

def joonista_hypotees_2gt1(rmses, akna_laius=0.1, a_min=0, a_max=15):
    fig, ax = plt.subplots()
    sns.lineplot(x=rmses[:, 0]+akna_laius/2, y=rmses[:, 1]**2, label='müra valemist (1)')
    sns.lineplot(x=rmses[:, 0]+akna_laius/2, y=rmses[:, 2]**2, label='mudeli variance\'id (2)')
    ax.set_ylabel('variance')
    plt.show()

def joonista_hypotees_143(rmses, akna_laius=0.1, a_min=0, a_max=15):
    _, ax = plt.subplots()
    sns.lineplot(x=rmses[:, 0]+akna_laius/2, y=np.clip(rmses[:, 1]**2+rmses[:, 4]**2, a_min=a_min, a_max=a_max) , label='1.+4.')
    sns.lineplot(x=rmses[:, 0]+akna_laius/2, y=np.clip(rmses[:, 3]**2, a_min=a_min, a_max=a_max), label='3.')
    ax.set_ylabel('variance')

def read_aggregated_data(path):
    df = pd.read_csv(path)

    df.train_size = df.train_size.astype(int)
    df.random_seed = df.random_seed.astype(int)
    df.multiplier = df.multiplier.astype(float)
    for i in range(5):
        df[f"ext_bpoints_in_{chr(i+ord('a'))}"] = df[f"ext_bpoints_in_{chr(i+ord('a'))}"].astype(int)

    # lisame mugavusväärtused
    df["points_in_equal_regions"] = (df["train_size"] / 5).astype(int)
    df["points_in_diff_region"] = (
        df["points_in_equal_regions"] * df["multiplier"]).astype(int)
    df["total_points"] = 4*df["points_in_equal_regions"] + \
        df["points_in_diff_region"]

    # tuunime välja väga kauged väärtused:
    cols_to_process = [*[f'abs_diff_in_{chr(a)}' for a in range(97, 97+5)],
                       *[f'rel_diff_in_{chr(a)}' for a in range(97, 97+5)],
                       *[f'raw_mean2_in_{chr(a)}' for a in range(97, 97+5)],
                       *[f'raw_mean3_in_{chr(a)}' for a in range(97, 97+5)],
                       *['mse_treeningul', 'mse_grid_testil',
                           'mse_treening_andmete_teine_myra'],
                       ]

    df[cols_to_process] = df.loc[:, cols_to_process][np.abs(
        stats.zscore(df.loc[:, cols_to_process])) < 3]
    df.dropna(inplace=True)

    df.sort_values(by=["train_size", "region", "multiplier",
                   "random_seed"], inplace=True, ignore_index=True)
    df = df.groupby(['region', 'multiplier', 'train_size']
                    ).mean().reset_index()

    df["combo_name"] = df["train_size"].astype(str) + "_" + df["region"] + "_" + df["multiplier"].astype(str)
    return df

def transform_no_scaling(df, minu: str, naabrid: list[str], kaugemad: list[str]):
    ma_olen_reg = minu
    # naabrid = ['a','c']
    naabrid_upper = [n.upper() for n in naabrid]
    # kaugemad = ['d','e']
    kaugemad_upper = [k.upper() for k in kaugemad]

    # y_col = "rel_diff_in_"+ma_olen_reg

    df_new = pd.DataFrame()
    df_new["my_bpoints"] = df["bpoints_in_"+ma_olen_reg]
    if len(naabrid) == 2:
        df_new["neighbour_bpoints"] = (
            df["bpoints_in_"+naabrid[0]] + df["bpoints_in_"+naabrid[1]])/2
    else:
        df_new["neighbour_bpoints"] = df["bpoints_in_"+naabrid[0]]

    if len(kaugemad) == 2:
        df_new["distant_bpoints"] = (
            df["bpoints_in_"+kaugemad[0]] + df["bpoints_in_"+kaugemad[1]])/2
    else:
        df_new["distant_bpoints"] = df["bpoints_in_"+kaugemad[0]]

    df_new["my_rel_diff"] = df["rel_diff_in_"+ma_olen_reg]

    df_new["my_raw_mean2"] = df["raw_mean2_in_"+ma_olen_reg]
    if len(naabrid) == 2:
        df_new["neighbour_raw_mean2"] = (
            df["raw_mean2_in_"+naabrid[0]] + df["raw_mean2_in_"+naabrid[1]])/2
    else:
        df_new["neighbour_raw_mean2"] = df["raw_mean2_in_"+naabrid[0]]

    if len(kaugemad) == 2:
        df_new["distant_raw_mean2"] = (
            df["raw_mean2_in_"+kaugemad[0]] + df["raw_mean2_in_"+kaugemad[1]])/2
    else:
        df_new["distant_raw_mean2"] = df["raw_mean2_in_"+kaugemad[0]]

    # minus olevad punktid
    df_new["my_points"] = (df["train_size"]/5 * df["multiplier"] * (df["region"] ==
                           ma_olen_reg.upper())) + (df["train_size"]/5 * (df["region"] != ma_olen_reg.upper()))
    df_new["my_points"] = df_new["my_points"].astype(int)

    if len(naabrid) == 2:
        df_new["neighbour_points"] = (df["train_size"]/5 * df["multiplier"] * (
            df["region"] == naabrid_upper[0])) + (df["train_size"]/5 * (df["region"] != naabrid_upper[0]))
        df_new["neighbour_points"] += (df["train_size"]/5 * df["multiplier"] * (
            df["region"] == naabrid_upper[1])) + (df["train_size"]/5 * (df["region"] != naabrid_upper[1]))
        df_new["neighbour_points"] = (df_new["neighbour_points"]/2).astype(int)
    else:
        df_new["neighbour_points"] = (df["train_size"]/5 * df["multiplier"] * (
            df["region"] == naabrid_upper[0])) + (df["train_size"]/5 * (df["region"] != naabrid_upper[0]))
        df_new["neighbour_points"] = df_new["neighbour_points"].astype(int)

    if len(kaugemad) == 2:
        df_new["distant_points"] = (df["train_size"]/5 * df["multiplier"] * (
            df["region"] == kaugemad_upper[0])) + (df["train_size"]/5 * (df["region"] != kaugemad_upper[0]))
        df_new["distant_points"] += (df["train_size"]/5 * df["multiplier"] * (
            df["region"] == kaugemad_upper[1])) + (df["train_size"]/5 * (df["region"] != kaugemad_upper[1]))
        df_new["distant_points"] = (df_new["distant_points"]/2).astype(int)
    else:
        df_new["distant_points"] = (df["train_size"]/5 * df["multiplier"] * (
            df["region"] == kaugemad_upper[0])) + (df["train_size"]/5 * (df["region"] != kaugemad_upper[0]))
        df_new["distant_points"] = df_new["distant_points"].astype(int)

    # et ei oleks ühtegi nulli enam
    df_new["my_points"] = (df_new["my_points"]+1).astype(float)
    df_new["neighbour_points"] = (df_new["neighbour_points"]+1).astype(float)
    df_new["distant_points"] = (df_new["distant_points"]+1).astype(float)

    df_new["my_points_m1"] = df_new["my_points"] ** (-1)
    df_new["my_points_log"] = np.log(df_new["my_points"])
    df_new["my_points_logm1"] = (np.log(df_new["my_points"])+1.0) ** (-1)

    df_new["neighbour_points_m1"] = df_new["neighbour_points"] ** (-1)
    df_new["neighbour_points_log"] = np.log(df_new["neighbour_points"])
    df_new["neighbour_points_logm1"] = (
        np.log(df_new["neighbour_points"]) + 1.0) ** (-1)

    df_new["distant_points_m1"] = df_new["distant_points"] ** (-1)
    df_new["distant_points_log"] = np.log(df_new["distant_points"])
    df_new["distant_points_logm1"] = (
        np.log(df_new["distant_points"]) + 1.0) ** (-1)

    df_new["mse_treeningul"] = df["mse_treeningul"]
    df_new.dropna(inplace=True)
    df_new.reset_index(inplace=True, drop=True)

    df_new.drop_duplicates(inplace=True, ignore_index=True)

    X = df_new.drop(columns=['my_rel_diff'])
    y = df_new['my_rel_diff']

    return X, y

def create_x_train(train_size, different_place, different_multiplier=1, x_range=(0, 10), n_places=5, seed=None):
    # utils.reset_seeds(seed)

    keskmine_num = train_size/n_places  # A
    erinev_num = different_multiplier*keskmine_num  # B
    piirkond = (x_range[1]-x_range[0])/n_places
    piirkonnad = [(x_range[0]+i*piirkond, x_range[0]+(i+1)*piirkond)
                  for i in range(n_places)]
    #print(f"piirkond: {piirkond}")
    #print(f"piirkonnad: {piirkonnad}")

    X_train = []
    for idx, p in enumerate(piirkonnad):
        if idx == different_place:
            X_train.append(np.random.uniform(p[0], p[1], int(erinev_num)))
        else:
            X_train.append(np.random.uniform(p[0], p[1], int(keskmine_num)))

    X_train = np.concatenate(X_train)
    return X_train


import numpy
import matplotlib.pyplot as plt
from pandas import read_csv
import math
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_squared_error
from keras.callbacks import CSVLogger

def listSum(list):
    for i in range(len(list)):
       list[len(list)-1-i] = sum(list,len(list)-1-i)
    return list

def sum(list, row):
    if row == len(list)-1:
        return list[row]
    else:
        return list[row] + list[row + 1]

Y = [] # output data list

# fix random seed for reproducibility
numpy.random.seed(7)

# load the dataset
dataframe = read_csv('panel_sum_cost.csv')
dataframe = dataframe.fillna(method='ffill') # filling any null or Nan in dataset
dataframe = dataframe[dataframe.Cost != 0] # removing rows that contain 0 cost

#formatting output data
allPatRec = dataframe.values
st_index = 0
end_index = 0
total = 0
calcTrainSize = int(len(allPatRec)*(80/100))
print(len(allPatRec), calcTrainSize)

splitPoint = False
alreadyChecked = False
for i in range(len(allPatRec)-1):
    str1 = allPatRec[i][0].split(" ") #splitting first column string to access claim number only
    str2 = allPatRec[i+1][0].split(" ")
    onePatRec = []

    if str1[1] == str2[1]:
        end_index = end_index+1
        if end_index == calcTrainSize:
            splitPoint = True

    else:
        if splitPoint == True and alreadyChecked == False:
            #if str2[3] = "1":
            calcTrainSize = end_index+1 #first row of next claim (patient) number
            alreadyChecked = True
        for k in range(st_index,end_index+1):
            onePatRec.append(allPatRec[k][20])
        costUntilrecoveryList = listSum(onePatRec)
        Y = Y + costUntilrecoveryList
        end_index = end_index+1 # to move end_index and st_index to the start of next patient
        st_index = end_index

#print("st_index, end_index", st_index , end_index )
# below code is to cater last patiend record since above code does not run for last patiend
# because if statement just updates indices but since there is not next record,
# else will not execute for last patient

lastPatRec = []
for k in range(st_index,end_index+1):
    lastPatRec.append(allPatRec[k][20])
costUntilrecoveryList = listSum(lastPatRec)
Y = Y + costUntilrecoveryList
Y = numpy.array(Y)
Y = Y.reshape(len(Y),1)
Y = Y.astype('float32') #converting those values to float

##print('split index', calcTrainSize)
##print(allPatRec[calcTrainSize])

del dataframe['Patient'] #removing first column
del dataframe['Cost'] # removing target column i.e., Cost
datasetX = dataframe.values # now it has only input features
datasetX = datasetX.astype('float32') #converting those values to float
##print(len(datasetX), datasetX[:1,:]) # printing first row of 2-d array
##X = datasetX[:,0:3] # accessing 2-d array. first 3 elements of each row
##print(X)
## normalize the dataset
scaler = MinMaxScaler(feature_range=(0, 1)) # scalling them between 0-1
datasetX = scaler.fit_transform(datasetX)
##print(datasetX[:1,:]) # printing first row of 2-d array
##print(len(Y), Y)
scalerY = MinMaxScaler(feature_range=(0, 1))
Y = scalerY.fit_transform(Y)
##print(Y[0]) # printing first row of 2-d array

## split into train and test sets
##train_size = int(len(datasetX) * 0.80) # splitting dataset list of lists into two lists of lists
##test_size = len(datasetX) - train_size
trainX, testX = datasetX[0:calcTrainSize,:], datasetX[calcTrainSize:len(datasetX),:]
trainY, testY = Y[0:calcTrainSize,:], Y[calcTrainSize:len(datasetX),:]

trainX = numpy.array(trainX)
trainY = numpy.array(trainY)
testX = numpy.array(testX)
testY = numpy.array(testY)
##print(trainY[len(trainY)-1], testY[0])

## reshape input to be [samples, time steps, features]
trainX = numpy.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1])) #now its 3 dimetional list instead of 2 dimentional but still having one value in each list of list of list
testX = numpy.reshape(testX, (testX.shape[0], 1, testX.shape[1])) #gives a new shape to an array without changing its data. shape returns the dimensions of array. if array has n rows and m columns, then array.shape is (n,m) and shape[0] returns n and shape[1] returns m

## create and fit the LSTM network
model = Sequential() # network has a visible layer with 1 input
model.add(LSTM(19, activation='tanh', input_shape=(1, 19), use_bias=True))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
csv_logger = CSVLogger('log.csv', append=True, separator=';')
model.fit(trainX, trainY, epochs=100, batch_size=1, verbose=2)

## make predictions
trainPredict = model.predict(trainX)
testPredict = model.predict(testX)
##print(len(trainPredict), trainPredict)
##print(len(trainY), trainY)

## invert predictions
trainPredict = scalerY.inverse_transform(trainPredict)
trainY = scalerY.inverse_transform(trainY)
testPredict = scalerY.inverse_transform(testPredict)
testY = scalerY.inverse_transform(testY)
##print(trainY[0])
##print(len(trainPredict), trainPredict[:,0])
##print(len(trainY), trainY[0])

## calculate root mean squared error # make sure dimensions of parameter arrays are same
trainScore = math.sqrt(mean_squared_error(trainY, trainPredict[:,0]))
print('Train Score: %.2f RMSE' % (trainScore))
testScore = math.sqrt(mean_squared_error(testY, testPredict[:,0]))
print('Test Score: %.2f RMSE' % (testScore))


"""
Main Script

This script is used for model training, testing, and prediction.

Usage:
    - Set the desired values for the variables.
    - Call the corresponding functions to perform the desired actions.
"""

import torch
from data_loader import data_prep
from model_training_es import load_pretrained_model, multi_class_classifier
from model_testing import load_trained_model, test_new_image, get_confusion_matrix

# ############# CHOOSE DEVICE ################
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"Using {device} device")
class_names = ["comfort", "danger", "death", "excitement", "fitness", "freedom", "power", "safety"]

def get_experiment_name (variables):
    """
    Get the experiment name based on the given variables.

    Args:
        variables (list): A list of variables containing data_prep_type, base_model_name, epochs, model_type,
                          and augmented_data.

    Returns:
        str: The experiment name.
    """
    data_prep_type, base_model_name, epochs, model_type, augmented_data = variables
    # specify experiment name
    augmented_variable = "augmented" if augmented_data else "not_augmented"
    experiment_name = str(
        data_prep_type + "_" + base_model_name + "_" + str(epochs) + "_epochs_" + model_type + "_" + augmented_variable)
    return experiment_name

def train_model(variables):
    """
    Train the model based on the given variables.

    Args:
        variables (list): A list of variables containing data_prep_type, base_model_name, epochs, model_type,
                          and augmented_data.

    Returns:
        tuple: A tuple containing the trained model and the path to the trained model file.
    """
    data_prep_type, base_model_name, epochs, model_type, augmented_data = variables
    # specify experiment name
    experiment_name = get_experiment_name(variables)
    # Load data with desired variables
    train_loader, val_loader, test_loader, train_dataset, val_dataset, test_dataset = data_prep(dataset_path, data_prep_type, augmented_data=augmented_data, )
    # Load desired base model to the GPU
    base_model = load_pretrained_model(base_model_name)
    base_model.to(device)
    # train the model and load it on the GPU
    trained_model, trained_model_PATH = multi_class_classifier(data_prep_type, train_loader, val_loader,test_loader, base_model_name, base_model, experiment_name, epochs, augmented_data)
    trained_model.cuda()
    return trained_model, trained_model_PATH

def test_model(variables, class_names):
    """
    Test the model based on the given variables.

    Args:
        variables (list): A list of variables containing data_prep_type, base_model_name, epochs, model_type,
                          and augmented_data.
        class_names (list): A list of class names.

    Returns:
        None
    """
    data_prep_type, base_model_name, epochs, model_type, augmented_data = variables
    # specify experiment name
    experiment_name = get_experiment_name(variables)
    # Load data with desired balancing
    train_loader, val_loader, test_loader, train_dataset, val_dataset, test_dataset = data_prep(dataset_path, data_prep_type, augmented_data)
    trained_model = load_trained_model(experiment_name, base_model_name)
    trained_model.to(device)
    get_confusion_matrix(experiment_name, trained_model, test_loader, class_names)
    return

def model_prediction(variables, image_path, base_model_name):
    """
    Perform model prediction on a new image based on the given variables.

    Args:
        variables (list): A list of variables containing data_prep_type, base_model_name, epochs, model_type,
                          and augmented_data.
        image_path (str): The path to the new image.
        base_model_name (str): The name of the base model.

    Returns:
        None
    """
    # specify experiment name
    experiment_name = get_experiment_name(variables)
    # load trained model
    trained_model = load_trained_model(experiment_name, base_model_name)
    # test new image
    test_new_image(experiment_name, trained_model, image_path)
    return

dataset_path = "../Local_structured_dataset"
variable = ["overall_b", "resnet50", 100, "pretrained_multi_class", True]
train_model(variable)
test_model(variable, class_names)
image_path = "goya.jpg"
model_prediction(variable, image_path, variable[1])

## To execute with many different combinations of models, data preps, epochs, etc:
# variable_patterns = [
## data_prep_type, base_model_name, epochs, model_arch, augmented_data
# ["ub", "vgg16", 20, "pretrained_multi_class", True],
# ["ub", "vgg16", 20, "pretrained_multi_class", False],
# ["ub", "resnet50", 20, "pretrained_multi_class", True],
# ["ub", "resnet50", 20, "pretrained_multi_class", False],
# ["overall_b", "vgg16", 100, "pretrained_multi_class", True],
# ["overall_b", "vgg16", 20, "pretrained_multi_class", False],
# ["overall_b", "resnet50", 20, "pretrained_multi_class", True],
# ["overall_b", "resnet50", 20, "pretrained_multi_class", False],
# ["partition_b", "vgg16", 20, "pretrained_multi_class", True],
# ["partition_b", "vgg16", 20, "pretrained_multi_class", False],
# ["partition_b", "resnet50", 20, "pretrained_multi_class", True],
# ["partition_b", "resnet50", 20, "pretrained_multi_class", False],
# ["ub", "vit", 20, "pretrained_multi_class", True],
# ["ub", "vit", 20, "pretrained_multi_class", False],
# ["ub", "mlp_mixer", 20, "pretrained_multi_class", True],
# ["ub", "mlp_mixer", 20, "pretrained_multi_class", False],
# ["overall_b", "vit", 20, "pretrained_multi_class", True],
# ["overall_b", "vit", 20, "pretrained_multi_class", True],
# # ["overall_b", "mlp_mixer", 20, "pretrained_multi_class", False],
# ["partition_b", "vit", 20, "pretrained_multi_class", True],
# ["partition_b", "vit", 20, "pretrained_multi_class", False],
# # ["partition_b", "mlp_mixer", 20, "pretrained_multi_class", True],
# # ["partition_b", "mlp_mixer", 20, "pretrained_multi_class", False],
# ["overall_b", "vit", 60, "pretrained_multi_class", True]
# ]

# for variables in variable_patterns:
#     train_model(variables)
#     test_model(variables, class_names)
#     model_prediction(variables, image_path)


# coding: utf-8
from firebase import firebase
import json
import time
import serial
import threading
import os
import base64
import RPi.GPIO as GPIO
i=0
GPIO.setmode(GPIO.BCM)
GPIO.setwarnings(False)
TRIG = 23
ECHO = 24
lf = 19
rf = 16
lb = 26
rb = 20
GPIO.setup(TRIG,GPIO.OUT)
GPIO.setup(ECHO,GPIO.IN)
GPIO.output(TRIG, False)
GPIO.setup(lf,GPIO.OUT)
GPIO.setup(rf,GPIO.OUT)
GPIO.setup(lb,GPIO.OUT)
GPIO.setup(rb,GPIO.OUT)
ser = serial.Serial('/dev/ttyACM6', 9600, timeout=1)
ser.isOpen()

def Straight():
    GPIO.output(rf,False)
    GPIO.output(lf,False)
    GPIO.output(rb,True)
    GPIO.output(lb,True)

def Back():
    GPIO.output(rb,True)
    GPIO.output(lb,True)
    GPIO.output(rf,False)
    GPIO.output(lf,False)
def RightF():
    GPIO.output(rb,False)
    GPIO.output(lb,True)
    GPIO.output(rf,False)
    GPIO.output(lf,False)

def LeftF():
    GPIO.output(rb,True)
    GPIO.output(lb,False)
    GPIO.output(rf,False)
    GPIO.output(lf,False)

def RightB():
    GPIO.output(rb,False)
    GPIO.output(lb,True)
    GPIO.output(rf,False)
    GPIO.output(lf,False)

def LeftB():
    GPIO.output(rb,True)
    GPIO.output(lb,False)
    GPIO.output(rf,False)
    GPIO.output(lf,False)

def Stop():
    GPIO.output(rb,False)
    GPIO.output(lb,False)
    GPIO.output(rf,False)
    GPIO.output(lf,False)

options={
   "1":Straight,
   "2":RightF,
   "3":RightB,
   "4":Back,
   "5":LeftB,
   "6":LeftF,
   "7":Stop
}
firebase_url='https://botnet3-f4eca.firebaseio.com/'
firebase = firebase.FirebaseApplication(firebase_url)

class myThread1(threading.Thread):
    def __init__(self):
        threading.Thread.__init__(self)

    def run(self):
        while 1:
           result=firebase.get('status/Move',None)
           options[result]()
           print (result)
           if firebase.get('status/Ultrasonic_Status',None):
              print ("on")
              action=Ultra()
              if action=="Get Away":
                 if result!=7 or result!=5 or result!=3 or result!=4:
                    Stop()
           dir= firebase.get('armcam/now',None)
           print (dir)
           ser.write(dir)

class myThread2(threading.Thread):
   def __init__(self):
       threading.Thread.__init__(self)
       time.sleep(0.4)
   def run(self):
       while 1:
          if firebase.get('status/Image',None):
              global i
              i=i+1
              cur_image="image";
              cur_image=cur_image+str(i)+".jpg"
              os.system("fswebcam "+"-r 50*50 "+cur_image)
              time.sleep(0.4)
              try:
                 encoded = base64.b64encode(open(cur_image, "rb").read())
                 time.sleep(0.3)
              except IOError:
                 print("file dose not exist yet")
              os.system("sudo rm "+cur_image)
              firebase.put('image','now',encoded)
          else:
              time.sleep(2)

def Ultra():
    GPIO.output(TRIG, True)
    time.sleep(0.00001)
    GPIO.output(TRIG, False)
    while GPIO.input(ECHO)==0:
       pulse_start = time.time()
    while GPIO.input(ECHO)==1:
       pulse_end = time.time()
    pulse_duration = pulse_end - pulse_start
    distance = pulse_duration * 17150
    distance = round(distance, 2)
    if distance>=1 and distance<=60:
       action="Get Away"
    else:
       action="Everything Good"
    return action

thread1=myThread1()
thread2=myThread1()
thread3=myThread1()
thread4=myThread1()
thread5=myThread1()
thread6=myThread1()
thread7=myThread1()
thread8=myThread1()
thread9=myThread1()
thread10=myThread1()
thread11=myThread1()
thread12=myThread1()
thread13=myThread1()

thread14=myThread2()
thread15=myThread2()
thread15=myThread2()
thread16=myThread2()
thread17=myThread2()

thread1.start()
thread2.start()
thread3.start()
thread4.start()
thread5.start()
thread6.start()
thread7.start()
thread8.start()
thread9.start()
thread10.start()
thread11.start()
thread12.start()
thread13.start()
thread14.start()
thread15.start()
thread16.start()
thread17.start()

while 1:
   pass


import argparse
import configparser
import logging
import urllib3

from pprint import pformat
from time import time, sleep

from pymisp import PyMISP

from lib import helper, huntlogic, misphandler

class MISPHunter():
    def __init__(self, ssl_verify=True, logger=None):

        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

        #### Debugging-specific vars
        # Overall Debugging Var - used for activating certain overly-verbose logs
        self.debugging = True
        # Set to False to use dummy Censys data
        self.do_it_live = True
        # Force extraction of IOCs, regardless of JSON blob diff/age
        self.force_ioc_extract = False
        # Ignore timers set for seed objects - don't read them,
        # don't write them, just run API calls with reckless abandon
        self.ignore_timers = False

        #### Everyday vars
        # Enable verbose logging
        self.verbose_logging = False
        # Scrub dates from JSON blobs (time/process intensive) - probably mark
        #   this for future deprecation
        self.remove_dates = False
        # Use certificates found in JSON blobs to create pivot objects and
        # potentially find new hosts. Probably should force this regardless.
        self.cert_pivoting = True
        # maxmium number of hosts a certificate can return before being truncated
        self.cert_pivot_threshold = 20
        # Determines how old an object can be before using API calls to update
        # it. Defaults to 24.
        self.update_threshold = 24
        # hours before an attribute is no longer considered newly-discovered and
        # the new-discovery tags are removed.
        self.new_discovery_threshold = 72

        self.logger = helper.get_logger(verbose=self.verbose_logging)

        # Read configs
        try:
            config = configparser.ConfigParser()
            config.read('./misphunter.conf')
            self.misp_url = config['MISP']['misp_url']
            self.misp_key = config['MISP']['misp_api']

            self.censys_v1_active = config['censys-v1']['active']
            self.censys_v1_id = config['censys-v1']['id']
            self.censys_v1_secret = config['censys-v1']['secret']
            self.censys_v1_rate = int(config['censys-v1']['rate_limit'])
            self.censys_v1_slug = config['censys-v1']['slug']
            self.censys_v1_api_counter = 0

            self.censys_v2_active = config['censys-v2']['active']
            self.censys_v2_id = config['censys-v2']['id']
            self.censys_v2_secret = config['censys-v2']['secret']
            self.censys_v2_rate = int(config['censys-v2']['rate_limit'])
            self.censys_v2_slug = config['censys-v2']['slug']
            self.censys_v2_api_counter = 0

            self.shodan_active = config['shodan']['active']
            self.shodan_key = config['shodan']['api']
            self.shodan_rate = int(config['shodan']['rate_limit'])
            self.shodan_slug = config['shodan']['slug']
            self.shodan_api_counter = 0

            self.ignored_json_keys = {}

            self.logger.info("\n\nLoaded configs okay...")

        except Exception as ex:
            self.logger.error(
                f"Exception: {ex} - occurred loading configs", exc_info=True)
            raise

        try:
            self.misp = PyMISP(self.misp_url, self.misp_key, ssl=ssl_verify)
            self.logger.info(f"MISP instance loaded successfully!")
        except Exception as ex:
            self.logger.error(
                f"Exception: {ex}  - occurred while loading MISP", exc_info=True)
            if ssl_verify:
                self.logger.warning(f"SSL verify is set to True. If you're using "
                f"a self-signed certificate, try running again with -k.")
            raise

        self.obj_index_mapping = {
            "misphunter-seed": "search-string",
            "misphunter-host": "host-ip",
            "misphunter-cert": "cert-sha256",
            "misphunter-dns": "domain",
            "misphunter-malware": "sha256"
        }

        self.rel_type_mapping = {
            "cert-ip": "ip-dst",
            "cert-sha256": "x509-fingerprint-sha256",
            "domain": "domain",
            "extracted-certificate": "x509-fingerprint-sha256",
            "extracted-domain": "domain",
            "found-host": "ip-dst",
            "host-ip": "ip-dst",
            "search-string": "text",
            "sha256": "sha256"
        }

        self.obj_pivot_mapping = {
            "misphunter-seed": {
                "found-host" : "misphunter-host"
            },
            "misphunter-host": {
                "extracted-certificate": "misphunter-cert",
                "extracted-domain": "misphunter-domain"
            },
            "misphunter-cert": {
                "cert-ip": "misphunter-host"
            }
        }

        # Keep rolling talley of scanner_ips to attempt to keep them blacklisted
        self.scanner_ips = []

    def run(self):
        self.logger.info(f"\n###############################################################################\n\n# "\
                "GIDDY-UP, HUNTER!\n"\
                ",______________________________________\n"
                "|_________________,----------._ [____]  \"\"-,__  __....-----=====\n"
                "               (_(||||||||||||)___________/   \"\"                |\n"
                "                  `----------'        [ ))\"-,                   |\n"
                "                                       \"\"    `,  _,--....___    |\n"
                "                                               `/           \"\"\"\"\n"
                "\n###############################################################################\n\n")
        # Reset stats for each run
        self.censys_v1_api_counter = 0
        self.censys_v2_api_counter = 0
        self.shodan_api_counter = 0

        self.run_stats = {
            "auto_blacklists_added" : {'total': 0},
            "certs_added": {'total': 0},
            "clones_added": {'total': 0},
            "hosts_added": {'total': 0},
            "tags_added": {'total': 0},
            "tags_removed": {'total': 0}
        }
        # Track start time for overall time it takes the script to run
        self.start_time = time()
        self.logger.debug(f"Running MISPHunter... Starting timer  at {self.start_time}!")
        # Set initial search time for API request limits
        self.search_time = time()

        # Search MISP for globally blocked IPs & create global block list
        self.global_blocks = misphandler.get_global_blocks(self)

        # Get groups of different seed types. Seeds are objets that start an event.
        # Right now these are censys-v2 and shodan...
        self.host_seed_services = []
        if self.censys_v1_active:
            self.host_seed_services.append(self.censys_v1_slug)
        if self.censys_v2_active:
            self.host_seed_services.append(self.censys_v2_slug)
        if self.shodan_active:
            self.host_seed_services.append(self.shodan_slug)

        # ...In the future, these can be seeds that return different types of results
        # (like a list of domains instead of IPs)
        # from things like RiskIQ, Domaintools, and even VirusTotal

        # TODO
        # self.dns_seed_services = []
        # if self.domaintools_active:
        #     self.dns_seed_services.append(self.domaintools_slug)

        if self.do_it_live:
            self.logger.info(f"THIS IS NOT A DRILL! - Using actual API queries instead of dummy data.")

        # Get all enabled MISP seed objects and their parent event IDs
        all_event_seeds = misphandler.get_all_event_seeds(self)

        # Loop through events one at a time
        for event_id, seeds in all_event_seeds.items():
            ### DEBUGGING - only deal with one specific event
            # if mh.debugging:
            #     if str(event_id) != "3978":
            #         continue

            event = misphandler.get_event(self, event_id)
            self.logger.info(f"\n###############################################################################\n\n# "
                f"PROCESSING EVENT ID {event_id}\n# "
                f"{event.info}"
                f"\n###############################################################################\n\n")

            mh.event_hosts = misphandler.get_event_objects(self, event, 'misphunter-host')
            # Above sets global lists of MISPObjects like mh.event_hosts = [<misphunter-host>, <misphunter-host>]
            mh.event_seeds = misphandler.get_event_objects(self, event, 'misphunter-seed')
            # mh.event_seeds = [<misphunter-seed>]
            mh.event_certs = misphandler.get_event_objects(self, event, 'misphunter-cert')
            # mh.event_certs = [<misphunter-cert>]
            # TODO - still haven't built these objects yet
            #   Issues #15 and #16
            mh.event_dns = misphandler.get_event_objects(self, event, 'misphunter-dns')
            # mh.event_dns = [<misphunter-dns>]
            mh.event_malware = misphandler.get_event_objects(self, event, 'misphunter-malware')
            # mh.event_malware = [<misphunter-malware>]

            # Reset updated objects and found pivots for each event processed
            mh.event_staged_objects = []
            mh.event_processed_object_uuids = []

            ### Set the first stage with seed objects
            event = huntlogic.set_the_stage(self, seeds, event)

            if mh.debugging:
                helper.log_stage_details(mh)

            # continue processing event_staged_objects until they're exhausted.
            # as each event is processed from the stage, add its UUID to
            #   event_processed_object_uuids and remove it from event_staged_objects
            while len(mh.event_staged_objects) > 0:
                mh.logger.debug(f"{len(mh.event_staged_objects)} objects "
                    f"remaining on stage.")
                for obj in mh.event_staged_objects:
                    event = huntlogic.process_stage_object(self, obj, event)

            # TODO - Issue #3
            # NOTE: This is probably mostly-addressed now by extract_pivots()
            # Second, process all enabled misphunter objects for this event that were not touched after process_seeds()
            #   Might want to handle this differently than I originally thought.
            #   If a host no longer shows up in a seed search, how can we tell if it's still related?
            #   Should it then be disabled?

            # Run relationships/relationship checks against all objects in the event.
            event = huntlogic.process_relationships(self, event)

            # Tag/Untag event and individual attributes as "new" if newly added within update_threshold
            event = huntlogic.process_new_tags(self, event)

            # Finally, set blacklisted=1 for objects that have a crappy return (e.g. certs with only 1 assoc. host)
            #   or dns/domain objects that are clearly shared hosts
            event = huntlogic.auto_blacklist(self, event)

            mh.logger.info(f"Finished processing event {event_id}!")

        total_api = self.censys_v2_api_counter + self.shodan_api_counter
        self.logger.info(f"\n\nTOTAL CENSYS API CALLS: {self.censys_v2_api_counter}\n"
            f"TOTAL SHODAN API CALLS: {self.shodan_api_counter}\n"
            f"TOTAL OVERALL EXTERNAL API CALLS: {total_api}\n\n")
        self.logger.info(f"Other stats:")
        for k, v in self.run_stats.items():
            self.logger.info(f"{k}:")
            self.logger.info(f"    {pformat(v)}")
        # self.logger.info({pformat(self.run_stats)})
        self.logger.info("DONE!")

def run_as_service(mh):
    mh.logger.info(f"Launching MISPHunter as a service...")
    while True:
        chk_file = "./misphunter_chkpt.txt"
        try:
            checkpoint = open(chk_file)
            chk = checkpoint.readline()
            checkpoint.close()
        except:
            chk = 0

        end_sec = time()
        end_sec_int = int(end_sec)
        one_hour = 60 * 60
        one_day = one_hour * 24
        two_day = one_day * 2
        seconds_passed = end_sec_int-int(chk)
        if seconds_passed >= one_hour:
            mh.run()
            mh.logger.info(f"Saving new checkpoint of {end_sec}.")
            f = open(chk_file,'w')
            new_checkpoint = str(int(end_sec)+1)
            f.write(new_checkpoint)
            f.close()
            sleep(one_hour+1)
        else:
            time_remaining = (one_hour+1) - seconds_passed
            mh.logger.info(f"Only {seconds_passed} seconds have passed since the last time we ran MISPHunter. Sleeping for remaining {time_remaining} seconds.")
            sleep(time_remaining)

if __name__ == "__main__":

    # Process CLI arguments
    msg = "MISPHunter"
    parser = argparse.ArgumentParser(description = msg)

    parser.add_argument("--cert-pivot-threshold", type=int, default=10,
        help="Maxmium number of hosts a certificate pivot can return before being truncated. Defaults to 10.")
    parser.add_argument("--new-discovery-threshold", type=int, default=72,
        help="Maximum number of hours before an attribute is no longer considered newly-discovered and the new-discovery "\
            "tags get removed. Defaults to 72.")
    parser.add_argument("--update-threshold", type=int, default=24,
        help="Determines how old an object can be before using API calls to update it. Defaults to 24.")
    parser.add_argument("-k", "--ssl-no-verify", action="store_false",
        help="Disables verification of SSL certificate of the MISP instance. If you are using a "\
            "self-signed certificate, make sure this option is set.")
    parser.add_argument("-s", "--run-as-service", action="store_true",
        help="Run MISPHunter as a service. This puts it into an infinite loop where it runs once an hour.")
    parser.add_argument("--debugging", action="store_true",
        help="[DEBUG] Set debugging flag to true. Blows up your logs.")
    parser.add_argument("--offline", action="store_false",
        help="[DEBUG] Forces 'offline' mode, causing no 3rd party API queries to be used.")
    parser.add_argument("--ignore-timers", action="store_true",
        help="[DEBUG] Ignore the age of objects and process everything with live API requests.")
    parser.add_argument("--force-ioc-extract", action="store_true",
        help="[DEBUG] Force IOC extraction on all discovered objects, regardless of their age.")
    parser.add_argument("-v", "--verbose", action="store_true",
        help="Enable DEBUG-level logging.")

    # Initialize parser

    args = parser.parse_args()

    # SSL Verify
    ssl_verify = args.ssl_no_verify

    mh = MISPHunter(ssl_verify=ssl_verify)

    # DEBUGGING Vars
    mh.debugging = args.debugging
    mh.do_it_live = args.offline
    mh.force_ioc_extract = args.force_ioc_extract
    mh.ignore_timers = args.ignore_timers

    # EVERYDAY Vars
    mh.verbose_logging = args.verbose
    # Determines how old an object can be before using API calls to update
    # it. Defaults to 24.
    mh.update_threshold = args.update_threshold
    # Use certificates found in JSON blobs to create pivot objects and
    # potentially find new hosts. Probably should force this regardless.
    mh.cert_pivoting = True
    # maxmium number of hosts a certificate pivot can return before
    # being truncated
    mh.cert_pivot_threshold = args.cert_pivot_threshold
    # hours before an attribute is no longer considered newly-discovered and
    # the new-discovery tags are removed.
    mh.new_discovery_threshold = args.new_discovery_threshold

    # Service flag
    service = args.run_as_service

    # DEBUG - REMOVE ME
    if mh.debugging:
        mh.logger.debug(f"debugging: {mh.debugging}")
        mh.logger.debug(f"do_it_live: {mh.do_it_live}")
        mh.logger.debug(f"force_ioc_extract: {mh.force_ioc_extract}")
        mh.logger.debug(f"ignore_timers: {mh.ignore_timers}")
        mh.logger.debug(f"verbose_logging: {mh.verbose_logging}")
        mh.logger.debug(f"update_threshold: {mh.update_threshold}")
        mh.logger.debug(f"cert_pivoting: {mh.cert_pivoting}")
        mh.logger.debug(f"cert_pivot_threshold: {mh.cert_pivot_threshold}")
        mh.logger.debug(f"new_discovery_threshold: {mh.new_discovery_threshold}")
        mh.logger.debug(f"run_as_service: {service}")

    # Set verbose logging
    if mh.verbose_logging:
        mh.logger.setLevel(logging.DEBUG)
        mh.logger.debug(f"Verbose logging enabled!")

    if service:
        run_as_service(mh)
    else:
        mh.logger.info("Service flag set to False. Running MISPHunter once then exiting.")
        mh.run()


def shellsort(array):
    gap = len(array) // 2 # Gap is initialized to half the array's size

    while gap > 0:
        i = 0
        j = gap
        while j < len(array):
            if array[i] > array[j]:
                array[i], array[j] = array[j], array[i] # Invert

            i += 1
            j += 1

            l = i
            while l-gap > -1:
                if array[l-gap] > array[l]:
                    array[l-gap], array[l] = array[l], array[l-gap] # Invert
                l -= 1

        gap //=2 # Reducing gap by half

    return array


"""Number of Students Doing HW at a Given Time

Provided parameters:
    (1) `start_time` -- integer array
    (2) `end_time` -- integer array
    (3) `query_time` -- integer

The `ith` student started their HW at time `start_time[i]` and finished it at the time `end_time[i]`.

Return the NUMBER OF STUDENTS doing their hw at time `query_time`. More formally, return the number of students where `query_time` lays in the interval (start_time[i], end_time[i]) inclusive.
"""

from queue import Queue

def students_doing_work(start_time, end_time, query_time):
    number_of_students = 0
    for i in range(len(end_time)):
        if start_time[i] <= query_time and end_time[i] >= query_time:
            number_of_students += 1
    return number_of_students

if __name__ == "__main__":
    print(students_doing_work([1, 2, 2], [2, 3, 5], 4))
    print(students_doing_work([1, 2, 2], [2, 3, 5], 2))


import cv2
import numpy as np
from math import sqrt as squareRoot
import time
import argparse
import numpy as np
from skimage import io, img_as_float
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt

def print_distortion_distance(cluster_prototypes, points_by_label, k):
    distances = np.zeros((k,))

    for k_i in range(k):
        if (points_by_label[k_i] is not None):
            distances[k_i] += np.linalg.norm(points_by_label[k_i] - cluster_prototypes[k_i], axis=1).sum()
        else:
            distances[k_i] = -1

    print('Distortion Distances:')
    print(distances)
def k_means_clustering(image, k, num_iterations):

    image_vectors = image.reshape(-1, image.shape[-1])
    # Create corresponding label array (Initialize with Label: -1)
    labels = np.full((image_vectors.shape[0],), -1)
    # Assign Initial Cluster Prototypes
    cluster_prototypes = np.random.rand(k, 3)

    # Iteration Loop
    for i in range(num_iterations):
        start=time.time()
        print('Iteration: ' + str(i + 1))
        points_by_label = [None for k_i in range(k)]

        # Label them via closest point
        for rgb_i, rgb in enumerate(image_vectors):
            # [rgb, rgb, rgb, rgb, ...]
            rgb_row = np.repeat(rgb, k).reshape(3, k).T

            # Find the Closest Label via L2 Norm
            closest_label = np.argmin(np.linalg.norm(rgb_row - cluster_prototypes, axis=1))
            labels[rgb_i] = closest_label

            if (points_by_label[closest_label] is None):
                points_by_label[closest_label] = []

            points_by_label[closest_label].append(rgb)

        # Optimize Cluster Prototypes (Center of Mass of Cluster)
        for k_i in range(k):
            if (points_by_label[k_i] is not None):
                new_cluster_prototype = np.asarray(points_by_label[k_i]).sum(axis=0) / len(points_by_label[k_i])
                cluster_prototypes[k_i] = new_cluster_prototype

        # Find Current Distortion Distances
        print_distortion_distance(cluster_prototypes, points_by_label, k)
        print(labels)
        print(time.time()-start)

    return (labels, cluster_prototypes)
img = cv2.imread('D:/Projects/FYP/image classification/results/4-K_means_clusterisation_applied.jpg')
k_means_clustering(img,4,4)


#!/usr/bin/env python3

with open('day22.txt') as f:
    data = f.read().strip().split('\n')
data = data[2:]
data = [x.split() for x in data]
data = [[x[0].replace('/dev/grid/node-x','')]+x[1:] for x in data]
data = [[x[0].replace('y',''),int(x[1].replace('T','')),int(x[2].replace('T','')),int(x[3].replace('T',''))] for x in data]
data = [[x[0].split('-')]+x[1:] for x in data]
data = [[(int(x[0][0]),int(x[0][1]))] + x[1:] for x in data]
ddata = {d[0]:d[1:] for d in data}

#print(ddata)

def part1(data):
    vpairs = []

    for a in ddata:
            for b in ddata:
                if a != b and ddata[a][1] > 0 and ddata[a][1] <= ddata[b][2]:
                    #print(a,ddata[a],b,ddata[b])
                    vpairs.append((a,b))
    return len(vpairs)

print('Part 1: {}'.format(part1(data)))

def part2(data):
    xmax = 36
    ymax = 26
    print(' ',*[' {}'.format(x) for x in range(10)]+[' {}'.format(x) for x in range(10)]+[' {}'.format(x) for x in range(10)]+[' {}'.format(x) for x in range(7)])
    for i in range(ymax+1):
        js = []
        for j in range(xmax+1):
            if ddata[(j,i)][1] > 100: js.append(' X ')
            elif (j,i) == (0,0): js.append('(.)')
            elif (j,i) == (xmax,0): js.append(' G ')
            elif ddata[(j,i)][1] == 0: js.append(' 0 ')
            else: js.append(' . ')
        #print(js)
        print(i,' ',*js, sep='')
part2(data)
print('Count the steps:\n- 33 steps to get (0,35) empty')
print('- 1 step to flip goal data and empty data')
print('- it now takes 5 steps to move goal data one step to the right')
print('- 34*5 steps to get the goal data to (0,1)')
print('- 4 steps to get (0,0) empty')
print('- 1 step to move goal data to (0,0)')
print('Part 2: {}'.format(33+1+34*5+5))


class Dsu:
    def __init__(self,size):
        self.fa=list(range(size))
        self.size = [1] * size
    def find(self, x):
        if self.fa[x] != x:
            self.fa[x] = self.find(self.fa[x])
        return self.fa[x]
    def union(self, x, y):
        x, y = self.find(x), self.find(y)
        if x == y:
            return
        if self.size[x] < self.size[y]:
            x, y = y, x
        self.fa[y] = x
        self.size[x] += self.size[y]

n,m=map(int,input().split())
dsu=Dsu(n)
for _ in range(m):
    z,x,y=map(int,input().split())
    x=x-1
    y=y-1
    if (z==1):
        dsu.union(x,y)
    elif (z==2):
        x,y=dsu.find(x),dsu.find(y)
        if (x==y):
            print("Y")
        else:
            print("N")


#!/usr/bin/python
# -*- coding: utf-8 -*-
import sys, math
from PyQt4 import QtGui, QtCore
import preview_ui
from vidGen import getFrameFromVideo

# Shows the preview window showcasing the changes made in the editor window
class Preview(QtGui.QWidget) :

    closeApp = QtCore.pyqtSignal() #signal , slot to be with caller

    def __init__(self, ui):
        super(Preview, self).__init__()
        self.ui = ui
        self.font = self.color = self.image = None

    def setup_connections(self) :
        self.ui.pushButton.clicked.connect( self.closeWidget )

    def moveStuff(self, scaleFactor, frameCoords, bannerCoords, bannerLabelCoords, bannerLabelText, font, color, image) :
        #resize whole widget
        self.resize( scaleFactor * frameCoords.width() + 30 , scaleFactor * frameCoords.height() + 30 )

        #resize video frame
        self.ui.bgLabel.resize( scaleFactor * frameCoords.width(), scaleFactor * frameCoords.height() )
        originalVideoLocation = self.caller.caller.file #calling main window
        if originalVideoLocation is not None :
            bgPixmap = QtGui.QPixmap( getFrameFromVideo( originalVideoLocation, 200 ) ) #make sure asked frame count is less than that of video
            self.ui.bgLabel.setPixmap( bgPixmap)

        #resize banner image
        self.ui.bannerlImage.setGeometry( bannerCoords )
        diffX = abs(self.ui.bannerlImage.geometry().x() - frameCoords.x() )
        diffY = abs(self.ui.bannerlImage.geometry().y() - frameCoords.y() )
        self.ui.bannerlImage.setGeometry( frameCoords.x() + scaleFactor * diffX, \
                                          frameCoords.y() + scaleFactor * diffY, \
                                          scaleFactor * self.ui.bannerlImage.geometry().width(), \
                                          scaleFactor * self.ui.bannerlImage.geometry().height()  )

        #resize label
        self.ui.bannerLabel.setGeometry( bannerLabelCoords )

        diffX = abs(self.ui.bannerLabel.geometry().x() - frameCoords.x() )
        diffY = abs(self.ui.bannerLabel.geometry().y() - frameCoords.y() )
        self.ui.bannerLabel.setGeometry( frameCoords.x() + scaleFactor * diffX, \
                                          frameCoords.y() + scaleFactor * diffY, \
                                          scaleFactor * self.ui.bannerLabel.geometry().width(), \
                                          scaleFactor * self.ui.bannerLabel.geometry().height()  )

        self.ui.bannerLabel.setText( bannerLabelText )

        if font is not None :
            self.font = QtGui.QFont(font)
            self.font.setPointSize( self.font.pointSize() * scaleFactor ) #resize font
            self.ui.bannerLabel.setFont(self.font)
        if color is not None :
            self.color = color
            self.ui.bannerLabel.setStyleSheet("QLabel { color: %s}" % color.name())

        if image is not None :
            pixmap = QtGui.QPixmap( image )
            self.ui.bannerlImage.setPixmap( pixmap.scaled( scaleFactor * self.ui.bannerlImage.width(), scaleFactor * self.ui.bannerlImage.height())) #resize image
            self.image = image

        self.ui.bannerLabel.raise_() #raise banner text on top of banner image

        #move OK button out of the way of the image. If it is not visible, user may exit with [X] button on top
        self.ui.pushButton.move( self.ui.pushButton.geometry().x(), self.ui.bgLabel.geometry().y() + self.ui.bgLabel.geometry().height() + 20 )
        self.repaint()

    def saveImage(self) :
        image =  QtGui.QImage(self.geometry().width(), self.geometry().height()  , QtGui.QImage.Format_ARGB32_Premultiplied);
        qp = QtGui.QPainter()
        qp.begin(image)
        # qp.fillRect(self.ui.bannerlImage.geometry(), Qt::yellow);
        # pix = QtGui.QPixmap(image)
        # qp.drawPixmap( QtCore.QRect(50, 50, 100, 100), pix, QtCore.QRect(50, 50, 100, 100) )
        if self.image is not None :
            qp.drawPixmap( self.ui.bannerlImage.geometry(), self.ui.bannerlImage.pixmap(),  self.ui.bannerlImage.pixmap().rect() )
            # qp.drawPixmap( self.ui.bannerlImage.geometry(), pix, self.ui.bannerlImage.geometry() )
        if self.font is not None :
            qp.setFont( self.font )
        if self.color is not None :
            qp.setPen(self.color)

        qp.drawText( self.ui.bannerLabel.geometry(), QtCore.Qt.AlignCenter | QtCore.Qt.AlignVCenter,  self.ui.bannerLabel.text());
        image.save("output.png")
        qp.end()

    def closeEvent(self, event) :
        self.closeWidget()

    def closeWidget(self) :
        self.saveImage()
        self.closeApp.emit()
        self.close()

def showPreview( caller  ) :
    ui = preview_ui.Ui_Preview()
    widget = Preview( ui )
    ui.setupUi(widget)
    widget.setup_connections()
    widget.caller = caller
    widget.show()
    return widget

if __name__ == "__main__":
    import sys
    app = QtGui.QApplication(sys.argv)
    widget = showPreview(app)
    sys.exit(app.exec_())


import ctypes
from numpy import ctypeslib
import numpy as np
import time

class CPP():
    def __init__(self, path):
        self.cpp = ctypes.CDLL(path)

        self.dft = self.cpp.dft
        self.dft.argtypes = [ctypeslib.ndpointer(ctypes.c_double), ctypeslib.ndpointer(ctypes.c_double), ctypes.c_int]

        self.fft = self.cpp.fft
        self.fft.argtypes = [ctypeslib.ndpointer(ctypes.c_double), ctypes.c_int]

    def get_dft(self, N: int):
        t = np.arange(0, N, 1)
        x = 1*np.cos(2 * np.pi / 1024 * 500 * t)
        y = 0*np.sin(2 * np.pi / 1024 * 3 * t)

        input = np.column_stack((x, y))
        output = np.column_stack((x, y))

        start = time.time() * 1000
        self.dft(input, output, N)
        end = time.time() * 1000

        return (output, end-start)

    def get_fft(self, N: int):
        t = np.arange(0, N, 1)
        x = 1*np.cos(2 * np.pi / 1024 * 500 * t)
        y = 0*np.sin(2 * np.pi / 1024 * 3 * t)

        input = np.column_stack((x, y))

        start = time.time() * 1000
        self.fft(input, N)
        end = time.time() * 1000

        return (input, end-start)


'''
import matplotlib.dates as mdates
import matplotlib.patches as patches
from dateutil.relativedelta import relativedelta
import numpy as np
'''
import time
import pprint
import math
from datetime import datetime, timedelta
from collections import defaultdict
import pandas as pd

from read_reports import parse_html
from draw_plots import plot_weekly_series, plot_weekly_balance

pp = pprint.PrettyPrinter(indent=2)

def calculate_risk_and_split_balance(balance, risk_manage):
    if risk_manage is None:
        return 1, balance, 0  # If no risk level is provided, return 100% risk and all balance is risky

    sorted_risk_levels = sorted(risk_manage.keys(), reverse=True)
    for level in sorted_risk_levels:
        if balance >= level:
            risk_percent = risk_manage[level] / 100
            risky_balance = balance * risk_percent
            buffer_balance = balance - risky_balance
            return risk_percent, risky_balance, buffer_balance

def process_deals(deals):
    processed_deals = []
    drawdown = 0
    series_length = 0
    drawdown_levels = {}
    series_start_balance = None  # добавляем переменную для хранения начала баланса на начало серии
    deals.pop(0)

    for deal in deals:
        profit = float(deal['Прибыль'].replace(" ", ""))
        balance = float(deal['Баланс'].replace(" ", ""))

        if series_start_balance is None or drawdown == 0:
            series_start_balance = balance - profit

        balance_change = balance - series_start_balance  # изменение баланса по серии

        if profit > 0:
            # If the previous deal(s) were losses, record them as a drawdown
            if drawdown < 0:
                processed_deals.append({
                    'Время': deal['Время'],
                    'Объем': deal['Объем'],
                    'Баланс': deal['Баланс'],
                    'Прибыль': balance_change,
                    'Просадка': drawdown,
                    'Размер серии': series_length + 1,
                    'Уровни просадки': drawdown_levels
                })
                # Reset the drawdown, series length, drawdown levels and series_start_balance
                drawdown = 0
                series_length = 0
                drawdown_levels = {}
            else:
                processed_deals.append({
                    'Время': deal['Время'],
                    'Объем': deal['Объем'],
                    'Баланс': deal['Баланс'],
                    'Прибыль': balance_change,
                    'Просадка': 0,
                    'Размер серии': 1,
                    'Уровни просадки': {}
                })
        else:
            drawdown += profit
            series_length += 1
            # Update the drawdown levels
            drawdown_levels[series_length] = drawdown

    return processed_deals

def recalculate_balance(processed_deals, initial_balance, drawdown_level, start_date=None, end_date=None, multiplier=3000, risk_manage=None, profit_mining=None):
    first_deposit = initial_balance
    balance = initial_balance
    balance_history = []
    withdrawals = 0  # Initialize the withdrawals counter
    deposits = 1  # Initialize the deposits counter

    # Convert start_date and end_date to datetime objects if they are not None and are strings
    if start_date is not None and isinstance(start_date, str):
        start_date = datetime.strptime(start_date, '%d.%m.%Y')
    if end_date is not None and isinstance(end_date, str):
        end_date = datetime.strptime(end_date, '%d.%m.%Y')

    for deal in processed_deals:
        # Parse the deal date
        deal_date = datetime.strptime(deal['Время'], '%Y.%m.%d %H:%M:%S')  # adjust the format as necessary

        # If start_date and end_date are provided, skip the deal if it's not within the date range
        if start_date is not None and deal_date < start_date:
            continue
        if end_date is not None and deal_date > end_date:
            continue

        # If the drawdown level of the deal is greater than the specified drawdown level,
        # use the loss corresponding to the specified drawdown level
        if deal['Размер серии'] > drawdown_level:
            loss = deal['Уровни просадки'].get(drawdown_level, 0)
        else:
            loss = deal['Прибыль']

        # Calculate risk and split balance
        risk_percent, risky_balance, buffer_balance = calculate_risk_and_split_balance(balance, risk_manage)

        # Apply multiplier based on risky balance instead of total balance
        balance_ratio = risky_balance / multiplier
        balance_ratio = max(math.floor(balance_ratio), 1)
        loss *= balance_ratio

        # Calculate the new balance after the deal
        last_positive_balance = risky_balance  # Save the last positive risky balance before changing it
        risky_balance += loss
        balance_change = loss

        # If risky balance is negative, set it to zero and adjust balance_change to be the negative of the last positive balance
        if risky_balance < 0:
            balance_change = -last_positive_balance
            risky_balance = 0  # Reset risky balance to zero

        # Recalculate the total balance by summing risky and buffer balances
        balance = risky_balance + buffer_balance

        # Calculate profit mining
        profit_mining_deduction = 0
        if profit_mining is not None:
            for level in sorted(profit_mining.keys(), reverse=True):
                if balance >= level and balance_change > 0:
                    profit_mining_deduction = balance_change * profit_mining[level] / 100
                    balance_change -= profit_mining_deduction
                    balance -= profit_mining_deduction
                    break

        balance_history.append({
            'Время': deal['Время'],
            'Прибыль': balance_change,
            'Баланс': balance,
            'Размер серии': deal['Размер серии'],
            'Множитель': balance_ratio,
            'Тип': 'сделка',
            'Сбор дохода': profit_mining_deduction,
        })

        # If balance is less than first_deposit, add a deposit
        if balance < first_deposit:
            deposit_date = deal_date + timedelta(seconds=5)
            deposit_amount = first_deposit - balance
            balance += deposit_amount
            deposits += 1  # Increase the deposits counter
            balance_history.append({
                'Время': deposit_date.strftime('%Y.%m.%d %H:%M:%S'),
                'Прибыль': deposit_amount,
                'Баланс': balance,
                'Размер серии': 0,
                'Множитель': 0,
                'Тип': 'пополнение',
                'Сбор дохода': 0,
            })

        # If balance_ratio exceeds 2 and balance exceeds first_deposit, add a withdrawal
        # But only if the number of withdrawals is less than the number of deposits
        if balance >= ((multiplier * 2) + first_deposit) and withdrawals < deposits:
            withdrawal_date = deal_date + timedelta(seconds=5)
            balance -= first_deposit
            withdrawals += 1  # Increase the withdrawals counter
            balance_history.append({
                'Время': withdrawal_date.strftime('%Y.%m.%d %H:%M:%S'),
                'Прибыль': -first_deposit,
                'Баланс': balance,
                'Размер серии': 0,
                'Множитель': 0,
                'Тип': 'снятие средств',
                'Сбор дохода': 0,
            })

    return balance_history

def count_series_size(processed_deals):
    series_size_count = {}
    for deal in processed_deals:
        series_size = deal['Размер серии']
        if series_size in series_size_count:
            series_size_count[series_size] += 1
        else:
            series_size_count[series_size] = 1
    return series_size_count

def count_income(balance_history):
    monthly_income = defaultdict(float)  # A dictionary to hold monthly income
    annual_income = defaultdict(float)  # A dictionary to hold annual income
    total_income = 0.0  # A variable to hold the total income

    # Go through each record in balance history
    for record in balance_history:
        # Parse the date
        date = datetime.strptime(record['Время'], '%Y.%m.%d %H:%M:%S')
        # Get the income
        income = int(record['Сбор дохода'])
        # Add the income to the appropriate month in the monthly income dictionary
        month_key = (date.year, date.month)
        monthly_income[month_key] += income
        # Add the income to the appropriate year in the annual income dictionary
        annual_income[date.year] += income
        # Add the income to the total income
        total_income += income

    # Convert the keys in the monthly income dictionary to strings for readability
    monthly_income = {(f'{year}-{month:02}'): income for (year, month), income in monthly_income.items()}

    # Calculate average annual income, considering only years with non-zero income
    non_zero_annual_incomes = [income for income in annual_income.values() if income != 0]
    average_annual_income = sum(non_zero_annual_incomes) / len(non_zero_annual_incomes) if non_zero_annual_incomes else 0

    return {
        'monthly_income': dict(monthly_income),
        'annual_income': dict(annual_income),
        'average_annual_income': average_annual_income,
        'total_income': total_income
    }

def save_to_excel(data, filename):
    df = pd.DataFrame(data)  # Создаем DataFrame из списка словарей
    df.to_excel(filename, index=False, startrow=2)  # Записываем DataFrame в xlsx файл

def chek_calculates(calc_params, history_file, label):

    risk_manage = {
        0: 100,
        15000: 50,
        45000: 25,
        135000: 12.5,
        405000: 6.25,
    }

    profit_mineing = {
        45000: 10,
        135000: 20,
    }

    deals = parse_html(history_file)
    processed_deals = process_deals(deals)
    count_series = count_series_size(processed_deals)
    save_to_excel(processed_deals, f'files/{label}_orig.xlsx')

    start_date = '01.01.2014'
    end_date = '01.01.2024'

    for current_params in calc_params:
        multiplier = current_params['initial_balance']
        label2 = f"{current_params['drawdown_level']:02d}_{current_params['initial_balance']}"
        calculated_deals = recalculate_balance(processed_deals, current_params['initial_balance'], current_params['drawdown_level'], start_date, end_date, multiplier, risk_manage, profit_mineing)
        render_plot_file = f'files/{label}_{label2}.png'
        plot_weekly_balance(calculated_deals, render_plot_file)
        save_to_excel(calculated_deals, f'files/{label}_{label2}.xlsx')
        incomes = count_income(calculated_deals)

        print('=====================')
        print(f'initial_balance: {current_params["initial_balance"]}')
        print(f'drawdown_level: {current_params["drawdown_level"]}')
        print('=====================')
        print('Годовой доход:')
        pp.pprint(incomes['annual_income'])
        print(f'Средний годовой доход: {int(incomes["average_annual_income"])}')
        print(f'Итоговый баланс: {calculated_deals[-1]["Баланс"]}')
        print(f'Количество трейдов: {len(calculated_deals)}\n')

    print('Распределение серий:')
    pp.pprint(count_series)

def main():
    label = 'GBP_H1_ma8_s35_p50'
    history_file = 'files/reports/GBP_H1_ma8_s35_p50.html'
    calc_params = [
        {'initial_balance': 500, 'drawdown_level': 8},
        {'initial_balance': 1000, 'drawdown_level': 9},
        {'initial_balance': 2000, 'drawdown_level': 10},
        {'initial_balance': 4000, 'drawdown_level': 11},
        ]
    chek_calculates(calc_params, history_file, label)

if __name__ == '__main__':
    main()


from math import fabs
while True:
    a = input("Введите натуральное число ")
    if a.isdecimal():
        a = int(a)
        break
    else:
        print("Ошибка: вы ввели не натуральное целое число")
sum = 0
while a > 0:
    sum += a % 10
    a //= 10
print(f"Сумма введённого натурального числа – {sum}")


import os
import re
import time
from math import log10, floor
from functools import reduce
import csv
import pandas as pd
from collections import namedtuple
from datetime import datetime
from types import SimpleNamespace

def round_sig(x, sig=2):
    return round(x, sig - int(floor(log10(abs(x)))) - 1)

def checksum(st):
    return reduce(lambda x, y: x + y, map(ord, st))

# TODO the regex are super-slow for very long SQL and/or VEGA

# !grep -n 'stdlog sql_execute' $logfile | head -1
# re_sql = re.compile(r"""sql_execute [0-9]+ ([0-9]+) (.*) .* .* {"query_str","execution_time_ms","total_time_ms"} {"(.*)","[0-9]+","[0-9]+"}""", re.MULTILINE|re.DOTALL)

# 2019-11-11T19:06:39.325374 1 16 MapDHandler.cpp:946 stdlog_begin sql_execute 25 0 mapd mike.hinchey 628-5DjJ {"query_str"} {"SELECT COUNT(*) AS n FROM zendesk_jira"}
re_sql = re.compile(
    r"""sql_execute [0-9]+ ([0-9]+) (.*) .* .* {"query_str"} {"(.*)"}""",
    re.MULTILINE | re.DOTALL,
)

def parse_sql(line, event):
    # print('\nLINE:', line, 'LINE_END')
    match = re_sql.match(line)
    # print(line, match)
    if match:
        event.func = "sql_execute"
        event.dur_ms = int(match.group(1))
        event.dbname = match.group(2)
        event.query = match.group(3).strip()
        event.query = event.query.replace('""', '"')
        event.query_checksum = checksum(event.query)

        parts = event.query.split(" ", 1)
        cmd = parts[0].upper()
        if cmd in ["SELECT", "WITH", "EXPLAIN"]:
            event.modify = False

        else:  # elif cmd in ['COPY', 'CREATE', 'DROP', 'TRUNCATE', 'UPDATE', 'INSERT', 'DELETE', 'ALTER', 'GRANT', 'REVOKE', 'OPTIMIZE']:
            event.modify = True
        return event

def exec_sql(event):
    # TODO connect event.dbname

    tstart = time.time()
    c = con.con.execute(query)
    tend = time.time()
    event.dur_ms = tend - tstart
    tsec = round_sig(1000 * (tend - tstart), 1)
    rs = c.fetchall()
    count = len(rs)
    # output.write(f"""|metrics time={tsec:n} count={count}\n""")

# !grep -n 'stdlog render_vega' $logfile | head -1
re_vega = re.compile(
    r"""render_vega [0-9]+ ([0-9]+) (.*) .* .* {"widget_id","compression_level","vega_json","nonce"} {"[0-9]+","[0-9]+","(.*)","[0-9]+"}""",
    re.MULTILINE | re.DOTALL,
)

def parse_vega(line, event):
    match = re_vega.match(line)
    if match:
        event.func = "render_vega"
        event.dur_ms = int(match.group(1))
        event.dbname = match.group(2)
        event.query = match.group(3).strip()
        event.query = event.query.replace('""', '"')
        event.query_checksum = checksum(event.query)
        return event

def exec_vega(event):
    # TODO connect event.dbname

    tstart = time.time()
    png = con.con.render_vega(vega)
    tend = time.time()
    event.dur_ms = tend - tstart
    tsec = round_sig(1000 * (tend - tstart), 1)
    return event

def transform_lines(line, event):
    pos = line.find(" stdlog ")
    z = 8
    if pos == -1:
        pos = line.find(" stdlog_begin ")
        z = 14
    if pos >= 0:
        line = line[pos + z :]
        if line.startswith("sql_execute "):
            return parse_sql(line, event)
        elif line.startswith("render_vega "):
            return parse_vega(line, event)

def read_log_file_sql(
    log_path, load_label, quit_on_error=True, max_queries=1000000, max_lines=20000000
):

    run_tstamp = None

    beginning = r""" . [0-9]+ [A-Za-z\.\:0-9]+ """

    with open(log_path, "r") as input:
        events = []
        ctlines = 0
        ct_events = 0
        prev_line = None
        lines = []
        event = SimpleNamespace()
        for line in input:
            if ctlines >= max_lines:
                print("max_lines", ctlines)
                break
            ctlines += 1
            if ct_events >= max_queries:
                print("max_queries", ct_events)
                break

            if len(lines) == 1 and (
                line.find("sql_execute") == -1 and line.find("render_vega") == -1
            ):
                # print('continue', line)
                continue
            # else:
            #     print('keep', len(lines), line)

            try:
                try:
                    tstamp = pd.to_datetime(line[:26])
                except ValueError as e:
                    tstamp = None
                match = re.match(beginning, line[26:])
                if tstamp and match:
                    if run_tstamp is None:
                        run_tstamp = tstamp
                    event.tstamp = tstamp
                    event.run_tstamp = run_tstamp

                    event = transform_lines("".join(lines), event)
                    if event:
                        # print(event)
                        events.append(event)
                        ct_events += 1

                    lines = [line]
                    event = SimpleNamespace()
                else:
                    # print('NO MATCH', line)
                    lines.append(line)

            except Exception as e:
                ct_events += 1
                if quit_on_error:
                    raise e
                else:
                    print(e)
        event = transform_lines("".join(lines), event)
        if event:
            events.append(event)
    len(events)

    df = pd.DataFrame([x.__dict__ for x in events])
    df["load_label"] = load_label
    df["load_timestamp"] = datetime.now()
    df["srcfile"] = log_path
    return df

# def read_log_files_sql(log_paths, load_label, quit_on_error = True, max_lines = 20000000):
#     return pd.concat([read_log_file_sql(p, load_label, quit_on_error, max_lines) for p in log_paths])


# -*- coding:utf-8 -*-

from app import create_app
from app.model.base_db import Result

app = create_app("default")

with open('../../../../data/jiedai.txt', 'r') as f:
    num = 0
    for line in f.readlines():
        doc_id = line.split('+doc_id+')[0]
        Result(model_id=str(num), doc_id=doc_id).save()
        num += 1


import atheris

with atheris.instrument_imports():
  # from ruamel import yaml as ruamel_yaml
  import sys
  import warnings

# Suppress all warnings.
warnings.simplefilter("ignore")

# ryaml = ruamel_yaml.YAML(typ="safe", pure=True)
# ryaml.allow_duplicate_keys = True

@atheris.instrument_func
def TestOneInput(input_bytes):
  fdp = atheris.FuzzedDataProvider(input_bytes)
  data = fdp.ConsumeIntList(2,2)

  print(data)

  # try:
  #   iterator = ryaml.load_all(data)
  #   for _ in iterator:
  #     pass
  # except ruamel_yaml.error.YAMLError:
  #   return

  # except Exception:
  #   input_type = str(type(data))
  #   codepoints = [hex(ord(x)) for x in data]
  #   sys.stderr.write(
  #       "Input was {input_type}: {data}\nCodepoints: {codepoints}".format(
  #           input_type=input_type, data=data, codepoints=codepoints))
  #   raise

def main():
  atheris.Setup(sys.argv, TestOneInput)
  atheris.Fuzz()

if __name__ == "__main__":
  main()


from odoo import fields, models, api, _
from odoo.exceptions import ValidationError

class AccountJournal(models.Model):
    _inherit = 'account.journal'

    pos_payment_method_ids = fields.One2many('pos.payment.method', 'journal_id', string='Point of Sale Payment Methods')

    @api.constrains('type')
    def _check_type(self):
        methods = self.env['pos.payment.method'].sudo().search([("journal_id", "in", self.ids)])
        if methods:
            raise ValidationError(_("This journal is associated with a payment method. You cannot modify its type"))

    def _get_journal_inbound_outstanding_payment_accounts(self):
        res = super()._get_journal_inbound_outstanding_payment_accounts()
        account_ids = set(res.ids)
        for payment_method in self.sudo().pos_payment_method_ids:
            account_ids.add(payment_method.outstanding_account_id.id or self.company_id.account_journal_payment_debit_account_id.id)
        return self.env['account.account'].browse(account_ids)


import numpy as np
import matplotlib.pyplot as plt
from sklearn import tree
from sklearn.ensemble import AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.datasets import make_gaussian_quantiles
from sklearn.metrics import classification_report

# 该数据集由sklearn随机生成
# 生成2维正态分布，生成的数据按分位数分为两类，500个样本,2个样本特征,样本特征均值为0,0
x1, y1 = make_gaussian_quantiles(mean=(0, 0), n_samples=500, n_features=2, n_classes=2)
# 生成2维正态分布，生成的数据按分位数分为两类，400个样本,2个样本特征，样本特征均值为3,3
x2, y2 = make_gaussian_quantiles(mean=(3, 3), n_samples=500, n_features=2, n_classes=2)
# 将两组数据合成一组数据
x_data = np.concatenate((x1, x2))
# - y2 + 1,由于y是0,1的特征矩阵，该方式可以将0,1进行互换
y_data = np.concatenate((y1, 1 - y2))

# 画图测试，可以发现生成的点原来的图是内黑色外黄色
plt.scatter(x1[:, 0], x1[:, 1], c=y1)
plt.show()

# 经过转换，得到了两团正态分布点，一团内黑外黄(0,0)，一团内黄外黑(3,3)
plt.scatter(x_data[:, 0], x_data[:, 1], c=y_data)
plt.show()

# 算法将使用决策树与Adaboost进行对比
# 决策树模型
model = tree.DecisionTreeClassifier(max_depth=3)

# 输入数据建立模型
model.fit(x_data, y_data)

# 获取数据值所在的范围
x_min, x_max = x_data[:, 0].min() - 1, x_data[:, 0].max() + 1
y_min, y_max = x_data[:, 1].min() - 1, x_data[:, 1].max() + 1

# 生成网格矩阵
xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.02),
                     np.arange(y_min, y_max, 0.02))

z = model.predict(np.c_[xx.ravel(), yy.ravel()])  # ravel与flatten类似，多维数据转一维。flatten不会改变原始数据，ravel会改变原始数据
z = z.reshape(xx.shape)
# 等高线图
cs = plt.contourf(xx, yy, z)
# 样本散点图
plt.scatter(x_data[:, 0], x_data[:, 1], c=y_data)
plt.show()

# 模型准确率
print("DTree model score = " + str(model.score(x_data, y_data)))

# AdaBoost模型(弱学习器使用决策树模型（决策树的深度 = 3），迭代次数为10)
model = AdaBoostClassifier(DecisionTreeClassifier(max_depth=3), n_estimators=10)
# 训练模型
model.fit(x_data, y_data)

# 获取数据值所在的范围
x_min, x_max = x_data[:, 0].min() - 1, x_data[:, 0].max() + 1
y_min, y_max = x_data[:, 1].min() - 1, x_data[:, 1].max() + 1

# 生成网格矩阵
xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.02),
                     np.arange(y_min, y_max, 0.02))

# 获取预测值
z = model.predict(np.c_[xx.ravel(), yy.ravel()])
z = z.reshape(xx.shape)
# 等高线图
cs = plt.contourf(xx, yy, z)
# 样本散点图
plt.scatter(x_data[:, 0], x_data[:, 1], c=y_data)
plt.show()

# 模型准确率
model.score(x_data, y_data)

print("AdaBoost model score = " + str(model.score(x_data, y_data)))


#definerer alfabetet som en string, hvor a=1, derfor blank space i starten
#desuden står den igen, det er fordi man på den måde kan få store bogstaver.
alf = '0abcdefghijklmnopqrstuvwxyzæøåABCDEFGHIJKLMNOPQRSTUVWXYZÆØÅ !?,.'

#definer forskellige vigtige funktioner
p = 17
q = 11

m = p*q

#eulers phifunktion
phim = (p-1) * (q-1)

#skal være indbyrdes primisk med phim
k = 7
if(phim%k == 0):
    print('phim og k er ikke indbyrdes primiske')
else:
    pass

#blokstørrelse skal defineres. ved blokstørrelse 1 skal der tages 2 tal, derfor ganges med 2
blokstørrelse = 1
blok = blokstørrelse*2

#brug find fuctionen til at finde ud af hvilken værdi tallene har
#denne variabel er til at skrive, hvad der skal enkrypteres
message = 'Citizen X'
print('Tekst som skal krypteres: ' + message + '\n')
print('krypteres med blokstoerrelse '+str(blokstørrelse)+
', p='+str(p)+', q='+str(q)+', og k='+str(k)+'\n')

plainmessage = ''

#her tilføjer jeg alle talenne til plaintext variablen
for a in message:
    if alf.index(a)<10:
        plainmessage += '0'
        plainmessage += str(alf.index(a))
    else:
        plainmessage += str(alf.index(a))

#denne er til for at sørge for at alting er splittet op i par, da jeg bruger blokstørrelse 1
#den skal måske forstås lidt bedre
plainmessage = [plainmessage[i:i+blok] for i in range(0, len(plainmessage), blok)]

#denne laver det fra en list til en integer, som jeg da kan regne med
#denne process fjerner 0'erne fra teksten, hvilket godt kan blive problematisk?
plainmessage = list(map(int, plainmessage))

print("Dette er den ikke-krypterede klartekst:")
print(plainmessage[0:len(plainmessage)])
print('\n')

#her definerer jeg variablen, som jeg skal bruge til den enkrypterede tekst
n = 0
#enkrypteret tekst variabel
encmessage = plainmessage.copy()

encmessage = [(x**k)%m for x in encmessage]

print("Dette er den krypterede tekst:")
print(encmessage)