import numpy as np
import cv2
import imutils
import os
import joblib
import json
from skimage.feature import hog
from skimage import data, exposure

datas=[]
labels = []

thisdict = {
    "a": 1,    "b": 2,    "c": 3,    "d": 4,    "e": 5,    "f": 6,    "g": 7,    "h": 8,    "i": 9,    "j": 10,
    "k": 11,    "l": 12,    "m": 13,    "n": 14,    "o": 15,    "p": 16,    "q": 17,    "r": 18,    "s": 19,
    "t": 20,    "u": 21,    "v": 22,    "w": 23,    "x": 24,    "y": 25,    "z": 26,
    "A":27,    "B":28,    "C":29,    "D":30,    "E":31,    "F":32,    "G":33,    "H":34,    "I":35,    "J":36,
    "K":37,    "L":38,    "M":39,    "N":40,    "O":41,    "P":42,    "Q":43,    "R":44,    "S":45,    "T":46,
    "U":47,    "V":48,    "W":49,    "X":50,    "Y":51,    "Z":52,
    "1":53,    "2":54,    "3":55,    "4":56,    "5":57,    "6":58,    "7":59,    "8":60,    "9":61,    "0":62}

def pickleOlustur(fileName,object,method=None):
    try:
        if method is None:
            with open(fileName, 'w') as fp:
                json.dump(object, fp)
        elif method == 1:
                joblib.dump(object, fileName)
        return '0'
    except BaseException as e:
        print(str(e))
        return '-1'

def pickleYukle(fileName,method=None):
    #data =joblib.load(fileName)
    if method is None:
        with open(fileName, 'r') as fp:
            data = json.loads(fp.read())
    elif method == 1:
        data =joblib.load(fileName)
    return data

def CalculateHOG(gray, method = None):
    gray = cv2.bilateralFilter(gray, 13, 15, 15)
    edged = cv2.Canny(gray, 100, 200)
    kernel = np.ones((3, 3), np.uint8)
    # opening = cv2.morphologyEx(edged, cv2.MORPH_CLOSE, kernel)
    # cv2.imshow("edged", edged)
    # cv2.imshow("opening", opening)
    # cv2.waitKey(0)
    fd, roi_hog_fd = hog(edged, orientations=8, pixels_per_cell=(2, 2),
                         cells_per_block=(1, 1), visualize=True, block_norm='L2-Hys')
    hog_image_rescaled = exposure.rescale_intensity(roi_hog_fd, in_range=(0, 10))
    resultImage = hog_image_rescaled

    # cv2.imshow("edged", edged)
    # cv2.imshow("roi_hog_fd", roi_hog_fd)
    # cv2.imshow("resultImage", resultImage)
    # cv2.waitKey(0)

    return resultImage

def intTryParse(value):
    try:
        return int(value), True
    except ValueError:
        return value, False

def modelOlustur(method=None):
    dosyaYolu = "D:\OutSource\PlateRecognition\Dataset\English.Alphabet.Dataset\English Alphabet Dataset"
    dirList = os.listdir(dosyaYolu)
    dictValues = list(thisdict.values())
    for dir in dirList:
        imagePathInt, res = intTryParse(dir)
        if res and imagePathInt in dictValues:
            resPath = dosyaYolu + "\\"+ dir
            imageList = os.listdir(resPath)
            for image in imageList:
                imagePath = resPath + "\\" + image
                myImage = cv2.imread(imagePath)
                myImageResized = cv2.resize(myImage, (30,30))
                myHogImage = CalculateHOG(myImageResized)
                datas.append(np.ravel(myHogImage))
                labels.append(imagePathInt)
            # Burada model olusturacagim simdi
    x = 2
        # image = CalculateHOG(path, method)
        # if image is None:
        #     continue
        # datas.append(np.ravel(np.array(image)))
    #     if imagePath.__contains__("butterfly"):
    #         labels.append(0)
    #     else:
    #         labels.append(1)
    # print("SVM fit basliyor")
    # clf = svm.SVC()
    # clf.fit(datas, labels)
    # # cv2.imshow("image", image)
    # #  cv2.waitKey(0)
    #  # dosyaya kaydederiz
    # return clf

if __name__ =="__main__":
    modelOlustur()


import numpy as np

import matplotlib.pyplot as plt
from matplotlib.ticker import MultipleLocator, AutoMinorLocator
from matplotlib.colors import Normalize, LogNorm

import tensorflow as tf

from argparse import ArgumentParser
from pathlib import Path
from tqdm import tqdm
import h5py
import os
import os.path

from xp_utils import XPSampler, sqrt_icov_eigen, calc_invs_eigen
from utils import batch_apply_tf
from model import GaussianMixtureModel, FluxModel, chi_band, gaussian_prior, \
                  grads_stellar_model, grads_stellar_params, \
                  get_batch_iterator,  identify_outlier_stars, \
                  identify_flux_outliers, train_stellar_model, \
                  plot_gmm_prior, assign_variable_padded, corr_matrix, \
                  calc_stellar_fisher_hessian, load_data, save_as_h5, load_h5
import model
import plot_utils

nscale = 1.

def load_training_data(fname, validation_frac=0.2, seed=1, thin=1):
    # Load training data
    with h5py.File(fname, 'r') as f:
        d = {key:f[key][:][::thin] for key in f.keys()}
        sample_wavelengths = f['flux'].attrs['sample_wavelengths'][:]
    #sample_wavelengths = np.load('wl.npy').astype('f4')

    # Ensure that certain fields are in float32 format
    f4_keys = [
        'flux', 'flux_err', 'flux_sqrticov',
        'flux_cov_eival_max', 'flux_cov_eival_min',
        'plx', 'plx_err',
        'stellar_ext', 'stellar_ext_err',
        'stellar_type', 'stellar_type_err'
    ]
    for k in f4_keys:
        d[k] = d[k].astype('f4')

    # Warn about NaNs
    check_nan_keys = [
        'flux', 'flux_err', 'flux_sqrticov',
        'flux_cov_eival_max', 'flux_cov_eival_min',
        'plx', 'plx_err',
        'stellar_type', 'stellar_type_err'
    ]
    for k in check_nan_keys:
        if np.any(np.isnan(d[k])):
            raise ValueError(f'NaNs detected in {k}!')

    # Replace NaN extinctions with 0 +- infinity
    idx = np.isnan(d['stellar_ext']) | np.isnan(d['stellar_ext_err'])
    print(f'Replacing {np.count_nonzero(idx)} NaN extinctions with 0+-inf.')
    d['stellar_ext'][idx] = 0.
    d['stellar_ext_err'][idx] = np.inf

    # Shuffle data, and put last X% into validation set
    rng = np.random.default_rng(seed=seed)
    n = len(d['gdr3_source_id'])
    shuffle_idx = np.arange(n, dtype='i8')
    rng.shuffle(shuffle_idx)

    n_val = int(np.ceil(n * validation_frac))
    d_train = {'shuffle_idx': shuffle_idx[:-n_val]}
    d_val = {'shuffle_idx': shuffle_idx[-n_val:]}

    # Copy data into training and validation sets
    for key in d:
        d_train[key] = d[key][d_train['shuffle_idx']]
        d_val[key] = d[key][d_val['shuffle_idx']]

    return d_train, d_val, sample_wavelengths

def down_sample_weighing(x_ini, all_x, bin_edges,
                         n_bins=100, max_upsampling=5.):
    # Use high-Extinction stars for empirical distribution of xi
    bin_edges = np.hstack([[-np.inf], bin_edges, [np.inf]])

    # Calculate the emperical distribution of x_ini under the given bins
    bin_indices = np.digitize(x_ini, bin_edges)
    counts = np.bincount(bin_indices, minlength=n_bins+3)
    weights = counts / counts.sum()  # convert counts to probabilities
    weights_per_bin = 1./(weights+0.05)

    weights_per_bin[0] = 0
    weights_per_bin[-1] = 0

    # Weigh all stars by the inverse of density of the ini sample
    bin_indices_all = np.digitize(all_x, bin_edges)
    weights_per_star = weights_per_bin[bin_indices_all]

    # Normalize the weights per star by median value
    weights_per_star /= (1e-5 + np.median(weights_per_star))

    weights_per_star = soft_clip_weights(weights_per_star, max_upsampling)

    return weights_per_star.astype('f4')

def plot_param_histograms_1d(stellar_type, weights, title, fname):
    n_params = stellar_type.shape[1]
    figsize = (3*n_params,3)

    fig,ax_arr = plt.subplots(
        1, n_params,
        figsize=figsize,
        layout='constrained'
    )

    for i,(ax,p) in enumerate(zip(ax_arr.flat,stellar_type.T)):
        p_range = (np.min(p), np.max(p))

        kw = dict(
            range=p_range, bins=100,
            log=True, density=True,
            histtype='step',
            alpha=0.8
        )
        ax.hist(p, label=r'$\mathrm{raw}$', **kw)
        ax.hist(p, label=r'$\mathrm{weighted}$', weights=weights, **kw)

        ax.xaxis.set_minor_locator(AutoMinorLocator())
        ax.grid(True, which='major', alpha=0.2)
        ax.grid(True, which='minor', alpha=0.05)

        ax.set_xlabel(rf'$\mathrm{{parameter\ {i}}}$')
        if i == 0:
            ax.legend()

    fig.suptitle(title)

    fig.savefig(fname)
    plt.close(fig)

def soft_clip_weights(weights, max_upsampling):
    return weights / (1 + weights/max_upsampling)

def weigh_prior(stellar_type_prior, d_train, scale=1., max_upsampling=5.):
    all_ln_prior = batch_apply_tf(
        stellar_type_prior.ln_prob,
        1024,
        d_train['stellar_type'],
        function=True,
        progress=True,
        numpy=True
    )
    all_prior = np.exp(scale*all_ln_prior)
    all_prior /= np.max(all_prior)
    weight_per_star = 1 / (all_prior + 1e-9)

    # Weigh stars for better representation
    weight_per_star = soft_clip_weights(weight_per_star, max_upsampling)

    return weight_per_star.astype('f4')

def train(data_fname, output_dir, stage=0, thin=1, E_low=0.1, n_epochs=512):
    '''
    Stage 0: Train the stellar model, using universal extinction curve.
    Stage 1: Train the extinction model with hqlE stars, using initial guess of
                    the slope of extinction curve.
    Stage 2: Train both extinction model and stellar model with hqlE stars
    Stage 3: Self-cleaning & Further optimize the model
    '''

    # General training parameters
    batch_size = 512
    n_bins = 100

    loss_hist = []

    # Ensure that output directories exist
    path_list = [
        'data', 'plots',
        'models/prior', 'models/flux',
        'index', 'hist_loss'
    ]
    for p in path_list:
        Path(os.path.join(output_dir,p)).mkdir(exist_ok=True, parents=True)

    # Helper function: save an h5 file, appending a base directory to filename
    def full_fn(fn):
        return os.path.join(output_dir, fn)

    if stage == 0:
        # Stage 0, begin without initial stellar model
        print(f'Loading training data from {data_fname} ...')
        d_train, d_val, sample_wavelengths = load_training_data(
            data_fname,
            thin=thin,
            seed=0,
        )
        n_train, n_val = [len(d['plx']) for d in (d_train,d_val)]
        print(f'Loaded {n_train} ({n_val}) training (validation) sources.')

        plot_training_data(d_train, output_dir=output_dir)

        # Initial guess of xi
        d_train['xi'] = np.zeros(n_train, dtype='f4')
        d_val['xi'] = np.zeros(n_val, dtype='f4')

        save_as_h5(d_val, full_fn('data/d_val.h5'))
        save_as_h5(d_train, full_fn('data/d_train.h5'))

        # Generate GMM prior
        fn_prior = full_fn('models/prior/gmm_prior')
        if os.path.exists(fn_prior+'-1.index'):
            print('Loading existing Gaussian Mixture Model prior ...')
            stellar_type_prior = GaussianMixtureModel.load(fn_prior+'-1')
        else:
            print('Generating Gaussian Mixture Model prior on stellar type ...')
            stellar_type_prior = GaussianMixtureModel(3, n_components=16)
            # Use at most this many stars to learn GMM prior
            n_prior_max = 1024*256
            stellar_type_prior.fit(d_train['stellar_type'][:n_prior_max])
            stellar_type_prior.save(fn_prior)
            print('  -> Plotting prior ...')
            plot_gmm_prior(stellar_type_prior, base_path=output_dir)

        weights_per_star = weigh_prior(stellar_type_prior, d_train)

        print('Plotting stellar-type histograms ...')
        plot_param_histograms_1d(
            d_train['stellar_type'],
            weights_per_star,
            r'$\mathrm{Training\ distribution:\ stellar\ type}$',
            os.path.join(output_dir, 'plots/training_stellar_type_hist1d')
        )

        # Generate tracks through stellar parameter space
        tracks_fn = full_fn('models/prior/tracks.h5')
        if os.path.exists(tracks_fn):
            print('Loading tracks through stellar parameter space ...')
            atm_tracks = model.load_stellar_type_tracks(tracks_fn)
        else:
            print('Generating tracks through stellar parameter space ...')
            atm_tracks = model.calculate_stellar_type_tracks(stellar_type_prior)
            model.save_stellar_type_tracks(atm_tracks, tracks_fn)
        print('Plotting tracks through stellar parameter space ...')
        for track in atm_tracks:
            plot_gmm_prior(
                stellar_type_prior,
                base_path=output_dir,
                overlay_track=track
            )

        # Initialize the parameter estimates at their measured (input) values
        for key in ('stellar_type', 'xi', 'stellar_ext', 'plx'):
            d_train[f'{key}_est'] = d_train[key].copy()
            d_val[f'{key}_est'] = d_train[key].copy()

        print('Creating flux model ...')
        n_stellar_params = d_train['stellar_type'].shape[1]
        p_low,p_high = np.percentile(
            d_train['stellar_type'],
            [16.,84.],
            axis=0
        )

        print('Training flux model on high-quality data ...')
        # Select a subset of "high-quality" stars with good measurements
        idx_hq = np.where(
            (d_train['plx']/d_train['plx_err'] > 10.)
          & np.all(d_train['stellar_type_err']<0.2, axis=1) # Type uncertainty
          & (d_train['stellar_ext_err'] < 0.1)
        )[0]
        n_hq = len(idx_hq)
        pct_hq = n_hq / d_train['plx'].shape[0]
        print(f'Training on {n_hq} ({pct_hq:.3%}) high-quality stars ...')

        stellar_model = FluxModel(
            sample_wavelengths, n_input=n_stellar_params,
            input_zp=np.median(d_train['stellar_type'],axis=0),
            input_scale=0.5*(p_high-p_low),
            hidden_size=32,
            l2=5., l2_ext_curve=0.5
        )

        # First, train the model with stars with good measurements,
        # with fixed slope of ext_curve
        title = (
            r'$\mathrm{Training\ distribution'
            r'\ (step\ 0a:\ HQ):'
            r'\ stellar\ type}$'
        )
        plot_param_histograms_1d(
            d_train['stellar_type'][idx_hq],
            weights_per_star[idx_hq],
            title,
            full_fn('plots/training_stellar_type_hist1d_step0a')
        )
        ret = train_stellar_model(
            stellar_model,
            d_train,
            weights_per_star,
            idx_train=idx_hq,
            optimize_stellar_model=True,
            optimize_stellar_params=False,
            lr_model_init=1e-4,
            batch_size=batch_size,
            n_epochs=n_epochs,
            model_update=['stellar_model','ext_curve_b'],
        )
        save_as_h5(ret, full_fn('hist_loss/hist_step0a.h5'))
        loss_hist.append(ret)
        stellar_model.save(full_fn('models/flux/xp_spectrum_model_initial'))

        # Plot stellar model along tracks through parameter space
        print('Plotting stellar model ...')
        for i,track in enumerate(atm_tracks):
            fig,ax = model.plot_stellar_model(stellar_model, track)
            fig.savefig(full_fn(f'plots/stellar_model_step0a_track{i}'))
            plt.close(fig)

        fig,_ = plot_roughness(d_train['stellar_type'], stellar_model)
        fig.savefig(full_fn('plots/roughness_step0a'))
        plt.close(fig)

        fig,_ = model.plot_extinction_curve(stellar_model, show_variation=False)
        fig.savefig(full_fn('plots/extinction_curve_step0a'))
        plt.close(fig)

        # Next, simultaneously train the stellar model and update stellar
        # parameters, using only the HQ data
        print('Training flux model and optimizing high-quality stars ...')
        ret = train_stellar_model(
            stellar_model,
            d_train,
            weights_per_star,
            idx_train=idx_hq,
            optimize_stellar_model=True,
            optimize_stellar_params=True,
            lr_model_init=1e-5*nscale,
            lr_stars_init=1e-4*nscale,
            batch_size=batch_size,
            n_epochs=n_epochs,
            model_update=['stellar_model','ext_curve_b'],
            var_update = ['atm','E','plx'],
        )
        save_as_h5(ret, full_fn('hist_loss/hist_step0b.h5'))
        loss_hist.append(ret)
        #plot_loss(ret, suffix='_intermediate')
        stellar_model.save(
            full_fn('models/flux/xp_spectrum_model_intermediate')
        )

        print('Plotting stellar model ...')
        for i,track in enumerate(atm_tracks):
            fig,ax = model.plot_stellar_model(stellar_model, track)
            fig.savefig(full_fn(f'plots/stellar_model_step0b_track{i}'))
            plt.close(fig)

        fig,_ = plot_roughness(d_train['stellar_type'], stellar_model)
        fig.savefig(full_fn('plots/roughness_step0b'))
        plt.close(fig)

        fig,_ = model.plot_extinction_curve(stellar_model, show_variation=False)
        fig.savefig(full_fn('plots/extinction_curve_step0b'))
        plt.close(fig)

        stellar_model = FluxModel.load(
            full_fn('models/flux/xp_spectrum_model_intermediate-1')
        )

        # Update E for stars with large E uncertainties (often, +-inf)
        print('Optimizing uncertain E estimates ...')
        idx_large_ext_err = np.where(d_train['stellar_ext_err'] > 0.1)[0]
        ret = train_stellar_model(
            stellar_model,
            d_train,
            weights_per_star,
            idx_train=idx_large_ext_err,
            optimize_stellar_model=False,
            optimize_stellar_params=True,
            batch_size=batch_size,
            lr_stars_init=1e-2,
            n_epochs=32*n_epochs,
            var_update=['E'],
        )

        # Next, update parameters of all the stars, holding the model fixed
        print('Optimizing all stars ...')
        ret = train_stellar_model(
            stellar_model,
            d_train,
            weights_per_star,
            optimize_stellar_model=False,
            optimize_stellar_params=True,
            batch_size=batch_size,
            n_epochs=n_epochs,
            var_update=['atm','E','plx'],
        )
        loss_hist.append(ret)

        # Self-cleaning: Identify outlier stars to exclude from further training,
        # using distance from priors
        idx_params_good = identify_outlier_stars(d_train)
        pct_good = np.mean(idx_params_good)
        print(f'Parameter outliers: {1-pct_good:.3%} of sources.')

        idx_flux_good = identify_flux_outliers(
            d_train, stellar_model,
            chi2_dof_clip=10.,
            #chi_indiv_clip=20.
        )

        pct_good = np.mean(idx_flux_good)
        print(f'Flux outliers: {1-pct_good:.3%} of sources.')

        idx_good = idx_params_good & idx_flux_good
        pct_good = np.mean(idx_good)
        print(f'Combined outliers: {1-pct_good:.3%} of sources.')

        title = (
            r'$\mathrm{Training\ distribution'
            r'\ (step\ 0c:\ cut\ flux/param\ outliers):'
            r'\ stellar\ type}$'
        )
        plot_param_histograms_1d(
            d_train['stellar_type'][idx_good],
            weights_per_star[idx_good],
            title,
            full_fn('plots/training_stellar_type_hist1d_step0c')
        )

        # Finally, simultaneously train the stellar model and update stellar
        # parameters, using all the (non-outlier) data
        print('Training flux model and optimizing all non-outlier stars ...')
        ret = train_stellar_model(
            stellar_model,
            d_train,
            weights_per_star,
            idx_train=np.where(idx_good)[0],
            optimize_stellar_model=True,
            optimize_stellar_params=True,
            lr_model_init=1e-5*nscale,
            lr_stars_init=1e-5*nscale,
            batch_size=batch_size,
            n_epochs=n_epochs,
            model_update=['stellar_model','ext_curve_b'],
            var_update = ['atm','E','plx'],
        )
        save_as_h5(ret, full_fn('hist_loss/hist_step0c.h5'))
        loss_hist.append(ret)

        # Plot stellar model
        print('Plotting stellar model ...')
        for i,track in enumerate(atm_tracks):
            fig,ax = model.plot_stellar_model(stellar_model, track)
            fig.savefig(full_fn(f'plots/stellar_model_step0c_track{i}'))
            plt.close(fig)

        fig,_ = plot_roughness(d_train['stellar_type'], stellar_model)
        fig.savefig(full_fn('plots/roughness_step0c'))
        plt.close(fig)

        fig,_ = model.plot_extinction_curve(stellar_model, show_variation=False)
        fig.savefig(full_fn('plots/extinction_curve_step0c'))
        plt.close(fig)

        np.save(full_fn('index/idx_good_wo_Rv.npy'), idx_good)
        stellar_model.save(full_fn('models/flux/xp_spectrum_model_final'))
        save_as_h5(d_train, full_fn('data/dtrain_final_wo_Rv.h5'))
        #save_as_h5(ret, full_fn('hist_loss/final_wo_Rv.h5'))

    if stage<2:

        d_val = load_h5(full_fn('data/d_val.h5'))
        d_train = load_h5(full_fn('data/dtrain_final_wo_Rv.h5'))
        atm_tracks = model.load_stellar_type_tracks(
            full_fn('models/prior/tracks.h5')
        )

        stellar_model = FluxModel.load(
            full_fn('models/flux/xp_spectrum_model_final-1')
        )

        print('Loading Gaussian Mixture Model prior on stellar type ...')
        stellar_type_prior = GaussianMixtureModel.load(
            full_fn('models/prior/gmm_prior-1')
        )

        # Initial weight of stars: equal
        weights_per_star = np.ones(len(d_train['plx']), dtype='f4')
        idx_hq = np.load(full_fn('index/idx_good_wo_Rv.npy'))

        # Run more steps in this stage
        n_epochs_1 = 256

        # Optimize the params of high-quality stars
        print('Optimizing params of hq stars ...')
        ret = train_stellar_model(
            stellar_model,
            d_train,
            weights_per_star,
            idx_train=np.where(idx_hq)[0],
            optimize_stellar_model=False,
            optimize_stellar_params=True,
            #lr_model_init=1e-7,
            #lr_stars_init=1e-5,
            batch_size=batch_size,
            n_epochs=n_epochs,
            var_update=['atm','E','plx'],
        )
        save_as_h5(ret, full_fn('hist_loss/hist_step1a.h5'))

        save_as_h5(d_train, full_fn('data/dtrain_final_wo_Rv_optimized.h5'))

        d_train = load_h5(full_fn('data/dtrain_final_wo_Rv_optimized.h5'))

        idx_large_E = (d_train['stellar_ext_est']>E_low)
        idx_hq_large_E = idx_hq & idx_large_E
        pct_use = np.mean(idx_hq_large_E)

        title = (
            r'$\mathrm{Training\ distribution'
            r'\ (step\ 1a:\ HQ,\ large\ E):'
            r'\ stellar\ type}$'
        )
        plot_param_histograms_1d(
            d_train['stellar_type'][idx_hq_large_E],
            weights_per_star[idx_hq_large_E],
            title,
            full_fn('plots/training_stellar_type_hist1d_step1a')
        )

        print(f'Learning (xi,E,plx) for {pct_use:.3%} of sources (high E) ...')
        ret = train_stellar_model(
            stellar_model,
            d_train,
            weights_per_star,
            idx_train=np.where(idx_hq_large_E)[0],
            optimize_stellar_model=False,
            optimize_stellar_params=True,
            #lr_model_init=1e-7,
            #lr_stars_init=1e-5,
            batch_size=batch_size,
            n_epochs=n_epochs,
            var_update=['E','plx','xi'],
        )

        save_as_h5(d_train, full_fn('data/dtrain_initial_guess_xi.h5'))

        d_train = load_h5(full_fn('data/dtrain_initial_guess_xi.h5'))

        weights_per_star = down_sample_weighing(
            d_train['xi_est'][idx_hq_large_E],
            d_train['xi_est'],
            bin_edges = np.linspace(-0.8, 0.8, n_bins+1)
        )
        weights_per_star *= weigh_prior(stellar_type_prior, d_train)
        weights_per_star = soft_clip_weights(weights_per_star, 10.)

        title = (
            r'$\mathrm{Training\ distribution'
            r'\ (step\ 1b:\ HQ,\ large\ E):'
            r'\ stellar\ type}$'
        )
        plot_param_histograms_1d(
            d_train['stellar_type'][idx_hq_large_E],
            weights_per_star[idx_hq_large_E],
            title,
            full_fn('plots/training_stellar_type_hist1d_step1b')
        )

        print(f'Learning (xi,E) for {pct_use:.3%} of sources (high E)')
        print('... in addition to mean extinction curve ...')
        ret = train_stellar_model(
            stellar_model,
            d_train,
            weights_per_star,
            idx_train=np.where(idx_hq_large_E)[0],
            optimize_stellar_model=True,
            optimize_stellar_params=True,
            lr_model_init=1e-5,
            lr_stars_init=1e-5,
            batch_size=batch_size,
            n_epochs=n_epochs,
            var_update=['E','xi'],
            model_update=['ext_curve_b'],
        )
        save_as_h5(ret, full_fn('hist_loss/hist_step1b.h5'))

        fig,_ = model.plot_extinction_curve(stellar_model, show_variation=True)
        fig.savefig(full_fn('plots/extinction_curve_step1b'))
        plt.close(fig)

        stellar_model.save(full_fn('models/flux/xp_spectrum_model_initial_update_b'))

        stellar_model = FluxModel.load(
            full_fn('models/flux/xp_spectrum_model_initial_update_b-1')
        )

        print(f'Learning (xi,E) for {pct_use:.3%} of sources (high E)')
        print('... in addition to extinction curve variation ...')
        ret = train_stellar_model(
            stellar_model,
            d_train,
            weights_per_star,
            idx_train=np.where(idx_hq_large_E)[0],
            optimize_stellar_model=True,
            optimize_stellar_params=True,
            #lr_model_init=5e-6,
            lr_stars_init=1e-5,
            batch_size=batch_size,
            n_epochs=n_epochs_1,
            var_update=['E','xi'],
            model_update=['ext_curve_w'],
        )
        save_as_h5(ret, full_fn('hist_loss/hist_step1c.h5'))

        fig,_ = model.plot_extinction_curve(stellar_model, show_variation=True)
        fig.savefig(full_fn('plots/extinction_curve_step1c'))
        plt.close(fig)

        np.save(full_fn('index/idx_hq.npy'), idx_hq)
        # Save initial guess of xi
        save_as_h5(d_train, full_fn('data/dtrain_Rv_initial.h5'))
        save_as_h5(ret, full_fn('hist_loss/Rv_initial.h5'))
        stellar_model.save(full_fn('models/flux/xp_spectrum_model_initial_Rv'))

    if stage<3:

        #n_epochs = 512

        stellar_model = FluxModel.load(
            full_fn('models/flux/xp_spectrum_model_initial_Rv-1')
        )
        d_train = load_h5(full_fn('data/dtrain_Rv_initial.h5'))
        d_val = load_h5(full_fn('data/d_val.h5'))

        idx_hq = np.load(full_fn('index/idx_hq.npy'))
        idx_large_E = (d_train['stellar_ext_est']>E_low)
        idx_hq_large_E = idx_hq & idx_large_E
        pct_use = np.mean(idx_hq_large_E)

        stellar_type_prior = GaussianMixtureModel.load(
            full_fn('models/prior/gmm_prior-1')
        )
        atm_tracks = model.calculate_stellar_type_tracks(stellar_type_prior)

        weights_per_star = down_sample_weighing(
            d_train['xi_est'][idx_hq_large_E],
            d_train['xi_est'],
            bin_edges = np.linspace(-0.8, 0.8, n_bins+1)
        )
        weights_per_star *= weigh_prior(stellar_type_prior, d_train)
        weights_per_star = soft_clip_weights(weights_per_star, 10.)

        print(f'Learning everything for {pct_use:.3%} of sources (high E) ...')
        ret = train_stellar_model(
            stellar_model,
            d_train,
            weights_per_star,
            idx_train=np.where(idx_hq_large_E)[0],
            optimize_stellar_model=True,
            optimize_stellar_params=True,
            lr_model_init=1e-5,
            lr_stars_init=1e-4,
            batch_size=batch_size,
            n_epochs=2*n_epochs,
            var_update=['atm','E','plx','xi'],
            model_update=['stellar_model','ext_curve_w','ext_curve_b'],
        )
        save_as_h5(ret, full_fn('hist_loss/hist_step2.h5'))

        print('Plotting stellar model ...')
        for i,track in enumerate(atm_tracks):
            fig,ax = model.plot_stellar_model(stellar_model, track)
            fig.savefig(full_fn(f'plots/stellar_model_step2_track{i}'))
            plt.close(fig)

        fig,_ = plot_roughness(d_train['stellar_type'], stellar_model)
        fig.savefig(full_fn('plots/roughness_step2'))
        plt.close(fig)

        fig,_ = model.plot_extinction_curve(stellar_model, show_variation=True)
        fig.savefig(full_fn('plots/extinction_curve_step2'))
        plt.close(fig)

        fig,_ = model.plot_RV_histogram(stellar_model, d_train)
        fig.savefig(full_fn('plots/RV_histogram_step2'))
        plt.close(fig)

        #fig,_ = model.plot_RV_skymap(stellar_model, d_train)
        #fig.savefig(full_fn('plots/RV_skymap_step2'))
        #plt.close(fig)

        save_as_h5(d_train, full_fn('data/dtrain_Rv_intermediate_0.h5'))
        save_as_h5(ret, full_fn('hist_loss/Rv_intermediate_0.h5'))
        stellar_model.save(
            full_fn('models/flux/xp_spectrum_model_intermediate_Rv')
        )

    if stage<4:

        stellar_model = FluxModel.load(
            full_fn('models/flux/xp_spectrum_model_intermediate_Rv-1')
        )
        stellar_type_prior = GaussianMixtureModel.load(
            full_fn('models/prior/gmm_prior-1')
        )

        d_train = load_h5(full_fn('data/dtrain_Rv_intermediate_0.h5'))
        d_val = load_h5(full_fn('data/d_val.h5'))
        atm_tracks = model.load_stellar_type_tracks(
            full_fn('models/prior/tracks.h5')
        )
        #idx_hq = np.load(full_fn('index/idx_hq.npy'))

        # Optimize all stellar params, in order to pick up
        # stars that were rejected due to extinction variation law

        weights_per_star = np.ones(d_train['stellar_type'].shape[0]).astype('float32')

        print('Learning stellar parameters for all sources ...')
        ret = train_stellar_model(
            stellar_model,
            d_train,
            weights_per_star,
            #idx_train=idx_hq_large_E,
            optimize_stellar_model=False,
            optimize_stellar_params=True,
            batch_size=batch_size,
            n_epochs=n_epochs//2,
            var_update = ['atm','E','plx','xi'],
        )

        save_as_h5(d_train, full_fn('data/dtrain_Rv_intermediate_1.h5'))
        save_as_h5(ret, full_fn('hist_loss/Rv_intermediate_1.h5'))

        d_train = load_h5(full_fn('data/dtrain_Rv_intermediate_1.h5'))
        d_val = load_h5(full_fn('data/d_val.h5'))

        # Remove outliers
        idx_params_good = identify_outlier_stars( # TODO: rename to "identify_param_outliers"
            d_train,
            sigma_clip_teff=4., # TODO: Make this clipping be agnostic about the parameter names
            sigma_clip_logg=4.,
            sigma_clip_feh=4.,
        )
        pct_good = np.mean(idx_params_good)
        print(f'Parameter outliers: {1-pct_good:.3%} of sources.')

        idx_flux_good = identify_flux_outliers(
            d_train, stellar_model,
            chi2_dof_clip=10.,
            #chi_indiv_clip=20.
        )
        pct_good = np.mean(idx_flux_good)
        print(f'Flux outliers: {1-pct_good:.3%} of sources.')

        idx_good = idx_params_good & idx_flux_good
        pct_good = np.mean(idx_good)
        print(f'Combined outliers: {1-pct_good:.3%} of sources.')

        idx_final_train = idx_good #& (d_train['stellar_ext_est']>0.1)
        pct_use = np.mean(idx_final_train)
        print(f'Training on {pct_use:.3%} of sources.')

        weights_per_star = down_sample_weighing(
            d_train['xi_est'][idx_hq_large_E],
            d_train['xi_est'],
            bin_edges = np.linspace(-0.8, 0.8, n_bins+1)
        )
        weights_per_star *= weigh_prior(stellar_type_prior, d_train)
        weights_per_star = soft_clip_weights(weights_per_star, 10.)

        title = (
            r'$\mathrm{Training\ distribution'
            r'\ (step\ 3a:\ cut\ flux/param\ outliers):'
            r'\ stellar\ type}$'
        )
        plot_param_histograms_1d(
            d_train['stellar_type'][idx_final_train],
            weights_per_star[idx_final_train],
            title,
            full_fn('plots/training_stellar_type_hist1d_step3a')
        )

        pct_use = np.mean(idx_final_train)
        print('Learning everything except extinction curve variation')
        print(f'using {pct_use:.3%} of sources (cut flux/param outliers) ...')
        ret = train_stellar_model(
            stellar_model,
            d_train,
            np.ones_like(weights_per_star, dtype='f4'),
            idx_train=np.where(idx_final_train)[0],
            optimize_stellar_model=True,
            optimize_stellar_params=True,
            lr_model_init=1e-5,
            lr_stars_init=1e-4,
            batch_size=batch_size,
            n_epochs=n_epochs//2,
            # Freeze extinction curve variation when training with
            # all stars (not necessarily having high E)
            var_update=['atm','E','plx','xi'],
            model_update=['stellar_model','ext_curve_b'],
        )

        stellar_model.save(full_fn('models/flux/xp_spectrum_model_final_Rv'))
        save_as_h5(ret, full_fn('hist_loss/final_Rv.h5'))

        fig,ax = model.plot_stellar_model(stellar_model, track)
        fig.savefig(full_fn(f'plots/stellar_model_step3_track{i}'))
        plt.close(fig)

        fig,_ = plot_roughness(d_train['stellar_type'], stellar_model)
        fig.savefig(full_fn('plots/roughness_step3'))
        plt.close(fig)

        fig,_ = model.plot_extinction_curve(stellar_model, show_variation=True)
        fig.savefig(full_fn('plots/extinction_curve_step3'))
        plt.close(fig)

        fig,_ = model.plot_RV_histogram(stellar_model, d_train)
        fig.savefig(full_fn('plots/RV_histogram_step3'))
        plt.close(fig)

        fig,_ = model.plot_RV_skymap(stellar_model, d_train)
        fig.savefig(full_fn('plots/RV_skymap_step3'))
        plt.close(fig)

        stellar_model.save(full_fn('models/flux/xp_spectrum_model_final_Rv'))
        save_as_h5(ret, full_fn('hist_loss/final_Rv.h5'))

        print('Optimizing stellar parameters in training set ...')
        ret = train_stellar_model(
            stellar_model,
            d_train,
            weights_per_star,
            #idx_train=np.where(idx_final_train)[0],
            optimize_stellar_model=False,
            optimize_stellar_params=True,
            batch_size=batch_size,
            n_epochs=n_epochs,
            var_update = ['atm','E','plx','xi'],
        )

        fig,_ = model.plot_RV_histogram(stellar_model, d_train)
        fig.savefig(full_fn('plots/RV_histogram_final'))
        plt.close(fig)

        fig,_ = model.plot_RV_skymap(stellar_model, d_train)
        fig.savefig(full_fn('plots/RV_skymap_final'))
        plt.close(fig)

        for key in ('stellar_type', 'xi', 'stellar_ext', 'plx'):
            d_val[f'{key}_est'] = d_val[key].copy()

        print('Optimizing stellar parameters in validation set ...')
        ret = train_stellar_model(
            stellar_model,
            d_val,
            weights_per_star,
            #idx_train=np.where(idx_final_train)[0],
            optimize_stellar_model=False,
            optimize_stellar_params=True,
            batch_size=batch_size,
            n_epochs=n_epochs,
            var_update = ['atm','E','plx','xi'],
        )

        save_as_h5(d_train, full_fn('data/dtrain_Rv_final.h5'))
        save_as_h5(d_val, full_fn('data/dval_Rv_final.h5'))

        return 0

def plot_roughness(theta, stellar_model):
    roughness_plot = batch_apply_tf(
        lambda x: stellar_model.roughness(tf.convert_to_tensor(x),
                                          reduce_stars=False),
        1024, theta,
        function=True,
        numpy=True,
        progress=False
    )
    fig,ax_arr = plot_utils.projection_grid(
        theta,
        c=roughness_plot,
        labels=(r'$T_{\mathrm{eff}} / (1000\,\mathrm{K})$',
                r'$\left[\mathrm{Fe/H}\right] / \mathrm{dex}$',
                r'$\log g / \mathrm{dex}$'),
        extents=[(13.,3.),(-2.7,1.0),(5.2,-0.3)],
        scatter=False,
        clabel=(r'$\langle\left| '
                r'\nabla_{\!\theta}\, f_{\lambda}\, '
                r'\right|^2\rangle$'),
        hist_kw=dict(bins=100),
        #fig_kw=dict(dpi=200),
        norm=Normalize(0,5)
    )
    for ax in ax_arr.flat:
        ax.xaxis.set_minor_locator(AutoMinorLocator())
        ax.yaxis.set_minor_locator(AutoMinorLocator())
    rough = np.std(roughness_plot)
    fig.suptitle(rf'$\mathrm{{roughness}} = {rough:.4f}$')
    fig.subplots_adjust(
        left=0.14, right=0.97,
        bottom=0.12, top=0.92,
        hspace=0.07, wspace=0.07
    )
    return fig,ax_arr

def plot_camd(d, color_key=None, color_dim=None, low_extinction=False):
    fig,ax = plt.subplots(figsize=(6,6), layout='constrained')

    # Calculate M_G and BP-RP
    zp_g,zp_bp,zp_rp = (25.6884, 25.3514, 24.7619)
    m_g = -2.5*np.log10(d['phot_g_mean_flux']) + zp_g
    m_bp = -2.5*np.log10(d['phot_bp_mean_flux']) + zp_bp
    m_rp = -2.5*np.log10(d['phot_rp_mean_flux']) + zp_rp
    bprp = m_bp - m_rp
    g_absmag = m_g + 5*np.log10(d['plx']) - 10.

    # Only plot stars with well determined Gaia parallaxes
    idx = (d['plx'] / d['plx_err'] > 3.)

    # Additionally require decent photometry
    for b in ('g','bp','rp'):
        idx &= (d[f'phot_{b}_mean_flux']/d[f'phot_{b}_mean_flux_error']>5)

    # Select low-extinction stars?
    if low_extinction:
        idx &= (d['stellar_ext'] + d['stellar_ext_err'] < 0.3)

    bprp = bprp[idx]
    g_absmag = g_absmag[idx]

    bprp_lim = (-1.0, 5.0)
    g_absmag_lim = (15., -5.5)

    if color_key is None:
        _,_,_,im = ax.hist2d(
            bprp, g_absmag,
            range=[sorted(bprp_lim),sorted(g_absmag_lim)],
            bins=50,
            norm=LogNorm()
        )
        ax.set_xlim(bprp_lim)
        ax.set_ylim(g_absmag_lim)
        clabel = r'$\mathrm{density}$'
    else:
        c = d[color_key][idx]

        if color_dim is not None:
            c = c[:,color_dim]
        clim = plot_utils.choose_lim(c)

        im = plot_utils.hist2d_reduce(
            bprp, g_absmag, c,
            ax=ax,
            xlim=bprp_lim,
            ylim=g_absmag_lim,
            hist_kw=dict(bins=50),
            imshow_kw=dict(vmin=clim[0],vmax=clim[1])
        )

        clabel = color_key.replace(r'_', r'\_')
        if color_dim is not None:
            clabel = rf'{clabel}\_{color_dim}'
        clabel = rf'$\mathtt{{{clabel}}}$'

    cb = fig.colorbar(im, ax=ax, label=clabel)
    ax.set_xlabel(r'$BP-RP$')
    ax.set_ylabel(r'$m_G + 5\log_{10}\hat{\varpi} - 10$')

    return fig,ax

def plot_training_data(d, output_dir=''):
    fig,ax = plot_camd(d)
    fn = os.path.join(output_dir, 'plots', f'camd_density')
    fig.savefig(fn)
    plt.close(fig)

    n_params = d['stellar_type'].shape[1]
    for key in ('stellar_type','stellar_type_err'):
        for dim in range(n_params):
            print(f'Plotting {key}_{dim} ...')
            fig,ax = plot_camd(
                d, color_key=key, color_dim=dim,
                low_extinction=True
            )
            fn = os.path.join(output_dir, 'plots', f'camd_{key}_{dim}')
            fig.savefig(fn)
            plt.close(fig)

    print(f'Plotting extinctions ...')
    fig,ax = plot_camd(d, color_key='stellar_ext')
    fn = os.path.join(output_dir, 'plots', 'camd_extinction')
    fig.savefig(fn)
    plt.close(fig)

def main():
    parser = ArgumentParser(
        description='Train forward model of XP spectra.',
        add_help=True
    )
    parser.add_argument(
        '-i', '--input',
        type=str,
        required=True,
        help='Filename of training data.'
    )
    parser.add_argument(
        '-o', '--output-dir',
        type=str,
        required=True,
        help='Directory in which to store output.'
    )
    parser.add_argument(
        '--stage',
        type=int,
        default=0,
        help='Stage at which to begin training.'
    )
    parser.add_argument(
        '--thin',
        type=int,
        default=1,
        help='Only use every Nth star in the training set.'
    )
    parser.add_argument(
        '--E_low',
        type=float,
        default=0.1,
        help='Lower limit of extinction for ext curve variation.'
    )

    args = parser.parse_args()
    train(args.input, args.output_dir, stage=args.stage, thin=args.thin, E_low=args.E_low)

    return 0

if __name__=='__main__':
    main()


# Module 2 as merchandiser, the user is a Merchandiser - able to edit, add, and remove items on the list

# This function does what it does
def categoryChooser(action):
    dairy = ["Fresh Milk", "NAN Gold", "Promil Gold"]
    dairyPrice = [150.0, 1590.0, 1480.0]

    wetGoods = ["Ground Beef", "Chicken Wings", "Pork Knuckles"]
    wetGoodsPrice = [450.0, 280.0, 350.0]

    print("\n||DAIRY - 1|\t|WET GOODS - 2||")
    category = int(input("\nEnter category number: "))

    if category == 1:
        if action == 1:
            editProduct(dairy,dairyPrice)

        elif action == 2:
            addProduct(dairy,dairyPrice)

        elif action == 3:
            remProduct(dairy,dairyPrice)

        else:
            action()

    elif category == 2:

        if action == 1:
            editProduct(wetGoods,wetGoodsPrice)

        elif action == 2:
            addProduct(wetGoods,wetGoodsPrice)

        elif action == 3:
            remProduct(wetGoods,wetGoodsPrice)

        else:
            action()

    else:
        print("\nInvalid input.")
        categoryChooser()

# This function lets you edit product in a list
def editProduct(category,price):

    print("\nCurrent inventory")
    num = 0

    while num != len(category):
        print("|", num+1 ,"|" + category[num] + ":\t\t₱", price[num])
        num += 1

    index = int(input("*Enter the number code to be changed: ")) - 1
    changeProduct = input("*Enter new product: ")
    changeProductPrice = float(input("*Enter price: "))

    category.pop(index)
    price.pop(index)
    category.insert(index,changeProduct)
    price.insert(index,changeProductPrice)

    num = 0
    print("\nCurrent inventory")
    while num != len(category):
        print("|", num+1 ,"|" + category[num] + ":\t\t₱", price[num])
        num += 1

    input("\n*Press any key to end the program.")
    print("\nProgram Ended")

# This function lets you add product in a list
def addProduct(category,price):

    newProduct = input("\n*Enter new product: ")
    newProductPrice = float(input("*Enter price: "))

    category.append(newProduct)
    price.append(newProductPrice)

    num = 0
    print("\nCurrent inventory")
    while num != len(category):
        print("|", num+1 ,"|" + category[num] + ":\t\t₱", price[num])
        num += 1

    input("\n*Press any key to end the program.")
    print("\nProgram Ended")

# This function lets you remove product in a list
def remProduct(category,price):

    print("\nCurrent inventory")
    num = 0

    while num != len(category):
        print("|", num+1 ,"|" + category[num] + ":\t\t₱", price[num])
        num += 1
    index = int(input("*Enter the number code to be removed: ")) - 1

    category.pop(index)
    price.pop(index)

    num = 0
    print("\nCurrent inventory")
    while num != len(category):
        print("|", num+1 ,"|" + category[num] + ":\t\t₱", price[num])
        num += 1

    input("\n*Press any key to end the program.")
    print("\nProgram Ended")

# This function lets you choose your action
def actions():
    print("\n||EDIT - 1|\t|ADD - 2|\t|REMOVE - 3||")
    action = int(input("*Enter inventory options: "))

    if action == 1 or 2 or 3:
        categoryChooser(action)

    else:
        print("\nInvalid input.")
        actions()

# START of the module
print("\nWelcome, master!")
actions()


import pyautogui as pag
import cv2

### 그 링크 코드랑 거의 똑같고 좌표 넣는 부분이랑 cv2.INTER_LINEAR 만 수정함

# 캡쳐할 영역의 왼쪽 상단 모서리와 오른쪽 하단 모서리의 좌표값을 미리구해둔다.

position = pag.position() #여기다가 HYE좌표 넣어주자
x=position.x
y=position.y
print(position)
#위에 구한값을 변수에 담아둔다
left_top = (x, y)
right_bottom = (x+100, y+100)

#좌측상단의 x, 우측하단의 y값을 저장해둔다
left_top_x = left_top[0]
right_bottom_y = right_bottom[1]

#캡쳐 범위의 폭과 높이를 구한다
capture_width = right_bottom[0]-left_top[0]
capture_height = right_bottom[1]-left_top[1]

#캡쳐 경로와 이름을 지정한다
path = "capture.png"
#스크린샷
pag.screenshot(path, region=(left_top_x, right_bottom_y, capture_width, capture_height))

'''# 원본 이미지
img_source = cv2.imread('capture.png')
cv2.imshow("original", img_source)

cv2.waitKey(0)'''

img_source = cv2.imread('capture.png')
# 2배 이미지
img_result = cv2.resize(img_source, None, fx=2, fy=2, interpolation = cv2.INTER_LINEAR)
cv2.imshow("x2", img_result)

cv2.waitKey(0)

'''# 4배 이미지
height, width = img_source.shape[:2]
img_result = cv2.resize(img_source, (4*width, 4*height), interpolation = cv2.INTER_LINEAR )
cv2.imshow("x4 INTER_LINEAR", img_result)

height, width = img_source.shape[:2]
img_result2 = cv2.resize(img_source, (4*width, 4*height), interpolation = cv2.INTER_CUBIC )
cv2.imshow("x4 INTER_CUBIC", img_result)

cv2.waitKey(0)

# INTER_CUBIC를 사용해 확대한 4배 이미지를 0.5배한 이미지
img_result = cv2.resize(img_result2, None, fx=0.5, fy=0.5, interpolation = cv2.INTER_AREA)
cv2.imshow("x0.5 INTER_AREA", img_result)

img_result = cv2.resize(img_result2, None, fx=0.5, fy=0.5) # cv2.INTER_LINEAR
cv2.imshow("x0.5 INTER_LINEAR", img_result)

cv2.waitKey(0)
'''

cv2.destroyAllWindows()


from typing import List, Tuple

def product_sum_count(bound: float,
                      minimal: int=2) -> List[Tuple[int, int, int]]:
    for i in range(minimal, int(bound) + 1):
        current = (i, i, 1)
        yield current

        for p, s, c in product_sum_count(bound / i, minimal):

            yield (p * i, s + i, c + 1)

def solve(bound: int=12_000) -> int:
    minimal_for_k = [0 for k in range(bound + 1)]

    for p, s, c in product_sum_count(2 * bound):
        k = p - s + c

        if k <= 1 or k > bound:
            continue

        current = minimal_for_k[k]

        if current == 0 or p < current:
            minimal_for_k[k] = p

    return sum(set(minimal_for_k))


import telebot
import random
from token_ import token_
from message_text import *
from telebot.types import InlineKeyboardButton, InlineKeyboardMarkup

main_list = list(list_values)
bot = telebot.TeleBot(token_, parse_mode=None)
main_dict = None
dict_values = {i: 0 for i in list_values}
gen = None

def generator(l):
    global main_dict
    tmp = list(l)
    random.shuffle(tmp)
    main_dict = {i: tmp[i] for i in range(len(tmp))}
    for i in range(0, len(tmp[:int(len(tmp)/3) * 3]), 3):
        yield i, i+1, i+2

def generate_simple_keyboard(v1, v2):
    btn_1 = InlineKeyboardButton(main_dict[int(v1)], callback_data=f'{v1}_{v2}_0')
    btn_2 = InlineKeyboardButton(main_dict[int(v2)], callback_data=f'{v2}_{v1}_0')
    keyboard = InlineKeyboardMarkup().add(btn_1, btn_2)
    return keyboard

def generate_main_key_board(gen):
    value_1, value_2, value_3 = next(gen)
    # print(value_1, value_2, value_3)
    btn_1 = InlineKeyboardButton(main_dict[int(value_1)], callback_data=f'{value_1}_{value_2}_{value_3}_1')
    btn_2 = InlineKeyboardButton(main_dict[int(value_2)], callback_data=f'{value_2}_{value_1}_{value_3}_1')
    btn_3 = InlineKeyboardButton(main_dict[int(value_3)], callback_data=f'{value_3}_{value_1}_{value_2}_1')
    keyboard = InlineKeyboardMarkup().add(btn_1, btn_2, btn_3)
    return keyboard

def generate_after_rez_keyboard():
    btn_1 = InlineKeyboardButton('Пройти заново', callback_data='again')
    btn_2 = InlineKeyboardButton('Уточнити результат', callback_data=f'cont')
    keyboard = InlineKeyboardMarkup().add(btn_1, btn_2)
    return keyboard

def send_results(chat_id):
    results = 'РЕЗУЛЬТАТ:\n'
    list_d = list(dict_values.items())
    list_d.sort(key=lambda x: x[1], reverse=True)
    for i in list_d:
        l = f'{i[1]} - {i[0]}\n'
        results += l
    bot.send_message(chat_id, results, reply_markup=generate_after_rez_keyboard())

@bot.message_handler(commands=['start'])
def start(message):
    global gen
    gen = generator(main_list)
    bot.send_message(message.chat.id, start_message_2,
                           reply_markup=generate_main_key_board(gen))

def start_again(call):
    global gen
    global main_list
    main_list = list(list_values)
    gen = generator(list_values)
    bot.edit_message_reply_markup(call.message.chat.id, call.message.message_id,
                                  reply_markup=generate_main_key_board(gen))

@bot.callback_query_handler(func=lambda x: True)
def callback_query(call):
    value = call.data.split('_')[-1]
    global main_dict
    global gen
    global dict_values
    print(call.data)
    if call.data == 'again':
        dict_values = {i: 0 for i in list_values}
        start_again(call)
    elif call.data == 'cont':
        start_again(call)

    elif value == '1':
        v1, v2, v3, _ = tuple(str(call.data).split('_'))

        dict_values[main_dict[int(v1)]] += 1
        bot.edit_message_reply_markup(call.message.chat.id, call.message.message_id,
                                      reply_markup=generate_simple_keyboard(v2, v3))
    else:
        v1, v2, _ = tuple(str(call.data).split('_'))
        dict_values[main_dict[int(v1)]] += 1
        main_list.pop(main_list.index(main_dict[int(v2)]))

        try:
            bot.edit_message_reply_markup(call.message.chat.id, call.message.message_id,
                                          reply_markup=generate_main_key_board(gen))
        except StopIteration:
            try:
                gen = generator(main_list)
                bot.edit_message_reply_markup(call.message.chat.id, call.message.message_id,
                                              reply_markup=generate_main_key_board(gen))
            except StopIteration:
                bot.edit_message_reply_markup(call.message.chat.id, call.message.message_id)
                send_results(call.message.chat.id)

bot.polling()


import socket
import socketserver
import time

class Socket_function():
    def sendeStr(self, komm_s, datenStr):
        datenBytes = bytes(datenStr, 'utf-8')
        komm_s.sendall(datenBytes)
        self.sendeTrennByte(komm_s)

    def sendeTrennByte(self, komm_s):
        trennByte = bytes([0])
        komm_s.sendall(trennByte)

    def empfangeStr(self, komm_s):

        weiter = True
        datenBytes = bytes()

        trennByte = bytes([0])

        while weiter:
            chunk = komm_s.recv(1)
            if chunk == trennByte or chunk == bytes([]):
                weiter = False
            else:
                datenBytes = datenBytes + chunk

        datenStr = str(datenBytes, 'utf-8')

        return datenStr


# Conversion de pesos mexicanos a dólares

menu = """

Bienvenido al Banco Nacional de México
La siguiente aplicación te permite convertir
a dólares las siguientes monedas:

1. Pesos mexicanos
2. Pesos argentinos
3. Pesos colombianos

Elige una opción: """

opcion = int(input(menu))

if opcion == 1:
    pesos = float(input("¿Cuántos pesos mexicanos tienes?: "))
    valor_dolar = 20
    dolares = pesos/valor_dolar
    print("Tienes " + str(dolares) + " dolares")

elif opcion == 2:
    pesos = float(input("¿Cuántos pesos argentinos tienes?: "))
    valor_dolar = 60
    dolares = pesos/valor_dolar
    print("Tienes " + str(dolares) + " dolares")

elif opcion == 3:
    pesos = float(input("¿Cuántos pesos colombianos tienes?: "))
    valor_dolar = 120
    dolares = pesos/valor_dolar
    print("Tienes " + str(dolares) + " dolares")

else:
    print("Ingresa una opción corecta")


import matplotlib
import numpy as np
from sklearn.decomposition import PCA
from sklearn.preprocessing import MinMaxScaler

from classifier.dataset import ProcessedDataset
from input.regressors import filter_single_class
from input.utils import resample

matplotlib.use('Agg')
import matplotlib.pyplot as plt

source_dir = '/data/user/teodoro/uniprot/dataset/pub_data/processed/tag'
train_set = ProcessedDataset(source_dir, type='train').get_content()
dev_set = ProcessedDataset(source_dir, type='dev').get_content()
test_set = ProcessedDataset(source_dir, type='test').get_content()

# _, train_targets, train_regressors = filter_single_class(train_set[0], train_set[1], train_set[2])
# _, dev_targets, dev_regressors = filter_single_class(dev_set[0], dev_set[1], dev_set[2])
# _, test_targets, test_regressors = filter_single_class(test_set[0], test_set[1], test_set[2])

#train_targets, train_regressors = resample(train_regressors, train_targets)

train_regressors = train_set[2]
dev_regressors = dev_set[2]
test_regressors = test_set[2]

scaler = MinMaxScaler()
train_regressors = scaler.fit_transform(train_regressors)
dev_regressors = scaler.transform(dev_regressors)
test_regressors = scaler.transform(test_regressors)

full_col = np.vstack((train_regressors, dev_regressors))
full_col = np.vstack((full_col, test_regressors))

pca = PCA(n_components=2)
X_col = pca.fit_transform(full_col)

colors = []
markers = []
for i in range(len(train_regressors)):
    colors.append('red')
    markers.append('o')
for i in range(len(dev_regressors)):
    colors.append('green')
    markers.append('*')
for i in range(len(test_regressors)):
    colors.append('blue')
    markers.append('+')

fig = plt.figure(figsize=(20, 18))
plt.scatter(X_col[:, 0], X_col[:, 1], c=colors, marker='+', alpha=0.3)
plt.xlabel('component 1')
plt.ylabel('component 2')
fig.savefig('pca_no_tag_cols.png', bbox_inches='tight', dpi=300)
plt.close()


from django.shortcuts import render, redirect
from django.views.generic import ListView
from django.db.models import Q
from django.core.cache import cache

from .models import Author
from .forms import AuthorForm
from book.models import Book

class AuthorListView(ListView):
    context_object_name = 'authors'
    model = Author

    def get_queryset(self, *args, **kwargs):
        query = self.request.GET.get('q')
        sort_option = self.request.GET.get('sort_by')

        if query:
            return Author.objects.filter(Q(name__icontains=query)|
                                      Q(surname__icontains=query)|
                                      Q(patronymic__icontains=query)).distinct()

        elif sort_option:
            if sort_option == 'name(descending)':
                return Author.objects.order_by('-name')
            if sort_option == 'name(ascending)':
                return Author.objects.order_by('name')
            if sort_option == 'surname(ascending)':
                return Author.objects.order_by('surname')
            if sort_option == 'surname(descending)':
                return Author.objects.order_by('-surname')

        return Author.objects.order_by('name')

def author_books(request, pk):
    books = cache.get('books')
    if not books:
        author = Author.objects.get(id=pk)
        books = Book.objects.filter(author=author)
        cache.set('books', books, 60)

    return render(request, 'book/book_list.html', {'books': books})

def author_form(request, pk=None):
    if request.method == 'GET':
        if pk == None:
            form = AuthorForm()
        else:
            author = Author.objects.get(id=pk)
            form = AuthorForm(instance=author)
        return render(request, 'author/author_form.html', {'form': form})
    else:
        if pk == None:
            form = AuthorForm(request.POST)
        else:
            author = Author.objects.get(id=pk)
            form = AuthorForm(request.POST, instance=author)
        if form.is_valid():
            form.save()
        return redirect('/author/')

def author_delete(request, pk):
    author = Author.objects.get(id=pk)
    author.delete()
    return redirect('/author/')


import os

from django.db import models
from django.template.defaultfilters import slugify
from django.urls import reverse
from django.utils.safestring import mark_safe

from museum.settings import STATIC_URL
from museum_site.constants import STATIC_PATH, DATE_HR
from museum_site.models.base import BaseModel
from museum_site.querysets.series_querysets import Series_Queryset

class Series(BaseModel):
    objects = Series_Queryset.as_manager()
    model_name = "Series"
    table_fields = ["Series", "Updated", "Latest", "First", "Total"]
    cell_list = ["view", "last_updated", "latest_article", "first_article", "total_articles"]
    guide_word_values = {"id": "pk", "latest": "latest", "title": "title"}
    sort_options = [
        {"text": "Newest Entry", "val": "latest"},
        {"text": "Title", "val": "title"}
    ]
    sort_keys = {
        # Key - Value from <select> used in GET params
        # Value - Django order_by param
        "title": ["title"],
        "latest": ["-last_entry_date", "title"],
        "id": ["id"],
        "-id": ["-id"],
    }

    # Constants
    PREVIEW_DIRECTORY = os.path.join("pages/series-directory/")
    PREVIEW_DIRECTORY_FULL_PATH = os.path.join(
        STATIC_PATH, "pages/series-directory/"
    )

    # Fields
    title = models.CharField(max_length=80)
    slug = models.SlugField(max_length=80, editable=False)
    description = models.TextField(default="")
    preview = models.CharField(max_length=80, default="", blank=True)
    first_entry_date = models.DateField(null=True, blank=True)
    last_entry_date = models.DateField(null=True, blank=True)
    visible = models.BooleanField(default=True)

    class Meta:
        ordering = ["title"]

    def __str__(self):
        return "[" + str(self.id) + "] " + self.title

    def to_select(self):
        return self.title

    def save(self, *args, **kwargs):
        self.slug = slugify(self.title)
        if self.id:
            article_set = self.article_set.all()
            if article_set:
                self.first_entry_date = article_set.order_by("publish_date").first().publish_date
                self.last_entry_date = article_set.order_by("publish_date").last().publish_date

        # Prevent blank preview URLs
        if not self.preview:
            self.preview = self.slug + ".png"
        super(Series, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return reverse("series_view", kwargs={"series_id": self.pk, "slug": self.slug})

    def preview_url(self):
        return os.path.join(self.PREVIEW_DIRECTORY, self.preview)

    def get_meta_tag_context(self):
        """ Returns a dict of keys and values for <meta> tags  """
        tags = {}
        tags["author"] = ["name", ""]
        tags["description"] = ["name", self.description]
        tags["og:title"] = ["property", self.title + " - Museum of ZZT"]
        tags["og:image"] = ["property", self.preview_url()]  # Domain and static path to be added elsewhere
        return tags

    def get_field_view(self, view="detailed"):
        return {"value": "{}".format(self.get_absolute_url(), self.title), "safe": True}

    def get_field_last_updated(self, view="detailed"):
        article = self.article_set.all().order_by("-publish_date").first()
        return {"label": "Last Updated", "value": self.last_entry_date}

    def get_field_latest_article(self, view="detailed"):
        article = self.article_set.all().order_by("-publish_date").first()
        return self.get_field_this_article(article, view, label="Latest")

    def get_field_first_article(self, view="detailed"):
        article = self.article_set.all().order_by("publish_date").first()
        return self.get_field_this_article(article, view, label="First")

    def get_field_this_article(self, article, view="detailed", label="?"):
        article.request = self.request
        article._init_access_level()
        article._init_icons()
        if view == "list" or view == "gallery":
            field = article.get_field_view("title", text_override=label)
        else:
            field = article.get_field_view("title")
        field["label"] = "{} Article".format(label)
        return field

    def get_field_total_articles(self, view="detailed"):
        return {"label": "Total Articles", "value": self.article_set.count()}

    def get_field_description(self, view="detailed"):
        return {"label": "Description", "value": self.description}

    def context_detailed(self):
        context = self.context_universal()
        context["roles"] = ["model-block", "detailed"]
        context["columns"] = []

        columns = [
            ["last_updated", "latest_article", "first_article", "total_articles", "description"],
        ]

        for col in columns:
            column_fields = []
            for field_name in col:
                field_context = self.get_field(field_name)
                column_fields.append(field_context)
            context["columns"].append(column_fields)
        return context

    def context_list(self):
        context = self.context_universal()
        context["roles"] = ["list"]
        context["cells"] = []

        for field_name in self.cell_list:
            cell_fields = self.get_field(field_name, view="list")
            context["cells"].append(cell_fields)
        return context

    def context_gallery(self):
        context = self.context_universal()
        context["roles"] = ["model-block", "gallery"]
        context["fields"] = [
            self.get_field("latest_article", view="gallery"),
            self.get_field("first_article", view="gallery"),
        ]
        return context

    def get_guideword_latest(self): return self.last_entry_date.strftime(DATE_HR)


import math
from methods import piece_permanent_approximation

def trapezoid_method(func, a, b, nseg):
    """
    Метод трапеций
    Второй порядок аппроксимации
    """
    dx = 1.0 * (b - a) / nseg
    sum = 0.5 * (func(a) + func(b))
    for i in range(1, nseg):
        sum += func(a + i * dx)

    return sum * dx

def trapezoid_method_accuracy(func, a, b, rtol=1e-8, nseg0=1):
    """Метод трапеций с заданной точностью"""
    nseg = nseg0
    old_ans = 0.0
    dx = 1.0 * (b - a) / nseg
    ans = 0.5 * (func(a) + func(b))
    try:
        for i in range(1, nseg):
            ans += func(a + i * dx)

        ans *= dx
        err_est = max(1, abs(ans))
        while err_est > abs(rtol * ans):
            old_ans = ans
            ans = 0.5 * (
                ans + piece_permanent_approximation.midpoint_rectangle_method(
                    func, a, b, nseg))

            nseg *= 2
            err_est = abs(ans - old_ans)

        return ans
    except ValueError:
        print('Неверно задана функция. Проверьте правильность введенных \
              данных, а также допустимые значения математических функций')


class LayoutMiddleware(object):
    """
    Sets session['layout'] if the current menu item has a persistent layout
    and clears it when another menu_item changes the layout.
    """
    def process_request(self, request):
        import cyclope # Leave this here, it prevents circular imports
        import cyclope.settings # Leave this here

        menu_item = cyclope.utils.menu_item_for_request(request)
        if menu_item:
            ## we force the layout for items with custom URL (external apps)
            ## so they can use the correct template to extend from
            if menu_item.persistent_layout or menu_item.custom_url:
                layout = menu_item.get_layout()
                request.session['layout'] = layout
                request.session['layout_template'] = layout.get_template_path()

            else:
                if request.session.has_key('layout'):
                    del request.session['layout']
        else:
            ## if no layout_template has been set we set it to the default so external apps'
            ## templates that extend from request.session.layout_template won't break
            ## even when accessed through a direct URL
            if not 'layout_template' in request.session and \
                   hasattr(cyclope.settings, 'CYCLOPE_DEFAULT_TEMPLATE'):
                request.session['layout_template'] = cyclope.settings.CYCLOPE_DEFAULT_TEMPLATE


from sys import stdout
import random
import statistics

T, N, Q = map(int, input().split())
indices = list(range(N))
for case_idx in range(T):
    result = list(range(1, N + 1))
    samples = set()
    for _ in range(Q // T):
        while True:
            i, j, k = random.sample(indices, 3)
            if sorted((result[i], result[j], result[k])) not in samples:
                break
        samples.add(sorted((result[i], result[j], result[k])))
        si, sj, sk = result[i], result[j], result[k]
        m = statistics.median([i, j, k])
        print(result[i], result[j], result[k])
        stdout.flush()
        ri = int(input())
        # print(ri, result[m])
        if ri != result[m]:
            if ri == si:
                result[m], result[i] = result[i], result[m]
            if ri == sj:
                result[m], result[j] = result[j], result[m]
            if ri == sk:
                result[m], result[k] = result[k], result[m]
        # print(result)
    print(' '.join(map(str, result)))
    stdout.flush()

    reply = input()


import time
import threading
import logging

_logger = logging.getLogger(__name__)

# This is an interface
# MedianTimeSource provides a mechanism to add several time samples which are
# used to determine a median time which is then used as an offset to the local
# clock.
class MedianTimeSource:
    # AdjustedTime returns the current time adjusted by the median time
    # offset as calculated from the time samples added by AddTimeSample.
    def adjusted_time(self) -> int:
        raise NotImplementedError

    # AddTimeSample adds a time sample that is used when determining the
    # median time of the added samples.
    def add_time_sample(self, id, time_val):
        raise NotImplementedError

    # Offset returns the number of seconds to adjust the local clock based
    # upon the median of the time samples added by AddTimeData.
    def offset(self) -> int:
        raise NotImplementedError

#
# # TODO
# # int64Sorter implements sort.Interface to allow a slice of 64-bit integers to
# # be sorted.
# class Int64Sorter(int):
#     pass

# maxAllowedOffsetSeconds is the maximum number of seconds in either
# direction that local clock will be adjusted.  When the median time
# of the network is outside of this range, no offset will be applied.
maxAllowedOffsetSecs = 70 * 60  # 1 hour 10 minutes

# similarTimeSecs is the number of seconds in either direction from the
# local clock that is used to determine that it is likley wrong and
# hence to show a warning.
similarTimeSecs = 5 * 60  # 5 minutes

# maxMedianTimeEntries is the maximum number of entries allowed in the
# median time data.  This is a variable as opposed to a constant so the
# test code can modify it.
maxMedianTimeEntries = 200

# medianTime provides an implementation of the MedianTimeSource interface.
# It is limited to maxMedianTimeEntries includes the same buggy behavior as
# the time offset mechanism in Bitcoin Core.  This is necessary because it is
# used in the consensus code.
class MedianTime(MedianTimeSource):
    def __init__(self, lock=None, known_ids=None, offsets=None, offset_secs=None, invalid_time_checked=None, ):
        """

        :param RWLOCK lock:
        :param map[string]struct{} known_ids:
        :param []int64 offsets:
        :param int64 offset_secs:
        :param bool invalid_time_checked:
        """
        self.lock = lock or threading.Lock()
        self.known_ids = known_ids or {}
        self.offsets = offsets or []
        self.offset_secs = offset_secs or 0
        self.invalid_time_checked = invalid_time_checked or False

    # AddTimeSample adds a time sample that is used when determining the median
    # time of the added samples.
    #
    # This function is safe for concurrent access and is part of the
    # MedianTimeSource interface implementation.
    def adjusted_time(self):
        with self.lock:
            now = int(time.time())
            return now + self.offset_secs

    # AddTimeSample adds a time sample that is used when determining the median
    # time of the added samples.
    #
    # This function is safe for concurrent access and is part of the
    # MedianTimeSource interface implementation.
    def add_time_sample(self, source_id, time_val):
        with self.lock:
            # Don't add time data from the same source.
            if source_id is self.known_ids:
                return
            self.known_ids[source_id] = {}

            now = int(time.time())
            offset_seconds = time_val - now
            num_offsets = len(self.offsets)

            # Check if the offsets if full
            if num_offsets == maxMedianTimeEntries and maxMedianTimeEntries > 0:
                self.offsets = self.offsets[1:]
                num_offsets -= 1

            # append this offset seconds
            self.offsets.append(offset_seconds)
            num_offsets += 1

            # sort offset seconds
            sorted_offsets = sorted(self.offsets)

            offset_duration = offset_seconds
            _logger.debug("Added time sample of %s (total: %s)" % (offset_duration, num_offsets))

            # NOTE: The following code intentionally has a bug to mirror the
            # buggy behavior in Bitcoin Core since the median time is used in the
            # consensus rules.
            #
            # In particular, the offset is only updated when the number of entries
            # is odd, but the max number of entries is 200, an even number.  Thus,
            # the offset will never be updated again once the max number of entries
            # is reached.

            # The median offset is only updated when there are enough offsets and
            # the number of offsets is odd so the middle value is the true median.
            # Thus, there is nothing to do when those conditions are not met.
            if num_offsets < 5 or num_offsets & 0x01 != 1:
                return

            # At this point the number of offsets in the list is odd, so the
            # middle value of the sorted offsets is the median.
            median = sorted_offsets[num_offsets // 2]

            # Set the new offset when the median offset is within the allowed
            # offset range.
            if abs(median) < maxAllowedOffsetSecs:
                self.offset_secs = median
            else:
                # The median offset of all added time data is larger than the
                # maximum allowed offset, so don't use an offset.  This
                # effectively limits how far the local clock can be skewed.
                self.offset_secs = 0

                if not self.invalid_time_checked:
                    self.invalid_time_checked = True

                    # Find if any time samples have a time that is close
                    # to the local time.
                    remote_hash_close_time = False
                    for offset in sorted_offsets:
                        if abs(offset) < similarTimeSecs:
                            remote_hash_close_time = True
                            break

                    if not remote_hash_close_time:
                        _logger.debug("Please check your date and time are correct!  btcd will not work " +
                                      "properly with an invalid time")

            _logger.debug("New time offset: %s" % self.offset_secs)
            return

            # Offset returns the number of seconds to adjust the local clock based upon the

    # median of the time samples added by AddTimeData.
    #
    # This function is safe for concurrent access and is part of the
    # MedianTimeSource interface implementation.
    def offset(self):
        with self.lock:
            return self.offset_secs


# -*- coding: utf-8 -*-
"""
-------------------------------------------------
    File Name:      Crawler__qyxg_xzcf__cbrc.py
    Description:    银监会-行政处罚
    Author:         Jack Deng
    Date:           2017-09-18
    version:        v.1.0
-------------------------------------------------
"""
import copy
import json
import re
import traceback
import uuid

from scrapy import Request

from hive_framework_milk.commons.utils.selector_util import clean_all_space
from hive_framework_milk.scrapy_spiders.spiders.spider_all import SpiderAll
from hive_worker_hot_coffee.scrapy_spiders.specific_spiders.qyxg_xzxk.field_mapping import map_field

class Crawler__qyxg_xzcf__cbrc(SpiderAll):
    """
    银监会-行政处罚
    """

    name = "Crawler__qyxg_xzcf__cbrc"
    start_urls = ["http://www.cbrc.gov.cn/forwardToXZCFPage2.html"]

    def custom_init(self, *args, **kwargs):
        self.which_one = [
            "银监会机关", "银监局", "银监分局"
        ]

    def parse(self, response):
        """
        get three more href to yield
        :param response:
        :return:
        """
        try:
            more_hrefs = response.xpath('//a[contains(., "更多")]/@href').extract()
            for index, href in enumerate(more_hrefs):
                # if index == 0: #debug
                url = response.urljoin(href)
                yield Request(
                    url=url,
                    dont_filter=True,
                    callback=self.get_pages,
                    errback=self.err_parse,
                    meta={'which' : self.which_one[index]}
                )
                self.logger.info("yield List reqeust url: {} which: {}".format(url, self.which_one[index]))
                # break # debug
        except Exception as err:
            self.logger.error(
                msg="Exception occurred on parsing for first page[{url}]:{error} trace{tb}".format(url=response.url,
                                                                                error=err,
                                                                                tb=traceback.format_exc()),
                )

    def get_pages(self, response):
        """
        get page numbers
        :param response:
        :return:
        """
        try:
            which = response.meta['which']
            for req in self.parse_list(response):
                yield req
            pages_count = int(re.search(r'请输入小于(\d+).*?', response.text).group(1))
            for page in range(2, pages_count + 1):
                if which == "银监会机关":
                    url = "".join([response.url.replace('.html', ''), "&current=", str(page)])
                else:
                    url = "".join([response.url, "?current=", str(page)])
                yield Request(
                    url=url,
                    dont_filter=True,
                    callback=self.parse_list,
                    errback=self.err_get_pages,
                    meta={'which' : which, 'page' : str(page)}
                )
                self.logger.info("yield {} page {} to get detail url".format(which, page))
        except Exception as err:
            self.logger.error(
                msg="Exception occurred on parsing for first page[{url}]:{error} trace{tb}".format(url=response.url,
                                                                                error=err,
                                                                                tb=traceback.format_exc()),
                )

    def parse_list(self, response):
        """
        parse one page to yield each detail url
        :param response:
        :return:
        """
        try:
            which = response.meta['which']
            try:
                page = response.meta['page']
            except:
                page = "1"
            trs = response.xpath('//div[@class="xia3"]//tr')
            hrefs = ["".join(td.xpath('.//a/@href').extract()) for td in trs.xpath('.//td[position()=1]')]
            titles = ["".join(td.xpath('.//a/@title').extract()) for td in trs.xpath('.//td[position()=1]')]
            pubdates = [clean_all_space(td.xpath('string(.)').extract()) for td in trs.xpath('.//td[last()]')]
            hrefs_pubs_titles = zip(hrefs[0:-1], pubdates[0:-1], titles[0:-1]) #
            for href, pubdate, title in hrefs_pubs_titles:
                url = response.urljoin(href)
                if "报告" in title or "公告" in title or "目录" in title:
                    self.logger.warning("{} page {} url {} is unrelated, wont crawl".format(
                        which, page, url
                    ))
                    continue
                yield Request(
                    url=url,
                    callback=self.parse_detail,
                    errback=self.err_list,
                    meta={'which' : which, 'page' : page, 'public_date' : pubdate, 'title' : title}
                )
                self.logger.info("yield {} page {} detail url {}".format(
                    which, page, url
                ))

            # yield Request(  # bugfix test
            #     url="http://www.cbrc.gov.cn/chinese/home/docView/CFEDE60A320448CAA3FD920B1F7A1357.html",
            #     callback=self.parse_detail,
            #     errback=self.err_list,
            #     meta={'which': "银监局", 'page': 0, 'public_date': "2016-03-24", 'title': "宁波银监局行政处罚信息公开表"}
            # )
        except Exception as err:
            self.logger.error(
                msg="Exception occurred on parsing for first page[{url}]:{error} trace{tb}".format(url=response.url,
                                                                                error=err,
                                                                                tb=traceback.format_exc()),
                )

    def parse_detail(self, response):
        """
        parse detail
        :param response:
        :return:
        """
        try:
            which = response.meta['which']
            page = response.meta['page']
            part_res = {
                "public_date" : response.meta['public_date'],
                "title" : response.meta['title']
            }
            if "附件信息" in response.text:
                self.logger.warning("{} page {} url {} is just a .pdf link, wont parse".format(
                    which, page, response.url
                ))
                return
            section1 = response.xpath('//div[@class="WordSection1"]')
            if not section1:
                section1 = response.xpath('//div[contains(@class, "Section")]')
            if not section1:
                raise Exception("{} page {} url {} get data failed, please check it out!".format(
                    which, page, response.url
                ))
            tables = section1.xpath('.//table[@class="MsoNormalTable"]')
            if not tables:
                tables = section1.xpath('.//table[@class="MsoTableGrid"]')
            if tables:
                for item in self.deal_tables(tables, part_res, response):
                    yield item
            else:
                for item in self.deal_main(section1, part_res, response):
                    yield item

        except Exception as err:
            self.logger.error(
                msg="Exception occurred on parsing for first page[{url}]:{error} trace{tb}".format(url=response.url,
                                                                                error=err,
                                                                                tb=traceback.format_exc()),
                )

    def deal_main(self, main_html, result, response):
        """
        catch main text
        :param main_html:
        :param result:
        :param response:
        :return:
        """
        try:
            item = self.result_item_assembler(response)
            item['bbd_html'] = ''
            punish_org_text = "".join(main_html.xpath('.//p[contains(., "监管局") and contains(., "中国")]').xpath('string(.)').extract())
            punish_org = re.search(r'(中国.*?监管局).*?', punish_org_text).group(1) if punish_org_text else ""
            punish_code = "".join(main_html.xpath('//p[contains(., "监罚")]').xpath('string(.)').extract())
            if response.meta['which'] != "银监会机关":  # dont parse province for 银监会机关
                province_text = "".join(
                    main_html.xpath('//div[@id="docTitle"]/div[contains(., "文章来源")]/text()').extract())
                province = re.search(r'文章来源.*?([\u4e00-\u9fa5]{2})', province_text).group(1)
            else:
                province = ""
            main = "".join(main_html.xpath('.//p[contains(., "监罚")]//following-sibling::*').extract())
            if not main:
                main = "".join(main_html.xpath('.//*').extract())
            result.update({
                "punish_code" : punish_code,
                "province" : province,
                "main" : main,
                "punish_org" : punish_org
            })
            item['_parsed_data'] = result
            yield item
            self.logger.info("parse main successed, {} page {} url {} data {}".format(
                response.meta['which'], response.meta['page'], response.url, json.dumps(result)
            ))
        except:
            self.logger.error("error on deal_tables {} trace {}".format(
                response.url, traceback.format_exc(),
            ))

    def deal_tables(self, tables, result, response):
        """
        parse table
        :param tables:
        :param result:
        :param response:
        :return:
        """
        try:
            item = self.result_item_assembler(response)
            item['bbd_html'] = ''
            punish_code_num = len(re.findall(r'(行政处罚决定书文号|处罚决定书文号)', response.text))
            if response.meta['which'] != "银监会机关":  # dont parse province for 银监会机关
                province_text = "".join(
                    response.xpath('//div[@id="docTitle"]/div[contains(., "文章来源")]/text()').extract())
                province = re.search(r'文章来源.*?([\u4e00-\u9fa5]{2})', province_text).group(1)
            else:
                province = ""
            result.update({'province' : province})
            if punish_code_num == 1 and len(tables) == 3:
                # 处理　"银监分局" 里的特殊情况
                trs = tables.xpath('.//tr')
                titles, values = [], []
                for tr in trs:
                    tds = tr.xpath('.//td')
                    if len(tds) == 1:
                        continue
                    # http://www.cbrc.gov.cn/chinese/home/docView/7E4321407D13435D99064AA007A538B7.html
                    titles.append(clean_all_space(tds[len(tds) - 2].xpath('string(.)').extract()))
                    values.append("".join(tds[-1].xpath('string(.)').extract()).strip())
                tmp_dict = dict(set(zip(titles, values)))
                tmp_dict.pop("", "")
                maped = map_field(tmp_dict)
                result.update(maped)
                item['_parsed_data'] = result
                yield item
                self.logger.info("parse special table successed, {} page {} url {} data {}".format(
                    response.meta['which'], response.meta['page'], response.url, json.dumps(result)
                ))
            else:
                for table in tables:
                    trs = table.xpath('.//tr')
                    td_firsts = [clean_all_space(td.xpath('string(.)').extract()) for td in trs.xpath('.//td[position()=1]') if
                                 len(trs.xpath('.//td')) > 1]
                    if "序号" in td_firsts:  #  one tr tag, one data
                        #  http://www.cbrc.gov.cn/chinese/home/docView/0C0C87BDA4C3431B925FF7BC2461FA55.html
                        titles = [clean_all_space(td.xpath('string(.)').extract()) \
                                  for td in trs[td_firsts.index("序号")].xpath('.//td')]
                        for tr in trs[td_firsts.index("序号") + 1 : ]:
                            values = ["".join(td.xpath('string(.)').extract()).strip() for td in tr.xpath('.//td')]
                            maped = map_field(dict(zip(titles, values)))
                            maped.pop("序号", "")
                            maped.pop("", "")
                            if re.search(r'[0-9]', maped['punish_date']):
                                punish_date = list(maped.pop('punish_date'))
                                new_punish_date = "{}-{}-{}".format(
                                    "".join(punish_date[:4]),
                                    "".join(punish_date[4:6]),
                                    "".join(punish_date[6:])
                                )
                                maped['punish_date'] = new_punish_date
                            res_dict = copy.deepcopy(result)
                            res_dict.update(maped)
                            new_item = copy.deepcopy(item)
                            new_item['_id'] = "{}".format(uuid.uuid4())
                            new_item['_parsed_data'] = res_dict
                            yield new_item
                            self.logger.info("parse one line one data table successed, {} page {} url {} data {}".format(
                                response.meta['which'], response.meta['page'], response.url, json.dumps(res_dict)
                            ))
                    else:  # title : value in each tr tag
                        titles, values = [], []
                        for tr in trs:
                            tds = tr.xpath('.//td')
                            if len(tds) == 1:
                                continue
                            # http://www.cbrc.gov.cn/chinese/home/docView/D0952CF472AB443486104FB03E6FE862.html
                            titles.append(clean_all_space(tds[len(tds) - 2].xpath('string(.)').extract()))
                            values.append("".join(tds[-1].xpath('string(.)').extract()).strip())

                        if len(titles) == len(values):
                            tmp_dict = dict(zip(titles, values))
                        else:
                            raise Exception('titles ,values get error')
                        tmp_dict.pop("", "")
                        maped = map_field(tmp_dict)
                        if re.search(r'[0-9]', maped['punish_date']) and re.search(r'[年|月|日]', maped['punish_date']):
                            # new_punish_date = maped['punish_date'].replace('年', '-').replace('月', '-').replace('日', '')
                            new_punish_date = "{0}-{1:0>2}-{2:0>2}".format(
                                re.search(r'(\d{4}).*?年', maped['punish_date']).group(1),
                                re.search(r'年.*?(\d{1,2}).*?月', maped['punish_date']).group(1),
                                re.search(r'月.*?(\d{1,2}).*?日', maped['punish_date']).group(1),
                            )
                            maped['punish_date'] = new_punish_date
                        res_dict = copy.deepcopy(result)
                        res_dict.update(maped)
                        new_item = copy.deepcopy(item)
                        new_item['_id'] = "{}".format(uuid.uuid4())
                        new_item['_parsed_data'] = res_dict
                        yield new_item
                        self.logger.info("parse table successed, {} page {} url {} data {}".format(
                            response.meta['which'], response.meta['page'], response.url, json.dumps(res_dict)
                        ))
        except:
            self.logger.error("error on deal_tables {} trace {}".format(
                response.url, traceback.format_exc(),
            ))

    def err_list(self, failure):
        response = failure.value.response
        self.logger.error("Exception occurred on crawling for list page[{url}]:{httpcode}".format(url=response.url,
                                                                                httpcode=response.status))

    def err_get_pages(self, failure):
        response = failure.value.response
        self.logger.error("Exception occurred on crawling for getpage page[{url}]:{httpcode}".format(url=response.url,
                                                                                httpcode=response.status))

    def err_parse(self, failure):
        response = failure.value.response
        self.logger.error("Exception occurred on crawling for first page[{url}]:{httpcode}".format(url=response.url,
                                                                                httpcode=response.status))


import os
import sys

import util
from btc_config import get_merged_config
from btc_rest_api import EPRestApi

def run_btc_test(epp_file):
    # BTC EmbeddedPlatform API object
    work_dir = os.path.dirname(epp_file)
    config = get_merged_config(project_directory=work_dir)
    ep = EPRestApi(config=config, port=1337)

    # Load a BTC EmbeddedPlatform profile (*.epp)
    ep.get_req(f"profiles/{epp_file.replace('/', '%2F')}?discardCurrentProfile=true", message="Loading profile")

    # Applying preferences to use the correct compiler
    try:
        ep.put_req('preferences', [ { 'preferenceName' : 'GENERAL_COMPILER_SETTING', 'preferenceValue' : 'GCC (64bit)' } ])
    except:
        pass

    # Update architecture
    ep.put_req('profiles', { 'path': epp_file })
    ep.put_req('architectures', message="Architecture Update...")

    # Execute requirements-based tests
    response = ep.get_req('scopes')
    scopes = response.json()
    scope_uids = [scope['uid'] for scope in scopes if scope['architecture'] == 'C-Code']
    toplevel_scope_uid = scope_uids[0]
    rbt_exec_payload = {
        'UIDs': scope_uids,
        'data' : {
            'execConfigNames' : [ 'SIL' ]
        }
    }
    response = ep.post_req('scopes/test-execution-rbt', rbt_exec_payload, message="Executing requirements-based tests")
    rbt_coverage = rbt_coverage = ep.get_req(f"scopes/{toplevel_scope_uid}/coverage-results-rbt?goal-types=MCDC")
    util.print_rbt_results(response, rbt_coverage)

    # Create project report
    response = ep.post_req(f"scopes/{toplevel_scope_uid}/project-report", message="Creating test report")
    report = response.json()['result']
    # export project report to a file called 'report.html'
    ep.post_req(f"reports/{report['uid']}", { 'exportPath': work_dir, 'newName': 'report' })

    # Save *.epp
    ep.put_req('profiles', { 'path': epp_file }, message="Saving profile")

    print('Finished with workflow.')

if __name__ == '__main__':
    run_btc_test(sys.argv[1])


from django.conf import settings
from rest_framework import serializers
from rest_framework.exceptions import ValidationError

from core.models import Comment
from transactions.models import (
    Currency, Transaction, Interaction, TopicSnapshot, CommentSnapshot,
    HourPriceSnapshot, CurrencyPriceSnapshot, ContributionCertificate)
from trade.models import (Payment, Reserve)

from users.models import User

from api.v1.core.fields import UserField

class CurrencyListSerializer(serializers.HyperlinkedModelSerializer):

    hour_price = serializers.HyperlinkedRelatedField(
        view_name='hourpricesnapshot-detail',
        queryset=HourPriceSnapshot.objects.all())
    currency_price = serializers.HyperlinkedRelatedField(
        view_name='currencypricesnapshot-detail',
        queryset=CurrencyPriceSnapshot.objects.all())

    class Meta:
        model = Currency
        fields = ('id', 'label', 'in_hours', 'hour_price', 'currency_price', 'enabled')

class InteractionSerializer(serializers.HyperlinkedModelSerializer):

    comment = serializers.HyperlinkedRelatedField(
        view_name='transactions:interaction-detail',
        queryset=Comment.objects.all())
    snapshot = serializers.PrimaryKeyRelatedField(
        queryset=CommentSnapshot.objects.all())

    class Meta:
        model = Interaction
        fields = ('url', 'comment', 'snapshot', 'claimed_hours_to_match')

class TransactionCreateSerializer(serializers.HyperlinkedModelSerializer):

    comment = serializers.HyperlinkedRelatedField(
        view_name='comment-detail', queryset=Comment.objects.all())
    payment_currency = serializers.PrimaryKeyRelatedField(
        queryset=Currency.objects.all())
    payment_sender = serializers.PrimaryKeyRelatedField(
        queryset=User.objects.all())

    class Meta:
        model = Transaction
        fields = ('comment', 'payment_amount', 'payment_currency',
                  'payment_sender')

    def create(self, validated_data):
        comment = validated_data.get('comment')
        amount = validated_data['payment_amount']
        currency = validated_data['payment_currency']
        sender = validated_data['payment_sender']

        quota = Transaction.user_quota_remains_today(sender)
        reserve = Reserve.user_reserve_remains(sender)

        if amount > (quota + reserve):
            raise ValidationError('Hour amount larger than sum of user daily_quota and user_reserve.')

        tx = comment.invest(
            hour_amount=amount,
            payment_currency_label=currency.label.lower(),
            investor=sender,
        )

        if not tx:
            raise ValidationError('Bad data')

        return tx

class TransactionListSerializer(serializers.HyperlinkedModelSerializer):

    comment = serializers.HyperlinkedRelatedField(
        view_name='comment-detail', queryset=Comment.objects.all())
    snapshot = serializers.PrimaryKeyRelatedField(
        queryset=CommentSnapshot.objects.all())
    hour_price = serializers.PrimaryKeyRelatedField(
        queryset=HourPriceSnapshot.objects.all())
    currency_price = serializers.PrimaryKeyRelatedField(
        queryset=CurrencyPriceSnapshot.objects.all())
    payment_currency = serializers.PrimaryKeyRelatedField(
        queryset=Currency.objects.all())
    payment_recipient = UserField(read_only=True)
    payment_sender = UserField(read_only=True)

    class Meta:
        model = Transaction
        fields = ('url', 'comment', 'snapshot', 'hour_price', 'currency_price',
                  'payment_amount', 'payment_currency', 'payment_recipient',
                  'payment_sender', 'hour_unit_cost', 'donated_hours',
                  'matched_hours', 'hour_amount')

class ContributionSerializer(serializers.HyperlinkedModelSerializer):

    transaction = serializers.HyperlinkedRelatedField(
        view_name='transaction-detail',
        queryset=Transaction.objects.all())
    interaction = serializers.HyperlinkedRelatedField(
        view_name='interaction-detail',
        queryset=Interaction.objects.all())
    comment_snapshot = serializers.HyperlinkedRelatedField(
        view_name='commentsnapshot-detail',
        queryset=CommentSnapshot.objects.all())
    received_by = UserField(read_only=True)

    class Meta:
        model = ContributionCertificate
        fields = ('type', 'url', 'interaction', 'transaction', 'received_by',
                  'comment_snapshot', 'broken', 'parent')

class TopicSnapshotSerializer(serializers.HyperlinkedModelSerializer):

    topic = serializers.HyperlinkedRelatedField(
        view_name='topic-detail', queryset=Comment.objects.all())

    class Meta:
        model = TopicSnapshot
        fields = ('id', 'created_date', 'topic', 'data', 'blockchain',
                  'blockchain_tx')

class CommentSnapshotSerializer(serializers.HyperlinkedModelSerializer):

    comment = serializers.HyperlinkedRelatedField(
        view_name='comment-detail', queryset=Comment.objects.all())

    class Meta:
        model = CommentSnapshot
        fields = ('id', 'created_date', 'comment', 'data', 'blockchain',
                  'blockchain_tx')

class HourPriceSnapshotSerializer(serializers.HyperlinkedModelSerializer):
    base = serializers.HyperlinkedIdentityField(view_name="currency-detail")

    class Meta:
        model = HourPriceSnapshot
        fields = ('id', 'url', 'name', 'base', 'endpoint', 'data')

class CurrencyPriceSnapshotSerializer(serializers.HyperlinkedModelSerializer):
    base = serializers.HyperlinkedIdentityField(view_name="currency-detail")

    class Meta:
        model = CurrencyPriceSnapshot
        fields = ('id', 'name', 'base', 'endpoint', 'data')