from django.contrib.auth.decorators import login_required
from django.core.exceptions import ObjectDoesNotExist
from django.shortcuts import render, redirect, get_object_or_404
# from django.http import HttpResponse

from store.models import Product, Variation

from .models import Cart, CartItem

# Create your views here.

def _cart_id(request):
    # Use the session id as cart id.
    cart = request.session.session_key
    if not cart:
        cart = request.session.create()
    return cart

def _add_user_to_cart(request, cart=None):
    try:
        if not cart:
            cart = Cart.objects.get(cart_id=_cart_id(request))
        cart.user = request.user
        cart.save()
    except Exception as e:
        print(e)
        cart = None
    return cart

def _update_carts(request, old_cart_id):
    """Update anonymous and users previous cart id to match"""

    users_cart = Cart.objects.filter(user__pk=request.user.id).first()
    anonymous_cart = Cart.objects.filter(cart_id=old_cart_id).first()

    try:
        if users_cart and anonymous_cart:
            # ideally all of this will be atomic
            # TODO: on update to postgresql use: https://docs.djangoproject.com/en/4.1/ref/models/querysets/#select-for-update
            users_cart.cart_id = _cart_id(request)
            users_cart.save()

            # move anonymous cart items to users previous cart. First check whether
            # the users previous cart contains the unique product, varient combination.
            # If so, increment the already instanciated cart_item quantity by the
            # New varient quantity.
            # Otherwise, move the product-varient combo to the users cart and add the
            # variation to the existing_product_varients list
            users_cart_items = CartItem.objects.filter(cart=users_cart)

            existing_product_varients = [[item.product.id, *list(item.variations.all())]
                                            for item in users_cart_items]

            anonymous_cart_items = CartItem.objects.filter(cart=anonymous_cart)

            for a_item in anonymous_cart_items:
                varient = [a_item.product.id, *list(a_item.variations.all())]
                if varient in existing_product_varients:
                    # Increment the cart_item quantity and save
                    index = existing_product_varients.index(varient)
                    existing_varient = users_cart_items[index]
                    existing_varient.quantity += a_item.quantity
                    existing_varient.save()
                else:
                    # Move the item over to the cart
                    a_item.cart = users_cart
                    a_item.save()
                    existing_product_varients.append(varient)
            # Declutter the database and get rid of the anonymous cart
            anonymous_cart.delete()

        elif anonymous_cart:
            # This block executes only if
            # users_cart is None and anonymous_cart is not None
            #
            # update anonymous cart id to current session and add
            # authenticated user to anonymous cart
            anonymous_cart.cart_id = _cart_id(request)
            _add_user_to_cart(request, anonymous_cart)
            users_cart = anonymous_cart
        elif users_cart:
            # This block executes only if
            # users_cart is not None and anonymous_cart is None
            #
            # update users previous cart id only
            users_cart.cart_id = _cart_id(request)
            users_cart.save()
    except Exception as e:
        print(e)
        users_cart = None
    finally:
        return users_cart

def _get_post_request_variation(product_id, varients):
    """Parse a dict for variations"""
    product = Product.objects.get(id=product_id)
    product_variations = []
    if isinstance(varients, dict):
        for key, value in varients.items():
            try:
                # Filter out only the relevant information
                variation = Variation.objects.get(product=product,
                                                variation_category__iexact=key,
                                                variation_value__iexact=value)
                # add the product variation
                product_variations.append(variation)
            except:
                # Not a varient
                pass
    return product_variations

def add_item(request, product_id):
    """Increment cart item by one"""
    # Get the product variations from Post request
    product_variation = []
    if request.method == 'POST':
        product_variation = _get_post_request_variation(product_id, request.POST)

    # Add the product to the cart
    cart, _ = Cart.objects.get_or_create(cart_id=_cart_id(request))
    product = Product.objects.get(id=product_id)
    cart_items = CartItem.objects.filter(product=product, cart=cart)

    # If the variation already exists in the cart, increment the quantity
    added = False
    for ci in cart_items:
        if product_variation == list(ci.variations.all()):
            ci.quantity += 1
            added = True
            ci.save()
            break

    # Otherwise, create a new item
    if not added:
        ci = CartItem.objects.create(product=product, cart=cart, quantity=1)
        ci.variations.add(*product_variation)
        ci.save()

    return redirect('cart')

def remove_item(request, product_id, cart_item_id):
    """Decerement cart item by one"""
    product = get_object_or_404(Product, id=product_id)
    cart, _ = Cart.objects.get_or_create(cart_id=_cart_id(request))
    try:
        # Decrement or remove the item varient from the cart
        cart_item = CartItem.objects.get(product=product, cart=cart, id=cart_item_id)
        if cart_item.quantity > 1:
                cart_item.quantity -= 1
                cart_item.save()
        else:
            cart_item.delete()
    except:
        pass
    return redirect('cart')

def remove_cart_item(request, product_id, cart_item_id):
    """Remove all items of the varient from the cart"""
    product = get_object_or_404(Product, id=product_id)
    cart = Cart.objects.get(cart_id=_cart_id(request))
    try:
        cart_item = CartItem.objects.filter(product=product, cart=cart, id=cart_item_id)
        cart_item.delete()
    except:
        pass
    return redirect('cart')

def cart(request, total=0, quantity=0, cart_items=None):
    try:
        cart = Cart.objects.get(cart_id=_cart_id(request))
        cart_items = CartItem.objects.filter(cart=cart, is_active=True)
        for cart_item in cart_items:
            total += (cart_item.product.price * cart_item.quantity)
            quantity += cart_item.quantity
        tax = 0.095 * total
        grand_total = total + tax
    except ObjectDoesNotExist:
        #TODO: Fix this Make handling better
        tax=0
        grand_total=0

    context = {
        'total': total,
        'tax': tax,
        'grand_total': grand_total,
        'quantity': quantity,
        'cart_items': cart_items,
    }

    return render(request, 'cart.html', context=context)

@login_required(login_url='login')
def checkout(request, total=0, quantity=0, cart_items=None):
    try:
        cart = Cart.objects.get(cart_id=_cart_id(request))
        cart_items = CartItem.objects.filter(cart=cart, is_active=True)
        for cart_item in cart_items:
            total += (cart_item.product.price * cart_item.quantity)
            quantity += cart_item.quantity
        tax = 0.095 * total
        grand_total = total + tax
    except ObjectDoesNotExist:
        #TODO: Fix this Make handling better
        tax=0
        grand_total=0

    context = {
        'total': total,
        'tax': tax,
        'grand_total': grand_total,
        'quantity': quantity,
        'cart_items': cart_items,
    }

    return render(request, 'checkout.html', context=context)


import cv2
import os

def start_capture():
    cwd = os.getcwd()

    cam = cv2.VideoCapture(0)

    cv2.namedWindow("Calibration Window")

    img_counter = 0

    while True:
        ret, frame = cam.read()
        cv2.imshow("Calibration Window", frame)
        if not ret:
            break
        k = cv2.waitKey(1)

        if k%256 == 27:
            # ESC pressed
            break
        elif k%256 == 32:
            # SPACE pressed
            img_name = "calib_{}.png".format(img_counter)
            data_path = cwd + "/data"
            fullpath = os.path.join(data_path, img_name)
            cv2.imwrite(fullpath, frame)

            img_counter += 1

    cam.release()

    cv2.destroyAllWindows()


##11.Take a list of 10 elements. Split it into middle and store the elements in two dfferent lists. E.g.-
##INITIAL list :
##58	24	13	15	63	9	8	81	1	78
##
##After spliting :
##58	24	13	15	63
##9	8	81	1	78
a=int(input())
b=[]
for i in range(a):
    c=int(input())
    b+=[c]
n=len(b)
half=n/2
half=int(half)
print(b[:half])
print(b[half:])


# Criando o banco de Dados
import tkinter as tk
import sqlite3
import pandas as pd

# Conexão com o banco de dados
conexao = sqlite3.connect('clientes.db')
con = conexao.cursor()

# Criando a tabela no banco
con.execute(''' CREATE TABLE clientes(
   nome text,
   sobrenome text,
   email text,
   telefone text
)''')
# Confirmando
conexao.commit()
# Fechar o banco de dados
conexao.close()


# -*- coding: utf-8 -*-

import pytest

import os

from betfair import betfair
from tests.utils import response_fixture_factory

@pytest.fixture
def client():
    return betfair.Betfair(app_key='test', cert_file='path/to/cert')

@pytest.fixture
def logged_in_client(client):
    client = betfair.Betfair(app_key='test', cert_file='path/to/cert')
    client.session_token = 'secret'
    return client

login_success = response_fixture_factory(
    os.path.join(betfair.IDENTITY_URLS[None], 'certlogin'),
    {
        'loginStatus': 'SUCCESS',
        'sessionToken': 'secret',
    },
)

login_failure = response_fixture_factory(
    os.path.join(betfair.IDENTITY_URLS[None], 'certlogin'),
    {'loginStatus': 'INVALID_USERNAME_OR_PASSWORD'},
)

login_bad_code = response_fixture_factory(
    os.path.join(betfair.IDENTITY_URLS[None], 'certlogin'),
    status=422,
)

keepalive_success = response_fixture_factory(
    os.path.join(betfair.IDENTITY_URLS[None], 'keepAlive'),
    {'status': 'SUCCESS'},
)

keepalive_failure = response_fixture_factory(
    os.path.join(betfair.IDENTITY_URLS[None], 'keepAlive'),
    {
        'status': 'FAIL',
        'error': 'NO_SESSION',
    },
)

logout_success = response_fixture_factory(
    os.path.join(betfair.IDENTITY_URLS[None], 'logout'),
    {'status': 'SUCCESS'},
)

logout_failure = response_fixture_factory(
    os.path.join(betfair.IDENTITY_URLS[None], 'logout'),
    {
        'status': 'FAIL',
        'error': 'NO_SESSION',
    },
)

login_required_methods = [
    'keep_alive',
    'logout',
    'list_event_types',
    'list_competitions',
    'list_time_ranges',
    'list_events',
    'list_market_types',
    'list_countries',
    'list_venues',
    'list_market_catalogue',
    'list_market_book',
    'list_market_profit_and_loss',
    'list_current_orders',
    'list_cleared_orders',
    'place_orders',
    'cancel_orders',
    'replace_orders',
    'update_orders',
]


#calculatorProgram.py

"""
Title: Calculator Program
Author: Jenna Tran
Date Created: 2023-02-08
"""

#---INPUT---#
def getNumber():
    """
    This function will retreive a number from the user
    """
    userinput = input("insert number:")
    userinput = isNumeric(userinput)
    if userinput == False:
        return getNumber()
    else:
        return userinput

def operationChoice():
    """
    user picks which operation to use
    """
    print('''
    1. Addition
    2. Subtraction
    3. Multiplication
    4. Division
    ''')
    choice = input("> ")
    choice = isNumeric(choice)
    if choice > 0 and choice < 5:
        return choice
    else:
        return operationChoice()

#---PROCESSING---#

def isNumeric(userinput):
    """
    This function will check if user input is valid.
    """

    try:
        userinput = int(userinput)
        return userinput
    except:
        print("Please pick a valid number.")
        return False

def addition(number1, number2):
    """
    this function will add two variables
    """
    answer = number1 + number2
    return answer

#---OUTPUT---#

def displayAnswer(answer):
    """
    function will display answer to the user
    """
    print("The solution to your calculation is : ", answer)

#---MAIN PROGRAM CODE---#

while True:
    input1 = getNumber()
    input2 = getNumber()
    choice = operationChoice()
    if choice == 1:
        answer = addition(input1, input2)
    displayAnswer(answer)


"""
Author: Jakidxav
Date: 06.25.2019

This program cleans and extends a positive and negative words list (link below). It then passes those lists to our training and development directories to create words counts based off of those lists and saves the output as a CSV file. We are going to use these word counts for our correlation analysis to determine if the rate of positive or negative word usage corresponds to higher reported PHQ-8 score.
"""

import csv
import pandas as pd
import numpy as np
import os

from clean_text import *
from process_word_counts import *

#change directory
with_i_dir_train = './trainPart/cleaned_with_i'
with_i_dir_dev = './devPart/cleaned_with_i'
with_i_dir_test = './testPart/cleaned_with_i'

#filenames to save to
to_save_train = 'pos_neg_counts_train.csv'
to_save_dev = 'pos_neg_counts_dev.csv'
to_save_test = 'pos_neg_counts_test.csv'

#positive and negative word lists obtained from here:
#http://www.wjh.harvard.edu/~inquirer/homecat.htm
pos_neg_file = 'inquirerbasic.xls'

#we need the participant numbers to create TaggedDocument
participant_nums = []

#positive and negative word counts lists
pos, neg = [], []
pos_freq, neg_freq = [], []

#change this for train or dev set directory
change_to_dir = with_i_dir_test

#where to save output
to_save = to_save_test

#extract positive and negative word lists
positive, negative = process_pos_neg_file(pos_neg_file)

#main processing step: creates a dataframe of word counts, then saves it to a CSV
positive_negative_counts(change_to_dir, to_save, participant_nums, positive, pos_freq, negative, neg_freq)


import json
import multiprocessing as mp
import numpy as np
import scipy as sp
import os
import pickle
import pandas as pd
import pkg_resources
import time
import copy
from sklearn.model_selection import train_test_split
import tensorly as tl
import signal
from scipy.stats import mode
from contextlib import contextmanager

from . import convex_opt
from .linalg import pca, approx_matrix_rank, pivot_columns, impute, approx_tensor_rank, impute_with_coefficients
from .util import extract_columns, check_dataframes, tucker_on_error_tensor, generate_settings, error
from .pipeline import PipelineObject
from .model import Model
from .ensemble import Ensemble, Model_collection
from . import experiment_design as ED

class AutoLearner:
    """An object that automatically selects pipelines by greedy D-optimal design.

    Basic attributes:
        p_type (str):                  Problem type. One of {'classification', 'regression'}.
        algorithms (list):             A list of algorithm types to be considered, in strings. (e.g. ['KNN', 'lSVM']).
        hyperparameters (dict):        A nested dict of hyperparameters to be considered.
        n_folds (int):                 Number of cross-validation folds.
        verbose (bool):                Whether or not to generate print statements that showcase the progress.
        n_cores (int):                 Maximum number of cores over which to parallelize (None means no limit).
        runtime_limit (int):           Maximum training time for AutoLearner, in seconds.
        fit_ensemble_despite_timeout (bool): Whether to still fit the ensemble despite a fitting timeout.

    Advanced attributes:
        mode (str):                    The running mode of TensorOboe.
        new_row (np.ndarray):          Predicted row of matricized error tensor, corresponding to the new dataset. Default None.
        dataset_ratio_threshold(float):The threshold of dataset ratio (number of points / number of features) for dataset subsampling, if the training set is tall and skinny (number of data points much larger than number of features).
        runtime_predictor_algorithm (str):
                                       Model for runtime prediction. One of {'LinearRegression', 'KNeighborsRegressor'}.
        ensemble_method (str):         Ensemble method. One of {'greedy', 'stacking', 'best_several'}.
        load_default_tensor (bool):    Whether to use the default error and runtime tensors.
        selection_method (str):        Method of selecting entries of new row to sample. One of {'qr', 'min_variance', 'random'}.
        scalarization (str):           (Only in Oboe) Scalarization of the covariance matrix for mininum variance selection. One of {'D', 'A', 'E'}.

    Attributes to be added in the future:
        X, Y (np.ndarray):             PCA decomposition of error matrix.
        sampled_indices (set):         Indices of new row that have been sampled.
        sampled_models (list):         List of models that have been sampled (i.e. k-fold fitted).
        fitted_indices (set):          Indices of new row that have been fitted (i.e. included in ensemble)
        fitted_models (list):          List of models that have been fitted.
    """

    def __init__(self,
                 p_type='classification',
                 method='Oboe',
                 mode='warm',
                 algorithms=None,
                 hyperparameters=None,
                 verbose=False,
                 n_cores=1,
                 n_folds=5,
                 runtime_limit=512,
                 dataset_ratio_threshold=100,
                 new_row=None,
                 load_defaults=True,
                 customized_defaults_path='',
                 load_imputed_error_tensor=True,
                 original_error_tensor_dir=None,
                 path_to_imputed_error_tensor='default',
                 save_imputed_error_tensor=True,
                 selection_method='ED', scalarization='D',
                 build_ensemble=True,
                 load_saved_latent_factors=True,
                 save_latent_factors=True,
                 ensemble_method='best_several',
                 ensemble_max_size=5,
                 runtime_predictor_algorithm='LinearRegression',
                 load_saved_runtime_predictors=True,
                 save_fitted_runtime_predictors=False,
                 random_state=0,
                 fit_ensemble_despite_timeout=True,
                 **stacking_hyperparams):

        method = method.lower()
        assert method in {'oboe', 'tensoroboe'}, "The method must be one of {'Oboe', 'TensorOboe'}."
        self.method = method

        if method == 'tensoroboe':
            if mode == 'initialize':  # impute error tensor, compute low rank factors, fit runtime predictors
                load_imputed_error_tensor=False
                save_imputed_error_tensor=True
                load_saved_latent_factors=False
                load_saved_runtime_predictors=False
                save_fitted_runtime_predictors=True
            elif mode == 'impute':  # impute error tensor, compute low rank factors, use fitted runtime predictors
                load_imputed_error_tensor=False
                save_imputed_error_tensor=True
                load_saved_latent_factors=False
                load_saved_runtime_predictors=True
                save_fitted_runtime_predictors=False
            elif mode == 'factorize':  # used imputed error tensor, compute low rank factors, use fitted runtime predictors
                load_imputed_error_tensor=True
                save_imputed_error_tensor=False
                load_saved_latent_factors=False
                load_saved_runtime_predictors=True
                save_fitted_runtime_predictors=False
            elif mode == 'warm':  # used imputed error tensor, use low rank factors, use fitted runtime predictors
                load_imputed_error_tensor=True
                save_imputed_error_tensor=False
                load_saved_latent_factors=True
                load_saved_runtime_predictors=True
                save_fitted_runtime_predictors=False

        self.verbose = verbose
        self.random_state = random_state
        self.selection_method = selection_method
        self.p_type = p_type.lower()
        self.fit_ensemble_despite_timeout = fit_ensemble_despite_timeout

        if self.method == 'oboe':
            DEFAULTS = pkg_resources.resource_filename(__name__, 'defaults/Oboe')

            # TODO: check if arguments to constructor are valid; set to defaults if not specified
            assert selection_method in {'qr', 'ED', 'random'}, "The method to select entries to sample must be " \
                "either qr (QR decomposition), ED (minimize variance with time constraints), " \
                "or random (time-constrained random selection, i.e., set the models that should be able to finish within time budget)."

            # model configurations
            with open(os.path.join(DEFAULTS, p_type + '.json')) as file:
                defaults = json.load(file)

            # attributes of ML problem
            self.p_type = p_type.lower()
            self.algorithms = algorithms or defaults['algorithms']
            self.hyperparameters = hyperparameters or defaults['hyperparameters']
            self.verbose = verbose

            # computational considerations
            self.n_cores = n_cores
            self.runtime_limit = runtime_limit

            # configurations for column selection in cold-start
            self.selection_method = selection_method
            self.scalarization = scalarization

            ERROR_MATRIX = pd.read_csv(os.path.join(DEFAULTS, 'error_matrix.csv'), index_col=0, header=0)
            RUNTIME_MATRIX = pd.read_csv(os.path.join(DEFAULTS, 'runtime_matrix.csv'), index_col=0, header=0)

            # error matrix factorization
            # TODO: determine whether to generate new error matrix or use default/subset of default
            self.error_matrix = extract_columns(ERROR_MATRIX, self.algorithms, self.hyperparameters)
            self.runtime_matrix = extract_columns(RUNTIME_MATRIX, self.algorithms, self.hyperparameters)
            assert check_dataframes(self.error_matrix, self.runtime_matrix)
            self.column_headings = np.array([eval(heading) for heading in list(self.error_matrix)])
            self.X, self.Y, _ = pca(self.error_matrix.values, rank=min(self.error_matrix.shape)-1)

            # sampled & fitted models
            self.new_row = new_row or np.zeros((1, self.error_matrix.shape[1]))
            self.sampled_indices = set()
            self.sampled_models = [None] * self.error_matrix.shape[1]
            self.fitted_indices = set()
            self.fitted_models = [None] * self.error_matrix.shape[1]

            # ensemble attributes
            self.build_ensemble = build_ensemble
            self.ensemble_method = ensemble_method
            self.stacking_hyperparams = stacking_hyperparams
            if self.build_ensemble:
                self.ensemble = Ensemble(p_type=self.p_type, algorithm=self.ensemble_method, max_size=ensemble_max_size,
                                         hyperparameters=self.stacking_hyperparams, verbose=self.verbose, method='oboe')
            else:
                self.ensemble = Model_collection(self.p_type, method='oboe')

            # runtime predictor
            self.runtime_predictor_algorithm = runtime_predictor_algorithm

            self.dataset_ratio_threshold = dataset_ratio_threshold

        elif self.method == 'tensoroboe':
            if load_defaults:
                DEFAULTS = pkg_resources.resource_filename(__name__, 'defaults/TensorOboe')
            else:
                DEFAULTS = customized_defaults_path

            if not os.path.exists(os.path.join(DEFAULTS, 'tmp')):
                os.makedirs(os.path.join(DEFAULTS, 'tmp'))
                if self.verbose:
                    print("created a tmp directory in DEFAULTS ...")

            if not load_imputed_error_tensor:
                assert original_error_tensor_dir != None
                if self.verbose:
                    print("loading Float16 not-imputed error tensor ...")
                    # print("loading original Float64 error tensor ...")

                ERROR_TENSOR = np.float64(np.load(os.path.join(original_error_tensor_dir, 'error_tensor_f16_compressed.npz'))['a'])
                # ERROR_TENSOR = np.vstack((np.load(os.path.join(original_error_tensor_dir, 'error_tensor_part_1.npy')),
                #     np.load(os.path.join(original_error_tensor_dir, 'error_tensor_part_2.npy'))))

            RUNTIME_TENSOR = np.float64(np.load(os.path.join(DEFAULTS, 'runtime_tensor_f16_compressed.npz'))['a'])

            with open(os.path.join(DEFAULTS, 'training_index.pkl'), 'rb') as handle:
                TRAINING_INDEX = pickle.load(handle)

            with open(os.path.join(DEFAULTS, self.p_type + '.json')) as file:
                configs_unzipped = json.load(file)

            # attributes of ML problem
            self.algorithms = algorithms or configs_unzipped['estimator']['algorithms']
            self.hyperparameters = hyperparameters or configs_unzipped['estimator']['hyperparameters']

            # computational considerations
            if n_cores is None:
                n_cores = mp.cpu_count()
            self.n_cores = n_cores
            self.runtime_limit = runtime_limit
            self.n_folds = n_folds

            # error tensor completion
            rank_for_imputation = (20, 4, 2, 2, 8, 20)
            if verbose:
                print("rank for EM-Tucker imputation: {}".format(rank_for_imputation))
            rank_tuple = '-'.join([str(item) for item in rank_for_imputation])
            if load_imputed_error_tensor:
                try:
                    if path_to_imputed_error_tensor == 'default':
                        error_tensor_imputed = np.float64(np.load(os.path.join(DEFAULTS, 'error_tensor_imputed_20-4-2-2-8-20_f16_compressed.npz'))['a'])
                    else:
                        if self.verbose:
                            print("loading customized tensor at {} ...".format(path_to_imputed_error_tensor))
                        error_tensor_imputed = np.load(path_to_imputed_error_tensor)
                except:
                    print("Error loading imputed error tensor!")
            else:
                _, _, error_tensor_imputed, _ = tucker_on_error_tensor(ERROR_TENSOR, rank_for_imputation, save_results=False, verbose=self.verbose)
                if save_imputed_error_tensor:
                    imputed_error_tensor_save_path = os.path.join(DEFAULTS, 'tmp', 'error_tensor_imputed_{}.npy'.format(rank_tuple))
                    np.save(imputed_error_tensor_save_path, error_tensor_imputed)
                    if self.verbose:
                        print("saved imputed error tensor to {}".format(imputed_error_tensor_save_path))

            if verbose:
                print("shape of the error tensor: {}".format(error_tensor_imputed.shape))

            self.error_tensor_imputed = error_tensor_imputed

            # error tensor factorization
            # TODO: determine whether to generate new error matrix or use default/subset of default

            # the maximum ranks for dataset and estimator dimensions
            k_dataset_for_factorization = 30
            k_estimator_for_factorization = 30

            if load_saved_latent_factors:
                try:
                    U_t = np.load(os.path.join(DEFAULTS, 'tmp', 'error_tensor_U_t.npy'))
                    S_t = np.load(os.path.join(DEFAULTS, 'tmp', 'error_tensor_S_t.npy'))
                    Vt_t = np.load(os.path.join(DEFAULTS, 'tmp', 'error_tensor_Vt_t.npy'))
                    factorize_error_tensor = False
                except:
                    factorize_error_tensor = True
                    if self.verbose:
                        print("No saved latent factors. Factorizing the error tensor now ...")
            else:
                factorize_error_tensor = True

            if factorize_error_tensor:
                if self.verbose:
                    print("Factorizing the error matrix to get latent factors ...")
                core_tr, factors_tr = tl.decomposition.tucker(error_tensor_imputed, rank=(k_dataset_for_factorization, 4, 2, 2, 8, k_estimator_for_factorization))
                pipeline_latent_factors = tl.unfold(tl.tenalg.multi_mode_dot(core_tr, factors_tr[1:], modes=[1, 2, 3, 4, 5]), mode=0)
                U_t, S_t, Vt_t = sp.linalg.svd(pipeline_latent_factors, full_matrices=False)
                if save_latent_factors:
                    np.save(os.path.join(DEFAULTS, 'tmp', 'error_tensor_U_t.npy'), U_t)
                    np.save(os.path.join(DEFAULTS, 'tmp', 'error_tensor_S_t.npy'), S_t)
                    np.save(os.path.join(DEFAULTS, 'tmp', 'error_tensor_Vt_t.npy'), Vt_t)
                    if self.verbose:
                        print("latent factors saved to {}".format(os.path.join(DEFAULTS, 'tmp')))
            else:
                if self.verbose:
                    print("Loading latent factors from storage ...")

            # not yet implemented the part of selecting specific algorithms and hyperparameters
            self.error_matrix = tl.unfold(error_tensor_imputed, mode=0)
            self.runtime_matrix = tl.unfold(RUNTIME_TENSOR, mode=0)
            self.Y = Vt_t

            self.training_index = TRAINING_INDEX

            self.pipeline_settings = generate_settings(configs_unzipped)

             # sampled & fitted models
            self.new_row = new_row or np.full((1, len(self.pipeline_settings)), np.nan)
            self.new_row_pred = new_row or np.full((1, len(self.pipeline_settings)), np.nan)
            self.sampled_indices = set()
            self.sampled_pipelines = [None] * len(self.pipeline_settings)
            self.fitted_indices = set()
            self.fitted_pipelines = [None] * len(self.pipeline_settings)

            # ensemble attributes
            self.build_ensemble = build_ensemble
            self.ensemble_method = ensemble_method
            self.stacking_hyperparams = stacking_hyperparams
            if self.build_ensemble:
                self.ensemble = Ensemble(p_type=self.p_type, algorithm=self.ensemble_method, max_size=ensemble_max_size, hyperparameters=self.stacking_hyperparams, verbose=self.verbose, method='tensoroboe')
            else:
                self.ensemble = Model_collection(self.p_type, method='tensoroboe')

            # find the best-on-average pipeline
            self.best_on_average_pipeline_index = np.argmin(np.nanmean(tl.unfold(self.error_tensor_imputed, mode=0), axis=0))

            # runtime predictor
            # TODO: let it customizable
            self.runtime_predictor_algorithm = runtime_predictor_algorithm
            self.runtime_predictor = convex_opt.initialize_runtime_predictor(runtime_matrix=self.runtime_matrix, runtimes_index=self.training_index, model_name=self.runtime_predictor_algorithm, load=load_saved_runtime_predictors, save=save_fitted_runtime_predictors, method=self.method, verbose=self.verbose)
            self.dataset_ratio_threshold = dataset_ratio_threshold

    def _fit(self, x_train, y_train, categorical, t_predicted, ranks=None, runtime_limit=None, remaining_global=None):
        """This private method is a single round of the doubling process. It fits an AutoLearner object on a new dataset.
        This will sample the performance of several algorithms on the new dataset, predict performance on the rest, then construct an optimal ensemble model.

        Args:
            x_train (np.ndarray):  Features of the training dataset.
            y_train (np.ndarray):  Labels of the training dataset.
            t_predicted (np.ndarray): Predicted running time.
            ranks (int):            Rank of error tensor factorization.
            runtime_limit (float): Maximum time to allocate to AutoLearner fitting.
            remaining_global (float): The remaining runtime
        """

        if self.method == 'oboe':
            if self.verbose:
                print("\nSingle round runtime target: {}".format(runtime_limit))

            # set to defaults if not provided
            rank = ranks or approx_matrix_rank(self.error_matrix, threshold=0.01)
            runtime_limit = runtime_limit or self.runtime_limit

            if self.verbose:
                print('Fitting AutoLearner with max runtime {}s'.format(runtime_limit))

            # cold-start: pick the initial set of models to fit on the new dataset
            if self.selection_method == 'qr':
                to_sample = pivot_columns(self.error_matrix)
            elif self.selection_method == 'ED':
                # select algorithms to sample only from subset of algorithms that will run in allocated time
                valid = np.where(t_predicted <= self.n_cores * runtime_limit/2)[0]
                Y = self.Y[:rank, valid]
                # TODO: check if Y is rank-deficient, i.e. will ED problem fail
                v_opt = ED.solve(t_predicted[valid], runtime_limit/4, self.n_cores, Y, self.scalarization)
                to_sample = valid[np.where(v_opt > 0.9)[0]]
                if np.isnan(to_sample).any():
                    to_sample = np.argsort(t_predicted)[:rank]

            elif self.selection_method == 'random':
                to_sample = []
                # set of algorithms that are predicted to finish within given budget
                to_sample_candidates = np.where(t_predicted <= runtime_limit/2)[0]
                # remove algorithms that have been sampled already
                to_sample_candidates = list(set(to_sample_candidates) - self.sampled_indices)
                # if the remaining time is not sufficient for random sampling
                if len(to_sample_candidates) == 0:
                    to_sample = np.array([np.argmin(t_predicted)])
                else:
                    to_sample = np.random.choice(to_sample_candidates, min(self.n_cores, len(to_sample_candidates)), replace=False)
            else:
                to_sample = np.arange(0, self.new_row.shape[1])

            if len(to_sample) == 0 and len(self.sampled_indices) == 0:
                # if no columns are selected in first iteration (log det instability), sample n fastest columns
                n = len(np.where(np.cumsum(np.sort(t_predicted)) <= runtime_limit/4)[0])
                if n > 0:
                    to_sample = np.argsort(t_predicted)[:n]
                else:
                    self.ensemble.fitted = False
                    return

            start = time.time()
            if self.selection_method != 'random':
                # we only need to fit models on the new dataset if it has not been fitted already
                to_sample = list(set(to_sample) - self.sampled_indices)
                if self.verbose:
                    print('Sampling {} entries of new row...'.format(len(to_sample)))

                p1 = mp.Pool(self.n_cores)
                sample_models = [Model(self.p_type, self.column_headings[i]['algorithm'],
                                       self.column_headings[i]['hyperparameters'], self.verbose, i) for i in to_sample]
                sample_model_errors = [p1.apply_async(Model.kfold_fit_validate, args=[m, x_train, y_train, 5])
                                       for m in sample_models]
                p1.close()
                p1.join()

                # predict performance of models not actually fitted on the new dataset
                self.sampled_indices = self.sampled_indices.union(set(to_sample))
                for i, error in enumerate(sample_model_errors):
                    cv_error, cv_predictions = error.get()
                    sample_models[i].cv_error, sample_models[i].cv_predictions = cv_error.mean(), cv_predictions
                    sample_models[i].sampled = True
                    self.new_row[:, to_sample[i]] = cv_error.mean()
                    self.sampled_models[to_sample[i]] = sample_models[i]
                imputed = impute(self.error_matrix, self.new_row, list(self.sampled_indices), rank=rank)

                # impute ALL entries
                # unknown = sorted(list(set(range(self.new_row.shape[1])) - self.sampled_indices))
                # self.new_row[:, unknown] = imputed[:, unknown]

                # TODO: differentiate self.new_row and self.new_row_pred like TensorOboe
                self.new_row = imputed.copy()
                self.new_row_pred = imputed.copy()

                # k-fold fit candidate learners of ensemble
                remaining = (runtime_limit - (time.time()-start)) * self.n_cores
                # add models predicted to be the best to list of candidate learners to avoid empty lists
                best_sampled_idx = list(self.sampled_indices)[int(np.argmin(self.new_row[:, list(self.sampled_indices)]))]
                assert self.sampled_models[best_sampled_idx] is not None
                candidate_indices = [best_sampled_idx]
                self.ensemble.candidate_learners.append(self.sampled_models[best_sampled_idx])
                for i in np.argsort(self.new_row[0]):
                    if t_predicted[i] + t_predicted[candidate_indices].sum() <= remaining:
                        candidate_indices.append(i)
                        # last = candidate_indices.pop()
                        # assert last == best_sampled_idx
                        # candidate_indices.append(last)
                        # if model has already been k-fold fitted, immediately add to candidate learners
                        if i in self.sampled_indices:
                            assert self.sampled_models[i] is not None
                            self.ensemble.candidate_learners.append(self.sampled_models[i])
                # candidate learners that need to be k-fold fitted
                to_fit = list(set(candidate_indices) - self.sampled_indices)
            else:
                remaining = (runtime_limit - (time.time()-start)) * self.n_cores
                to_fit = to_sample.copy()

            # fit models predicted to have good performance and thus going to be added to the ensemble
            p2 = mp.Pool(self.n_cores)
            candidate_models = [Model(self.p_type, self.column_headings[i]['algorithm'],
                                      self.column_headings[i]['hyperparameters'], self.verbose, i) for i in to_fit]
            candidate_model_errors = [p2.apply_async(Model.kfold_fit_validate, args=[m, x_train, y_train, 5])
                                      for m in candidate_models]
            p2.close()
            p2.join()

            # update sampled indices
            self.sampled_indices = self.sampled_indices.union(set(to_fit))
            for i, error in enumerate(candidate_model_errors):
                cv_error, cv_predictions = error.get()
                candidate_models[i].cv_error, candidate_models[i].cv_predictions = cv_error.mean(), cv_predictions
                candidate_models[i].sampled = True
                self.new_row[:, to_fit[i]] = cv_error.mean()
                self.sampled_models[to_fit[i]] = candidate_models[i]
                self.ensemble.candidate_learners.append(candidate_models[i])
            # self.new_row = impute(self.error_matrix, self.new_row, list(self.sampled_indices), rank=rank)

            if self.verbose:
                print('\nFitting ensemble of max size {}...'.format(len(self.ensemble.candidate_learners)))
            # ensemble selection and fitting in the remaining time budget
            self.ensemble.fit(x_train, y_train, runtime_limit=remaining, fitted_base_learners=self.fitted_models)
            for model in self.ensemble.base_learners:
                assert model.index is not None
                self.fitted_indices.add(model.index)
                self.fitted_models[model.index] = model
            self.ensemble.fitted = True

            if self.verbose:
                print('\nAutoLearner fitting complete.')

        elif self.method == 'tensoroboe':
            if self.verbose:
                print("\nSingle round runtime target: {}".format(runtime_limit))

            # set to defaults if not provided
            ranks = ranks or approx_tensor_rank(self.error_tensor_imputed, threshold=0.01)
            runtime_limit = runtime_limit or self.runtime_limit

            if self.verbose:
                print('Fitting AutoLearner with maximum runtime {} seconds'.format(runtime_limit))

            # cold-start: pick the initial set of models to fit on the new dataset
            valid = np.where(t_predicted <= self.n_cores * runtime_limit/8)[0]
            Y = self.Y[:ranks[0], valid]

            if self.verbose:
                print("Selecting an initial set of models to evaluate ...")

            selected_columns_qr, t_sum, case = ED.pivot_columns_time(Y, t_predicted[valid], runtime_limit/8,
                                                            columns_to_avoid=None,
                                                            rank=Y.shape[0])
            if case == 'greedy_initialization':
                if self.verbose:
                    print(case)
                to_sample = valid[selected_columns_qr]

            elif case == 'qr_initialization':
                to_sample = valid[ED.greedy_stepwise_selection_with_time(Y=Y,
                                                            t=t_predicted[valid],
                                                            initialization=selected_columns_qr,
                                                            t_elapsed=t_sum,
                                                            t_max=runtime_limit/8,
                                                            idx_to_exclude=None)]
            # TODO: check if Y is rank-deficient, i.e. will ED problem fail

            if self.verbose:
                print(t_predicted[to_sample])

            if np.isnan(to_sample).any():
                to_sample = np.argsort(t_predicted)[:ranks[0]]

            if len(to_sample) == 0 and len(self.sampled_indices) == 0:
                # if no columns are selected in first iteration (log det instability), sample n fastest columns
                n = len(np.where(np.cumsum(np.sort(t_predicted)) <= runtime_limit/4)[0])
                if n > 0:
                    to_sample = np.argsort(t_predicted)[:n]
                else:
                    self.ensemble.fitted = False
                    return

            start = time.time()
            if self.selection_method != 'random':
                candidate_indices = []
                # we only need to fit models on the new dataset if it has not been fitted already
                to_sample = list(set(to_sample) - self.sampled_indices)
                if self.verbose:
                    print('Sampling {} entries of new row...'.format(len(to_sample)))

    #             return to_sample

                sampled_pipelines_single_round = [PipelineObject(p_type=self.p_type, config=self.pipeline_settings_on_dataset[i], index=i, verbose=self.verbose) for i in to_sample]

                p1 = mp.Pool(self.n_cores)
                sampled_pipeline_errors_single_round = [p1.apply_async(PipelineObject.kfold_fit_validate, args=[p, x_train, y_train, categorical, self.n_folds, runtime_limit/4, self.random_state]) for p in sampled_pipelines_single_round]
                p1.close()
                p1.join()

                if self.verbose:
                    print("pool fitting completed")

                # predict performance of models not actually fitted on the new dataset

                for i, error in enumerate(sampled_pipeline_errors_single_round):
                    cv_error, cv_predictions, t_elapsed = error.get()

                    if not np.isnan(cv_error):
                        sampled_pipelines_single_round[i].cv_error, sampled_pipelines_single_round[i].cv_predictions = cv_error, cv_predictions
                        sampled_pipelines_single_round[i].sampled = True
                        self.new_row[:, to_sample[i]] = cv_error
                        self.sampled_pipelines[to_sample[i]] = sampled_pipelines_single_round[i]
                        self.sampled_indices = self.sampled_indices.union(set([to_sample[i]]))
                        candidate_indices.append(to_sample[i])
                        self._t_predicted[to_sample[i]] = t_elapsed
                    else:
                        self._t_predicted[to_sample[i]] = max(t_elapsed, self._t_predicted[to_sample[i]])

                self.new_row_pred = impute_with_coefficients(self.Y[:ranks[0], :], self.new_row, list(self.sampled_indices))

                for idx in np.argsort(self.new_row[0, :])[:5]: # np.argsort automatically put nans at the end of the list
                    if not np.isnan(self.new_row[0, idx]):
                        if self.verbose:
                            print(self.sampled_pipelines[idx])
                        self.ensemble.candidate_learners.append(self.sampled_pipelines[idx])

                # currently disabled: impute ALL entries
                # unknown = sorted(list(set(range(self.new_row.shape[1])) - self.sampled_indices))
                # self.new_row[:, unknown] = imputed[:, unknown]

                # k-fold fit candidate learners of ensemble
                remaining = (runtime_limit - (time.time() - start)) * self.n_cores
                first = (case == 'qr_initialization') and not self.ever_once_selected_best

                if remaining > 0 or first:
                # add models predicted to be the best to list of candidate learners
                    if self.verbose:
                        if remaining < 0 and first:
                            print("Insufficient time in this doubling round, but we add models predicted to be the best at least once.")
                        print("length of sampled indices: {}".format(len(self.sampled_indices)))

        #             self.ensemble.candidate_learners.append(self.sampled_pipelines[best_sampled_idx])
                    for i in np.argsort(self.new_row_pred[0]):
                        if (first and len(candidate_indices) <= 3) or t_predicted[i] + t_predicted[candidate_indices].sum() <= remaining / 4:
                            candidate_indices.append(i)
                            # if model has already been k-fold fitted, immediately add to candidate learners
                            if i in self.sampled_indices:
                                assert self.sampled_pipelines[i] is not None
                                self.ensemble.candidate_learners.append(self.sampled_pipelines[i])
                    # candidate learners that need to be k-fold fitted
                    to_fit = list(set(candidate_indices) - self.sampled_indices)
                    if self.verbose:
                        print("{} candidate learners need to be k-fold fitted".format(to_fit))
                else:
                    if self.verbose:
                        print("Insufficient time in this doubling round.")
            else:
                remaining = (runtime_limit - (time.time() - start)) * self.n_cores
                to_fit = to_sample.copy()

            remaining = (runtime_limit - (time.time() - start)) * self.n_cores

            if remaining > 0 or first:
                if len(to_fit) > 0:
                    # fit models predicted to have good performance and thus going to be added to the ensemble
                    if self.verbose:
                        print("Fitting {} pipelines predicted to be the best ...".format(len(to_fit)))

                    candidate_pipelines = [PipelineObject(p_type=self.p_type, config=self.pipeline_settings_on_dataset[i], index=i, verbose=self.verbose) for i in to_fit]

    #                 print(remaining_global)
                    p2 = mp.Pool(self.n_cores)
                    candidate_pipeline_errors = [p2.apply_async(PipelineObject.kfold_fit_validate, args=[p, x_train, y_train, categorical, self.n_folds, remaining_global/2, self.random_state]) for p in candidate_pipelines] # set a not-quite-small limit for promising models
                    p2.close()
                    p2.join()

                    for i, error in enumerate(candidate_pipeline_errors):
                        cv_error, cv_predictions, t_elapsed = error.get()

                        if not np.isnan(cv_error):
                            candidate_pipelines[i].cv_error, candidate_pipelines[i].cv_predictions = cv_error, cv_predictions
                            candidate_pipelines[i].sampled = True
                            self.new_row[:, to_fit[i]] = cv_error
                            self.sampled_pipelines[to_fit[i]] = candidate_pipelines[i]
                            self.ensemble.candidate_learners.append(candidate_pipelines[i])
                            # update sampled indices
                            self.sampled_indices = self.sampled_indices.union(set([to_fit[i]]))
                            self._t_predicted[to_fit[i]] = t_elapsed
                        else:
                            self._t_predicted[to_fit[i]] = max(t_elapsed, self._t_predicted[to_fit[i]])

    #                 self.new_row = impute(self.error_matrix, self.new_row, list(self.sampled_indices), rank=rank)
                    self.ever_once_selected_best = True

            if self.verbose:
                print("Number of candidate learners in the ensemble: {}".format(len(self.ensemble.candidate_learners)))

            if len(self.ensemble.candidate_learners) > 0:
                self.ensemble.fitted = True

                if self.verbose:
                    print('\nFitting ensemble of maximum size {}...'.format(len(self.ensemble.candidate_learners)))
                # ensemble selection and fitting in the remaining time budget
                self.ensemble.fit(x_train, y_train, categorical, remaining, self.sampled_pipelines)
                for pipeline in self.ensemble.base_learners:
                    assert pipeline.index is not None
                    self.sampled_indices.add(pipeline.index)
                    self.sampled_pipelines[pipeline.index] = pipeline

                if self.verbose:
                    print('\nAutoLearner fitting complete.')
            else:
                if self.verbose:
                    print("Insufficient time in this round.")

    def fit(self, x_train, y_train, categorical=None):

        """Fit an AutoLearner object, iteratively doubling allowed runtime, and terminate when reaching the time limit."""

        if self.method == 'oboe':
            num_points, num_features = x_train.shape

            if self.verbose:
                print('\nShape of training dataset: {} data points, {} features'.format(num_points, num_features))

            if num_points > 10000 and num_points / num_features > self.dataset_ratio_threshold:
                num_points_new = int(min(5000, num_features * self.dataset_ratio_threshold))
                sampling_ratio = num_points_new / num_points
                print(sampling_ratio)
                df_x_train = pd.DataFrame(x_train)
                df_y_train = pd.DataFrame(y_train, columns=['labels'])
                df_resampled = df_x_train.join(df_y_train).groupby('labels').apply(pd.DataFrame.sample, frac=sampling_ratio).reset_index(drop=True)
                x_train = df_resampled.drop(['labels'], axis=1).values
                y_train = df_resampled['labels'].values
                if self.verbose:
                    print('\nTraining dataset resampled \nShape of resampled training dataset: {} data points, {} features'.format(x_train.shape[0], x_train.shape[1]))

            t_predicted = convex_opt.predict_runtime(x_train.shape, model_name=self.runtime_predictor_algorithm, method=self.method)

            # split data into training and validation sets
            try:
                x_tr, x_va, y_tr, y_va = train_test_split(x_train, y_train, test_size=0.15, stratify=y_train, random_state=0)
            except ValueError:
                x_tr, x_va, y_tr, y_va = train_test_split(x_train, y_train, test_size=0.15, random_state=0)

            ranks = [approx_matrix_rank(self.error_matrix, threshold=0.05)]
            if self.build_ensemble:
                t_init = 2**np.floor(np.log2(np.sort(t_predicted)[:int(1.1*ranks[0])].sum()))
                t_init = max(1, t_init)
            else:
                t_init = self.runtime_limit / 2
            times = [t_init]
            losses = [0.5]

            # predict runtime for the training set of the new dataset
            t_predicted = convex_opt.predict_runtime(x_train.shape, runtime_matrix=self.runtime_matrix, method='oboe')

            e_hat, e_hat_pred, actual_times, sampled, ensembles = [], [], [], [], []

            start = time.time()

            def doubling():
                k, t = ranks[0], times[0]
                counter, self.best = 0, 0
                while time.time() - start < self.runtime_limit - t:
                    if self.verbose:
                        print('Fitting with k={}, t={}'.format(k, t))
#                     if self.build_ensemble:
#                         self.ensemble = Ensemble(self.p_type, self.ensemble_method, self.stacking_hyperparams)
#                     else:
#                         self.ensemble = Model_collection(self.p_type)
                    self._fit(x_tr, y_tr, categorical=None, ranks=k, runtime_limit=t, t_predicted=t_predicted)
                    if self.build_ensemble and self.ensemble.fitted:
                        if self.verbose:
                            print("\nGot a new ensemble in the round with rumtime target {} seconds".format(t))
                        loss = error(y_va, self.ensemble.predict(x_va), self.p_type)

                        # TEMPORARY: Record intermediate results

                        e_hat.append(np.copy(self.new_row))
                        e_hat_pred.append(np.copy(self.new_row_pred))
                        actual_times.append(time.time() - start)
                        sampled.append(self.sampled_indices)
                        ensembles.append(self.ensemble)
                        losses.append(loss)

                        if loss == min(losses):
                            ranks.append(k+1)
                            self.best = counter
                        else:
                            ranks.append(k)

                        counter += 1

                    times.append(2*t)
                    k = ranks[-1]
                    t = times[-1]

            class TimeoutException(Exception): pass

            @contextmanager
            def time_limit(seconds):
                def signal_handler(signum, frame):
                    raise TimeoutException("Time limit reached.")
                signal.signal(signal.SIGALRM, signal_handler)
                signal.alarm(seconds)
                try:
                    yield
                finally:
                    signal.alarm(0)

            try:
                # set aside 3 seconds for initial and final processing steps
                with time_limit(self.runtime_limit - 3):
                    doubling()
            except TimeoutException as e:
                if self.verbose:
                    print("Time limit reached.")

        elif self.method == 'tensoroboe':
            num_points, num_features = x_train.shape

            self.ever_once_selected_best = False

            if self.verbose:
                print('\nShape of training dataset: {} data points, {} features'.format(num_points, num_features))

            if num_points > 10000 and num_points / num_features > self.dataset_ratio_threshold:
                num_points_new = int(min(5000, num_features * self.dataset_ratio_threshold))
                sampling_ratio = num_points_new / num_points
                if self.verbose:
                    print("dataset too skewed in having too many data points; sampling data points with sampling ratio {}".format(sampling_ratio))
                df_x_train = pd.DataFrame(x_train)
                df_y_train = pd.DataFrame(y_train, columns=['labels'])
                df_resampled = df_x_train.join(df_y_train).groupby('labels').apply(pd.DataFrame.sample, frac=sampling_ratio).reset_index(drop=True)
                x_train = df_resampled.drop(['labels'], axis=1).values
                y_train = df_resampled['labels'].values
                if self.verbose:
                    print('\nTraining dataset resampled \nShape of resampled training dataset: {} data points, {} features'.format(x_train.shape[0], x_train.shape[1]))

            if self.verbose:
                print("Splitting training set into training and validation ..")

            # split data into training and validation sets
    #         try:
    #             x_tr, x_va, y_tr, y_va = train_test_split(x_train, y_train, test_size=0.2, stratify=y_train, random_state=0)
    #         except ValueError:
    #             x_tr, x_va, y_tr, y_va = train_test_split(x_train, y_train, test_size=0.2, random_state=0)

            # for now, do not do validation, but instead do cross-validation
            x_tr = x_train
            x_va = x_train
            y_tr = y_train
            y_va = y_train

            num_points_tr, num_features_tr = x_tr.shape

            self._t_predicted = np.maximum(self._predict_runtime(x_tr), 0.5)

            def p2f(x):
                return float(x.strip('%'))/100

            # pipeline settings on dataset
            self.pipeline_settings_on_dataset = []

            for item in self.pipeline_settings:
                item_copy = copy.deepcopy(item)
                if item_copy['dim_reducer']['algorithm'] == 'PCA':
                    item_copy['dim_reducer']['hyperparameters']['n_components'] = int(min((self.n_folds - 1) * num_points_tr/self.n_folds, p2f(item_copy['dim_reducer']['hyperparameters']['n_components']) * num_features_tr))
                elif item_copy['dim_reducer']['algorithm'] == 'SelectKBest':
                    item_copy['dim_reducer']['hyperparameters']['k'] = int(p2f(item_copy['dim_reducer']['hyperparameters']['k']) * num_features_tr)

                self.pipeline_settings_on_dataset.append(item_copy)

            start = time.time()

            ranks = [approx_tensor_rank(self.error_tensor_imputed, threshold=0.05)]

            t_init = 2**np.floor(np.log2(np.sort(self._t_predicted)[:int(5*ranks[0][-1])].sum()))
    #         t_init = max(2**np.floor(np.log2(self.runtime_limit/2)), t_init) # better on large datasets
            t_init = max(2**np.floor(np.log2(self.runtime_limit/8)), t_init)

            if self.verbose:
                print("runtime limit of initial round: {} seconds".format(t_init))
            times = [t_init]
            losses = [0.5]

            # fit and validate the best-on-average pipeline
            if self.verbose:
                print("fitting and kfold_fit_validating the best-on-average pipeline")
            best_on_average_pipeline = PipelineObject(
                p_type=self.p_type, config=self.pipeline_settings_on_dataset[self.best_on_average_pipeline_index],
                index=self.best_on_average_pipeline_index, verbose=self.verbose)
            self.ensemble.base_learners.append(best_on_average_pipeline)
            self.ensemble.fit(x_tr, y_tr, categorical)
            loss = self.ensemble.kfold_fit_validate(x_va, y_va, categorical, n_folds=self.n_folds, timeout=t_init)[0]
            losses.append(loss)

            e_hat, e_hat_pred, actual_times, sampled, ensembles = [], [], [], [], []

            if self.verbose:
                print("Doubling process started ...")
            def doubling():
                k, t = ranks[0], times[0]
                counter, self.best = 0, 0
                while time.time() - start < self.runtime_limit - t:
                    if self.verbose:
                        print('Fitting with ranks={}, t={}'.format(k, t))
    #                 if self.build_ensemble:
    #                     self.ensemble = Ensemble(self.p_type, self.ensemble_method, self.stacking_hyperparams)
    #                 else:
    #                     self.ensemble = Model_collection(self.p_type)
                    self._fit(x_tr, y_tr, categorical, self._t_predicted, ranks=k, runtime_limit=t, remaining_global=self.runtime_limit-time.time()+start)
                    if self.build_ensemble and self.ensemble.fitted:
                        if self.verbose:
                            print("\nGot a new ensemble in the round with runtime target {} seconds".format(t))
    #                     loss = error(y_va, self.ensemble.predict(x_va), self.p_type)
                        loss = self.ensemble.kfold_fit_validate(x_va, y_va, categorical, n_folds=self.n_folds, timeout=(self.runtime_limit-time.time()+start)/2)[0]

                        # TEMPORARY: Record intermediate results
                        e_hat.append(np.copy(self.new_row))
                        e_hat_pred.append(np.copy(self.new_row_pred))
                        actual_times.append(time.time() - start)
                        sampled.append(self.sampled_indices)
                        ensembles.append(self.ensemble)
                        losses.append(loss)

                        if loss == min(losses):
                            rank_new = (k[0], k[1], k[2], k[3], k[4], k[5]+1)
                            if self.verbose:
                                print("new approximate rank for the error tensor: {}".format(rank_new))
                            ranks.append(rank_new)
                            self.best = counter
                        else:
                            ranks.append(k)

                        counter += 1

                    times.append(2*t)
                    k = ranks[-1]
                    t = times[-1]

            class TimeoutException(Exception): pass

            @contextmanager
            def time_limit(seconds):
                def signal_handler(signum, frame):
                    raise TimeoutException("Time limit reached.")
                signal.signal(signal.SIGALRM, signal_handler)
                signal.alarm(seconds)
                try:
                    yield
                finally:
                    signal.alarm(0)

            try:
                # set aside 3 seconds for initial and final processing steps
                with time_limit(self.runtime_limit - 3):
                    doubling()
            except TimeoutException as e:
                if self.verbose:
                    print("Time limit reached.")

        if not self.ensemble.fitted:
            if self.build_ensemble and self.fit_ensemble_despite_timeout and self.verbose:
                print('\n Ensemble not fitted in the time limit. Fitting ensemble of max size {}...'.format(len(self.ensemble.candidate_learners)))
            # ensemble selection and fitting in the remaining time budget
            if (not self.build_ensemble) or (self.build_ensemble and self.fit_ensemble_despite_timeout):
                self.ensemble.fit(x_train, y_train, runtime_limit=None, fitted_base_learners=self.fitted_models)
                for model in self.ensemble.base_learners:
                    assert model.index is not None
                    self.fitted_indices.add(model.index)
                    self.fitted_models[model.index] = model
                self.ensemble.fitted = True

        if self.build_ensemble:
            # after all iterations, restore best model

            try:
                self.new_row = e_hat[self.best]
                self.ensemble = ensembles[self.best]
                self.predict_with_most_common_class = False

                return {'ranks': ranks[:-1], 'runtime_limits': times[:-1], 'validation_loss': losses,
                    'filled_new_row': e_hat, 'predicted_new_row': e_hat_pred, 'actual_runtimes': actual_times, 'sampled_indices': sampled, 'models': ensembles}
            except IndexError:
                self.y_train_mode = mode(y_train.flatten())[0][0]
                if not self.ensemble.fitted:
                    print("No ensemble built within time limit. The auto-learner is set to predict by the mode of class labels in the training set. Please try increasing the time limit or allocate more computational resources.")
                    self.predict_with_most_common_class = True
                else:
                    self.predict_with_most_common_class = False

    def predict(self, x_test):
        """Generate predictions on test data.

        Args:
            x_test (np.ndarray): Features of the test dataset.
        Returns:
            np.ndarray: Predicted labels.
        """
        if self.build_ensemble:
            if self.predict_with_most_common_class:
                return np.full(x_test.shape[0], self.y_train_mode)
            else:
                return self.ensemble.predict(x_test)
        else: # just select a collection of promising models
            return self.ensemble.predict(x_test) # the self.ensemble object here is a Model_collection

    def _predict_runtime(self, x_train):
        # predict runtime for the training set of the new dataset.
        if self.verbose:
            print("Predicting pipeline running time ..")
        return convex_opt.predict_runtime(x_train.shape, saved_model='Class', model=self.runtime_predictor)

# The code below is deprecated
    def refit(self, x_train, y_train):
        """Refit an existing AutoLearner object on a new dataset. This will simply retrain the base-learners and
        stacked learner of an existing model, and so algorithm and hyperparameter selection may not be optimal.
        Args:
            x_train (np.ndarray): Features of the training dataset.
            y_train (np.ndarray): Labels of the training dataset.
        """
        assert self.ensemble.fitted, "Cannot refit unless model has been fit."
        self.ensemble.refit(x_train, y_train)

    def get_models(self):
        """Get details of the selected machine learning models and the ensemble.
        """
        return self.ensemble.get_models()

    def get_pipeline_accuracies(self, y_test):
        """
        Get accuracies of selected pipelines.
        """
        return self.ensemble.get_pipeline_accuracies(y_test)

    def get_model_accuracies(self, y_test):
        """
        Get accuracies of selected pipelines.
        """
        return self.get_pipeline_accuracies(y_test)

    def _greedy_initial_selection(self, x_train, y_train, t_predicted, runtime_limit):
        if self.verbose:
            print("Fitting fast pipelines that perform well on average.")

        candidate_pipeline_indices = list(set(np.where(t_predicted <= runtime_limit / 8)[0]).intersection(
            set(np.argsort(np.nanmedian(tl.unfold(self.error_tensor_imputed, mode=0), axis=0))[
                :int(len(self.pipeline_settings_on_dataset)/50)])))

        candidate_pipelines = [PipelineObject(p_type=self.p_type, config=self.pipeline_settings_on_dataset[i], index=i, verbose=self.verbose) for i in candidate_pipeline_indices]

        p1 = mp.Pool(self.n_cores)
        candidate_pipeline_errors = [p1.apply_async(PipelineObject.kfold_fit_validate, args=[p, x_train, y_train, self.n_folds, runtime_limit/4, self.random_state]) for p in candidate_pipelines]
        p1.close()
        p1.join()

        if self.verbose:
            print("Initial greedy fitting completed")

        for i, error in enumerate(candidate_pipeline_errors):
            cv_error, cv_predictions, t_elapsed = error.get()
            if not np.isnan(cv_error):
                candidate_pipelines[i].cv_error, candidate_pipelines[i].cv_predictions = cv_error, cv_predictions
                candidate_pipelines[i].sampled = True
                self.new_row[:, candidate_pipeline_indices[i]] = cv_error
                self.sampled_pipelines[candidate_pipeline_indices[i]] = candidate_pipelines[i]
                self.ensemble.candidate_learners.append(candidate_pipelines[i])
                # update sampled indices
                self.sampled_indices = self.sampled_indices.union(set([candidate_pipeline_indices[i]]))
                self._t_predicted[candidate_pipeline_indices[i]] = t_elapsed
            else:
                self._t_predicted[candidate_pipeline_indices[i]] = max(t_elapsed, self._t_predicted[candidate_pipeline_indices[i]])

        if len(self.ensemble.candidate_learners) > 0:
            self.ensemble.fitted = True
            self.ensemble.fit(x_train, y_train)
        else:
            if self.verbose:
                print("Insufficient time to fit fast and on average best-performing pipelines.")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @Time       : 2020/4/13 17:16
# @Author     : ShadowY
# @File       : get_juejin_info.py
# @Software   : PyCharm
# @Version    : 1.0
# @Description: 获取掘金后端信息
import requests
import json
import pymongo

header = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.17 Safari/537.36',
    'Referer': 'https://juejin.im/',
    'X-Agent': 'Juejin/Web',
    'Content-Type': 'application/json'
}
payload = {"operationName": "",
           "query": "",
           "variables": {
               "tags": [],
               "category": "5562b419e4b00c57d9b94ae2",
               "first": 20,
               "after": "",
               "order": "POPULAR"},
           "extensions": {
               "query": {
                   "id": "653b587c5c7c8a00ddf67fc66f989d42"
               }
           }
           }

def get_one_page(page_url, count=9999):
    """
    获取掘金一页信息
    :param page_url:
    :param count:
    :return:
    """
    infos = list()
    while True:
        res = requests.post(page_url, headers=header, data=json.dumps(payload))
        res.encoding = 'utf-8'
        html = res.text
        p_json = json.loads(html)['data']['articleFeed']['items']
        count -= 1
        if count == 0 or not p_json['pageInfo']['hasNextPage']:  # 判断是否结束
            break
        payload['variables']['after'] = p_json['pageInfo']['endCursor']
        datas = p_json['edges']
        for data in datas:
            info = {
                'comment': data['node']['commentsCount'],  # 评论数
                'title': data['node']['title'],  # 标题
                'likes': data['node']['likeCount'],  # 点赞数
                'update': data['node']['updatedAt'],  # 更新时间
                'username': data['node']['user']['username'],  # 用户名称
                'cont_url': data['node']['originalUrl'],  # 文章链接
            }
            print(info)
            infos.append(info)
    return infos

def write2mongodb(data):
    """
    写入mongodb数据库
    :param data: 列表数据
    :return:
    """
    db_client = pymongo.MongoClient(host='localhost', port=27017)
    db = db_client['tkh']
    jue = db['juejin']
    jue.insert_many(data)

if __name__ == '__main__':
    url = 'https://web-api.juejin.im/query'
    all_data = get_one_page(url, 20)
    write2mongodb(all_data)
    print('写入数据库成功')


import os
import re
from pathlib import Path

import bson
import pandas as pd
from bs4 import BeautifulSoup as bs
from git import Repo
from loguru import logger
from markdownify import markdownify
from tqdm import tqdm

def main():
    # https://podatki.gov.si/dataset/osnovni-podatki-o-predpisih-rs
    vpliva_na = pd.read_csv("data/vplivana.csv")
    osnovni_raw = pd.read_csv("data/osnovni.csv")
    osnovni = osnovni_raw.dropna(subset=["D_SPREJEMA"]).copy()
    osnovni["date_accepted"] = pd.to_datetime(osnovni["D_SPREJEMA"], format="%d.%m.%y")
    # https://podatki.gov.si/dataset/neuradna-preciscena-besedila-predpisov
    with open("data/vsebina.bson/pisrs/vsebina.bson", "rb") as f:
        laws = bson.decode_all(f.read())

    laws_list = [law["idPredpisa"] for law in laws]
    laws_changes = [law["idPredpisaChng"] for law in laws]

    law_id = "ZAKO4697"
    law_code = get_law_code(law_id, osnovni)
    affected_ids = vpliva_na[vpliva_na.VPLIVA_NA == law_id]
    affected = osnovni[
        ((osnovni.ID.isin(affected_ids.ID)) & (osnovni.KRATICA.str.contains(law_code)))
        | (osnovni.ID == law_id)
    ]
    affected_laws = affected[affected.ID.str.startswith("ZAKO")].sort_values(
        "date_accepted"
    )

    data_location = "/tmp/slovenian_laws/"
    os.mkdir(data_location)
    repo = Repo.init(data_location)
    for _, affected_law_row in tqdm(affected_laws.iterrows()):
        #  affected_law_id = affected_laws["ID"].iloc[0]
        affected_law_id = affected_law_row["ID"]
        affected_law_title = affected_law_row["NASLOV"]
        affected_law_code = affected_law_row["KRATICA"]
        commit_msg = (
            affected_law_code + " - " + affected_law_id + " - " + affected_law_title
        )
        affected_law_date = affected_law_row["date_accepted"]
        affected_law = get_law(affected_law_id, laws)
        if not affected_law:
            continue
        vsebina = affected_law["vsebina"]
        vsebina_clean = re.sub(r"( |\n|\r)+", " ", vsebina)

        soup = bs(vsebina_clean)  # make BeautifulSoup
        prettyHTML = soup.prettify()
        #  vsebina_md = markdownify(vsebina, convert=["style"])
        #  vsebina_md = re.sub(
        #      "/(<!--.*?-->)|(<!--[\S\s]+?-->)|(<!--[\S\s]*?$)/g", "", vsebina_md
        #  )
        #  Path(data_location + law_code + ".md").write_text(vsebina_md)
        Path(data_location + law_code + ".html").write_text(prettyHTML)
        repo.git.add(all=True)
        repo.git.commit(date=affected_law_date, m=commit_msg)

def get_law(affected_law_id, laws):
    laws_list = [law["idPredpisa"] for law in laws]
    if affected_law_id in laws_list:
        law_number = laws_list.index(affected_law_id)
        law = laws[law_number]
    else:
        logger.warning(f"No law available for {affected_law_id}")
        law = ""
    return law

def get_law_code(law_id, osnovni):
    law_code = osnovni[osnovni.ID == law_id].iloc[0].KRATICA
    return law_code

def get_law_id(law_code, osnovni):
    law_id = osnovni[osnovni.KRATICA == law_code].iloc[0].ID
    return law_id

def get_markdown(vsebina):
    vsebina_md = markdownify(vsebina)
    return vsebina_md

if __name__ == "__main__":
    main()


from typing import List
import numpy as np

def sample_array(arr, rate):
    arr = np.array(arr)
    if rate < 1:
        rate = int(1 / rate)
        num_samples = rate - 1
        sampled = np.hstack(
            [
                np.linspace(arr[i], arr[i + 1], num_samples + 2, endpoint=False)[1:]
                for i in range(len(arr) - 1)
            ]
        )
        sampled = np.hstack([arr, sampled])
        sampled.sort()
    else:
        sampled = arr[::rate]
    return sampled

def as_beats(frames):
    frames = [f"{frame}:({mag})" for frame, mag in frames]
    return ",".join(frames)

class BeatVisuals:

    def __init__(self, beats, times, fps=24):
        self.fps = fps
        self.beats = times[beats]
        self.beats = np.insert(self.beats, 0, 0)
        self.max_frames = int(max(times) * fps)

    # times are in sec
    # adsr_times = decay, release
    # adsr_mags = on_value, decay_value, off_value
    def beats_deforum(
        self, decay: float, adsr_mags: List[float], every_n: int = 1
    ):
        on_value, off_value = adsr_mags
        frames = []
        beats = sample_array(self.beats, every_n)
        for i in range(0, len(beats)):
            time = round(beats[i], 3)
            keyframes = np.transpose(
                (
                    [time, time + decay],
                    [on_value, off_value],
                )
            )
            frames.extend([(int(k[0] * self.fps), k[1]) for k in keyframes])

        frames = sorted(frames, key=lambda x: x[0])

        # remove duplicate time entries
        found_keyframes = set()
        for kf, val in frames:
            if kf in found_keyframes:
                frames.remove((kf, val))
            else:
                found_keyframes.add(kf)
        return frames

    def as_xy(self, frames):
        frames = np.array(frames)
        x, y = frames[:, 0], frames[:, 1]
        x = x / self.fps
        return x, y

    def beats_prompts(self, prompts):
        prompt_idx = 0
        res = {0: prompts[0]}
        for i in range(len(self.beats)):
            time = round(self.beats[i], 3)
            frame = int(time * self.fps)
            prompt_idx = 0 if prompt_idx == len(prompts) - 1 else prompt_idx + 1
            res[frame] = prompts[prompt_idx]
        return res


import numpy as np
import matplotlib
import matplotlib.pyplot as plt

import azureml
from azureml.core import Workspace, Run
from azureml.core import Workspace
from azureml.core.model import Model
import os
import pickle
from sklearn.externals import joblib
from utils import load_data

from sklearn.metrics import confusion_matrix
# display the core SDK version number
print("Azure ML SDK Version: ", azureml.core.VERSION)

ws = Workspace.from_config()
model=Model(ws, 'sklearn_mnist')

model.download(target_dir=os.getcwd(), exist_ok=True)

# verify the downloaded model file
file_path = os.path.join(os.getcwd(), "sklearn_mnist_model.pkl")

os.stat(file_path)
data_folder = os.path.join(os.getcwd(), 'data')
# note we also shrink the intensity values (X) from 0-255 to 0-1. This helps the neural network converge faster
X_test = load_data(os.path.join(data_folder, 'test-images.gz'), False) / 255.0
y_test = load_data(os.path.join(data_folder, 'test-labels.gz'), True).reshape(-1)

clf = joblib.load( os.path.join(os.getcwd(), 'sklearn_mnist_model.pkl'))
y_hat = clf.predict(X_test)

conf_mx = confusion_matrix(y_test, y_hat)
print(conf_mx)
print('Overall accuracy:', np.average(y_hat == y_test))
# normalize the diagonal cells so that they don't overpower the rest of the cells when visualized
row_sums = conf_mx.sum(axis=1, keepdims=True)
norm_conf_mx = conf_mx / row_sums
np.fill_diagonal(norm_conf_mx, 0)

fig = plt.figure(figsize=(8,5))
ax = fig.add_subplot(111)
cax = ax.matshow(norm_conf_mx, cmap=plt.cm.bone)
ticks = np.arange(0, 10, 1)
ax.set_xticks(ticks)
ax.set_yticks(ticks)
ax.set_xticklabels(ticks)
ax.set_yticklabels(ticks)
fig.colorbar(cax)
plt.ylabel('true labels', fontsize=14)
plt.xlabel('predicted values', fontsize=14)
plt.savefig('conf.png')
plt.show()


import os
import argparse
import logging
import re
from pprint import pprint

class HackAssembler:
    output_file_path = ""
    variable_address_base = 16 #hard-coded
    variable_address_offset = 0
    lable_offset = 0
    symbol_table = {
        "R0": 0,
        "R1": 1,
        "R2": 2,
        "R3": 3,
        "R4": 4,
        "R5": 5,
        "R6": 6,
        "R7": 7,
        "R8": 8,
        "R9": 9,
        "R10": 10,
        "R11": 11,
        "R12": 12,
        "R13": 13,
        "R14": 14,
        "R15": 15,
        "SCREEN": 16384,
        "KBD": 24576,
        "SP": 0,
        "LCL": 1,
        "ARG": 2,
        "THIS": 3,
        "THAT": 4,
    }

    comp_table = {
        # a = 0
        "0": "0101010",
        "1": "0111111",
        "-1": "0111010",
        "D": "0001100",
        "A": "0110000",
        "!D": "0001101",
        "!A": "0110001",
        "-D": "0001111",
        "-A": "0110011",
        "D+1": "0011111",
        "A+1": "0110111",
        "D-1": "0001110",
        "A-1": "0110010",
        "D+A": "0000010",
        "D-A": "0010011",
        "A-D": "0000111",
        "D&A": "0000000",
        "D|A": "0010101",

        #a = 1
        "M": "1110000",
        "!M": "1110001",
        "-M": "1110011",
        "M+1": "1110111",
        "M-1": "1110010",
        "D+M": "1000010",
        "D-M": "1010011",
        "M-D": "1000111",
        "D&M": "1000000",
        "D|M": "1010101",
    }

    dest_table = {
        "null": "000",
        "M": "001",
        "D": "010",
        "MD": "011",
        "A": "100",
        "AM": "101",
        "AD": "110",
        "AMD": "111",
    }

    jump_table = {
        "null": "000",
        "JGT": "001",
        "JEQ": "010",
        "JGE": "011",
        "JLT": "100",
        "JNE": "101",
        "JLE": "110",
        "JMP": "111",
    }

    src_lines = []
    out_lines = []
    logger = None

    def __init__(self, src_file_path, output_file_path):
        print("Initializing Hack Assembler")
        print("source file: " + src_file_path)
        print("output file: " + output_file_path)

        self.output_file_path = output_file_path

        # print("symbol table:")
        # print(self.symbol_table)

        self.logger = logging.getLogger()
        handler = logging.StreamHandler()
        self.logger.addHandler(handler)
        self.logger.setLevel(logging.DEBUG)

        with open(src_file_path, "r") as f:
            self.src_lines = [line.strip().split("//")[0] for line in f.readlines() if ((not line.strip().startswith(r"//")) and line.strip())]

        self.logger.debug("cleaned source:")
        line_number = 0
        for line in self.src_lines:
            self.logger.debug("{:3}\t{}".format(line_number, line))
            line_number += 1

    def parse_variable(self, line):
        variable_pattern = re.compile("@([\S]+)")
        m = re.match(variable_pattern, line)

        if m is None:
            return False

        variable = m.group(1)
        assert (not variable[0].isdigit())
        if variable not in self.symbol_table:
            v_addr = self.variable_address_base + self.variable_address_offset
            self.logger.debug('found new variable: "{}", add to symbol tabel with address: {}'
                              .format(variable, v_addr))
            self.variable_address_offset += 1

        return True

    def extract_label(self):
        temp_lines = []
        line_number = 0
        for line in self.src_lines:
            m = re.match("\((.+)\)", line)
            if m is not None:
                label = m.group(1)
                if label not in self.symbol_table:
                    self.symbol_table[label]  = line_number
                    self.logger.debug('new label "{}" added to symbol table with value {}'
                                      .format(label, line_number))
            else:
                line_number += 1
                temp_lines.append(line)

        self.src_lines = temp_lines
        self.logger.debug("updated symbol table:")
        self.logger.debug(self.symbol_table)

    def handle_lines(self):
        temp_lines = []
        for line in self.src_lines:
            if line.startswith("@"):
                line  = self.handle_A_instruction(line)
            else:
                line = self.handle_C_instruction(line)

            temp_lines.append(line)

        self.out_lines = temp_lines
        self.logger.debug("assemble finished!")
        for src, out in zip(self.src_lines, self.out_lines):
            self.logger.debug("src: {}, assemble: {}".format(src, out))

    def add_variable_to_symbol_table(self, var):
        var_addr = self.variable_address_base + self.variable_address_offset
        self.symbol_table[var] = var_addr
        self.variable_address_offset += 1

    def handle_A_instruction(self, line):
        result = ""
        line = line.strip()
        var = line[1:]
        if not var.isdigit():
            if var not in self.symbol_table:
                self.add_variable_to_symbol_table(var)

            result = self.decimal_to_binary(self.symbol_table[var], 15)
        else:
            result = self.decimal_to_binary(var, 15)

        result = "0" + result
        self.logger.debug("A instruction: {}, result = {}, length = {}".format(line, result, len(result)))
        return result

    def handle_C_instruction(self, line):
        result = "111"
        dest = "null"
        comp = None
        jump = "null"

        assert ("=" in line) or (";" in line)
        temp = re.split("=|;", line)
        if len(temp) == 3:
            dest = temp[0].strip()
            comp = temp[1].strip()
            jump = temp[2].strip()
        elif len(temp) == 2:
            #print(temp)
            if "=" in line:
                dest = temp[0].strip()
                comp = temp[1].strip()
            else:
                comp = temp[0].strip()
                jump = temp[1].strip()
        else:
            comp = temp[0].strip()

        #print("line: {}, dest: {}, comp: {}, jump: {}".format(line, dest, comp, jump))
        result = result + self.comp_table[comp] + self.dest_table[dest] + self.jump_table[jump]
        self.logger.debug("C-line: {}, result: {}".format(line, result))
        assert len(result) == 16
        return result

    def decimal_to_binary(self, value, width):
        #return str(bin(n).replace("0b", ""))
        value = int(value)
        format_str = "{0:0" + str(width) + "b}"
        return format_str.format(value)

    def output_file(self):
        with open(self.output_file_path, "w") as f:
            for line in self.out_lines:
                f.write(line + "\n")

    def assemble(self):
        self.extract_label()
        self.logger.debug("source code after label extraction:")
        for i, line in  enumerate(self.src_lines):
            self.logger.debug("{:3} {}".format(i, line))

        self.handle_lines()
        self.output_file()

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--src', type=str, required=True)
    parser.add_argument('--out', type=str, required=True)
    args = parser.parse_args()
    return args

if __name__ == "__main__":
    args = parse_args()
    print(args)
    assembler = HackAssembler(args.src, args.out)
    assembler.assemble()


""" Attempt at replacing eigen() with scipy's eig()
    Doesn't appear to work (e.g. with water-gas or pbm-copper)
    I think errors develop over time in eig()
"""
from numpy import amax, array, column_stack, concatenate, diag, dot, eye, \
    sqrt, zeros
from scipy.linalg import eig, inv, solve

from gpr.misc.structures import State
from gpr.sys.analytical import ode_solver_cons
from gpr.sys.eigenvalues import Xi1, Xi2
from gpr.sys.primitive import M_prim
from gpr.vars.eos import total_energy
from gpr.vars.wavespeeds import c_0

THERMAL = True
STICK = True
RELAXATION = True
STAR_TOL = 1e-6

n1 = 3 + int(THERMAL)
n2 = 6 + 2 * int(THERMAL)
n3 = 8 + int(THERMAL)
n4 = 11 + int(THERMAL)
n5 = 14 + int(THERMAL)

def Pvec(Q, MP):
    """ Vector of primitive variables
        NOTE: Uses atypical ordering
    """
    P = State(Q, MP)
    ret = Q.copy()

    ret[1] = P.p()
    ret[2:5] /= P.ρ

    if THERMAL:
        ret[14:17] /= P.ρ

    return ret

def Pvec_to_Cvec(P, MP):
    """ Returns the vector of conserved variables, given the vector of
        primitive variables
    """
    Q = P.copy()

    ρ = P[0]
    p = P[1]
    v = P[2:5]
    A = P[5:14].reshape([3, 3])
    J = P[14:17]
    λ = 0

    Q[1] = ρ * total_energy(ρ, p, v, A, J, λ, MP)
    Q[2:5] *= ρ
    if THERMAL:
        Q[14:17] *= ρ

    return Q

def check_star_convergence(QL_, QR_, MPL, MPR):

    PL_ = State(QL_, MPL)
    PR_ = State(QR_, MPR)

    cond1 = amax(abs(PL_.Σ()[0] - PR_.Σ()[0])) < STAR_TOL

    if THERMAL:
        cond2 = abs(PL_.T() - PR_.T()) < STAR_TOL
    else:
        cond2 = True

    return cond1 and cond2

def Y_matrix(P, MP, sgn):

    ρ = P.ρ
    A = P.A
    σρ = P.dσdρ()
    σA = P.dσdA()

    e0 = array([1, 0, 0])
    Π1 = σA[0, :, :, 0]

    tmp = zeros([5, 5])
    tmp[:3, 0] = -σρ[0]
    tmp[0, 1] = 1
    tmp[:3, 2:5] = -Π1

    if THERMAL:

        tmp[3, 0] = P.dTdρ()
        tmp[3, 1] = P.dTdp()
        tmp[4, 0] = -1 / ρ
        tmp[4, 2:5] = inv(A)[0]

    else:

        p = P.p()
        σ = P.σ()
        c0 = c_0(ρ, p, A, MP)

        B = zeros([2, 3])
        B[0, 0] = ρ
        B[1] = σ[0] - ρ * σρ[0]
        B[1, 0] += ρ * c0**2

        rhs = column_stack((-σρ[0], e0))
        C = solve(Π1, rhs)

        BA_1 = dot(B, inv(A))
        Z = eye(2) - dot(BA_1, C)
        W = concatenate([eye(2), -BA_1], axis=1)
        tmp[3:5] = solve(Z, W)

    b = zeros([5, n1])
    b[:n1, :n1] = eye(n1)
    X = solve(tmp, b)

    Ξ1 = Xi1(P, 0, MP)
    Ξ2 = Xi2(P, 0, MP)

    Ξ = dot(Ξ1, Ξ2)
    w, vr = eig(Ξ)

    D_1 = diag(1 / sqrt(w.real))
    Q_1 = vr
    Q = inv(Q_1)

    Y0 = dot(Q_1, dot(D_1, Q))

    return sgn * dot(Y0, dot(Ξ1, X))

def riemann_constraints(Q, sgn, MP):

    M = M_prim(Q, 0, MP)
    λ, L = eig(M, left=True, right=False)

    λ = λ.real
    L = L.T

    Lhat = L[λ.argsort()]
    Lhat[:n1] *= 0

    P = State(Q, MP)

    σρ = P.dσdρ()
    σA = P.dσdA()

    Lhat[:3, 0] = -σρ[0]
    Lhat[0, 1] = 1
    for i in range(3):
        Lhat[:3, 5 + 3 * i : 8 + 3 * i] = -σA[0, :, i]

    if THERMAL:
        Lhat[3, 0] = P.dTdρ()
        Lhat[3, 1] = P.dTdp()

    Lhat[-n1:, 2:5] *= -sgn
    if THERMAL:
        Lhat[-n1:, 14] *= -sgn

    return Lhat

def star_stepper(QL, QR, MPL, MPR, NV=17):

    PL = State(QL, MPL)
    PR = State(QR, MPR)

    LhatL = riemann_constraints(QL, -1, MPL)
    LhatR = riemann_constraints(QR, 1, MPR)

    cL = zeros(NV)
    cR = zeros(NV)

    if STICK:

        YL = Y_matrix(PL, MPL, -1)
        YR = Y_matrix(PR, MPR, 1)

        if THERMAL:
            xL = concatenate([PL.Σ()[0], [PL.T()]])
            xR = concatenate([PR.Σ()[0], [PR.T()]])
            yL = concatenate([PL.v, PL.J[:1]])
            yR = concatenate([PR.v, PR.J[:1]])
        else:
            xL = PL.Σ()[0]
            xR = PR.Σ()[0]
            yL = PL.v
            yR = PR.v

        x_ = solve(YL - YR, yR - yL + dot(YL, xL) - dot(YR, xR))

    else:  # slip conditions - only implemented for non-thermal

        if THERMAL:
            YL = Y_matrix(PL, MPL, -1)[[0,3]]
            YR = Y_matrix(PR, MPR, 1)[[0,3]]

            xL = array([PL.Σ()[0], PL.T()])
            xR = array([PR.Σ()[0], PR.T()])
            yL = array([PL.v[0], PL.J[0]])
            yR = array([PR.v[0], PR.J[0]])

            M = YL[:, [0, -1]] - YR[:, [0, -1]]
            x_ = solve(M, yR - yL + dot(YL, xL) - dot(YR, xR))
            x_ = array([x_[0], 0, 0, x_[1]])

        else:
            YL = Y_matrix(PL, MPL, -1)[0]
            YR = Y_matrix(PR, MPR, 1)[0]

            xL = PL.Σ()[0]
            xR = PR.Σ()[0]
            yL = PL.v[0]
            yR = PR.v[0]

            x_ = (yR - yL + dot(YL, xL) - dot(YR, xR)) / (YL - YR)[0]
            x_ = array([x_, 0, 0])

    cL[:n1] = x_ - xL
    cR[:n1] = x_ - xR

    PLvec = Pvec(QL, MPL)
    PRvec = Pvec(QR, MPR)
    PL_vec = solve(LhatL, cL) + PLvec
    PR_vec = solve(LhatR, cR) + PRvec
    QL_ = Pvec_to_Cvec(PL_vec, MPL)
    QR_ = Pvec_to_Cvec(PR_vec, MPR)

    return QL_, QR_

def star_states(QL, QR, MPL, MPR, dt):

    QL_ = QL.copy()
    QR_ = QR.copy()

    while not check_star_convergence(QL_, QR_, MPL, MPR):

        if RELAXATION:
            ode_solver_cons(QL_, dt / 2, MPL)
            ode_solver_cons(QR_, dt / 2, MPR)

        QL_, QR_ = star_stepper(QL_, QR_, MPL, MPR)

    return QL_, QR_


"""
 Created by Tang on 2020/2/13 11:47
"""
PER_PAGE = 15
RECENT_BOOK_PER_PAGE = 60

RECENT_BOOK_COUNT = 30
BEANS_EVERY_DRIFT = 1

BEANS_UPLOAD_ONE_BOOK = 0.5


import pandas as pd
import numpy as np

############################### df.where() #######################
# replace values where the condition is False, the default replace value is NaN

df = pd.DataFrame(
    {"AAA": [4, 5, 6, 7], "BBB": [10, 20, 30, 40], "CCC": [100, 50, -30, -50]}
)
# print(type(df['AAA'])) # <class 'pandas.core.series.Series'>
# print(type(df.loc[:, 'AAA'])) # <class 'pandas.core.series.Series'>

df_mask = pd.DataFrame(
    {"AAA": [True] * 4, "BBB": [False] * 4, "CCC": [True, False] * 2}
)
# print(df.where(df_mask, -1000))
# print(df.where(df_mask))

s = pd.Series(np.random.randn(10))
# print(s.where(s>0))

# print(type(df.groupby('AAA')['BBB'].idxmin())) # <class 'pandas.core.series.Series'>
# print(df.groupby('AAA')['BBB'].idxmin()) # groupby里面的字段内的数据(即AAA对应的字段)重构后都会变成索引（所以重复的会变成一个），即AAA中的1，2，3都变为索引
'''
AAA
1    1
2    5
3    6
Name: BBB, dtype: int64
'''

############################### 3.Missing data #######################
df = pd.DataFrame(
    np.random.randn(6, 1),
    index=pd.date_range("2013-08-01", periods=6, freq="B"),
    columns=list("A"),
)
# print(df)
df.loc[df.index[3], "A"] = np.nan
new_index = pd.date_range("2013-08-01", periods=7, freq="B")
# print(df.reindex(new_index)) # 按照new_index的顺序重新规划df，并将新出现index的地方补充为NaN
# print(df.reindex(new_index).ffill()) # 使得NaN和前一个数保持一致

############################### 4.Grouping #######################

############################### 5.Merge #######################
rng = pd.date_range("2000-01-01", periods=6)
df1 = pd.DataFrame(np.random.randn(6, 3), index=rng, columns=["A", "B", "C"])
df2 = df1.copy()
# print(df1.append(df2, ignore_index=False))
# print(df1.append(df2, ignore_index=False)['2000-01-01'])
# print(df1.append(df2, ignore_index=False)['A'])

df = pd.DataFrame(
    data={
        "Area": ["A"] * 5 + ["C"] * 2,
        "Bins": [110] * 2 + [160] * 3 + [40] * 2,
        "Test_0": [0, 1, 0, 1, 2, 0, 1],
        "Data": np.random.randn(7),
    }
)
df["Test_1"] = df["Test_0"] - 1
# print(df)
# print(pd.merge(df))
# print(df.index)