"""
The following script has 4 main sections.
We first calculate the optimal investment weights of a trading strategy
by optimization. Then, we calculate the in and out of sample returns
for the strategy and the chosen benchmark.
Ultimately, we calculate performance and risk ratios and plot portfolio
characteristics of the investment strategy in comparison to the benchmark.
 """

# load libraries
import yfinance as yf
import pandas as pd
import numpy as np
import math as m
import matplotlib.pyplot as plt
import matplotlib.ticker as mtick
import matplotlib.dates as mdates
from scipy.optimize import minimize
from scipy.optimize import Bounds
from sklearn.linear_model import LinearRegression
import dataframe_image as dfi

#############################################################################
#                                                                           #
#                                #SECTION 1                                 #
#       Define key parameters, import needed files and define investment    #
#       universe of stocks which the strategy invests in                    #
#                                                                           #
#############################################################################

# define parameters of investment period start (start_backtesting as maximum time period!)
# actual time could be shorter depending on availability of data!
start_backtesting = np.datetime64("2011-01-01")
end_backtesting = np.datetime64("2021-12-31")
end_out_sample = np.datetime64("2022-05-21")

# Import needed CSV files
stocks_index = pd.read_csv("files/index_constituents_data.csv", index_col=0)
benchmark = pd.read_csv("files/benchmark.csv", index_col=0)
Gross_Price = pd.read_csv("files/Gross_Prices_EUR.csv", index_col=0)
Net_Price = pd.read_csv("files/Net_Prices_EUR.csv", index_col=0)

# format index to date
Gross_Price.index = pd.to_datetime(Gross_Price.index)
Net_Price.index = pd.to_datetime(Net_Price.index)
benchmark.index = pd.to_datetime(benchmark.index)

"""
Select stocks which the strategy invests in
our strategy goes long in the top 10 highest dividend stocks and short in the
top 10 lowest dividend stocks --> We select these stocks as our investmnet stocks
Also: drop BHP stock for which dividends have not been calculated correctly!
"""

stocks_index = stocks_index.sort_values(by="Yield", ascending=False)
stocks_index = stocks_index.drop("BHP.L")
stocks_invest = stocks_index.iloc[np.r_[0:10, 39:49]]

# select prices of stocks which we invest in
Gross_Price_selected = Gross_Price[stocks_invest.index]
Net_Price_selected = Net_Price[stocks_invest.index]

#############################################################################
#                                                                           #
#                                #SECTION 2                                 #
#     Calculate Optimal Strategy stock weights with various constraints     #
#                       and visualize optimization                          #
#                                                                           #
#############################################################################

# estimate expected returns and var-cov matrix for optimization (with in sample data!)

rf_daily = 0
ER = Gross_Price_selected[Gross_Price_selected.index < end_backtesting].pct_change().mean()
S = Gross_Price_selected[Gross_Price_selected.index < end_backtesting].pct_change().cov()

# define functions for optimization (variance, return,  negative sharp ratio)
def pvar(w, S):
    return (w.T @ S @ w)

def pret(w, ER):
    return (w.T @ ER)

def sharpe(w, ER, S):
    return -(w.T @ (ER - rf_daily)) / ((w.T @ S @ w) ** 0.5)

# ---------------------------------------------------------------------------
# Calculate Optimized Portfolio
# ---------------------------------------------------------------------------
N = len(ER)
x0 = np.ones(N) / N

"""
#set up constraints
first constraint -> total investment = 100% long
second constraint- > Values smaller than - min_weight (shocks which are shorted)
third constraint -> Values larger than min_weight (stock which are longed)
bounds = maximum long and short % for stocks
"""

min_weight = 0.02

cons_multiple = ({"type": "eq", "fun": lambda x: np.sum(x) - 1},
                 {"type": "ineq", "fun": lambda x: -min_weight - x[10:20]},
                 {"type": "ineq", "fun": lambda x: x[0:10] - min_weight})

bounds = Bounds(-0.2, 0.2)

# only constraint for unconstrained portfolio --> Total stock weights == 100%
cons_simple = ({"type": "eq", "fun": lambda x: np.sum(x) - 1})

# calculate optimized values (GMVP and MSRP constrained and unconstrained)
GMVP_const = minimize(pvar, x0, method='SLSQP', args=S, constraints=cons_multiple,
                      options={'disp': True, 'ftol': 1e-9}, bounds=bounds)

GMVP_unconst = minimize(pvar, x0, method="SLSQP", args=S, constraints=cons_simple,
                        options={'disp': True, 'ftol': 1e-9})

MSRP_const = minimize(sharpe, x0, method='SLSQP', args=(ER, S), constraints=cons_multiple,
                      options={'disp': True, 'ftol': 1e-9}, bounds=bounds)

MSRP_unconst = minimize(sharpe, x0, method='SLSQP', args=(ER, S), constraints=cons_simple,
                        options={'disp': True, 'ftol': 1e-9})

# complete data set of selected stocks with calculated constrained MSRP values & export for presentation
stocks_invest = stocks_invest.assign(weights=MSRP_const.x)
stocks_invest = stocks_invest.assign(weights_unconst=MSRP_unconst.x)

stocks_invest_export = stocks_invest.copy()
stocks_invest_export["Yield"] = stocks_invest_export["Yield"].map('{:,.2%}'.format)
stocks_invest_export["weights"] = stocks_invest_export["weights"].map('{:,.2%}'.format)
stocks_invest_export["index_weights"] = stocks_invest_export["index_weights"].map('{:,.2%}'.format)
stocks_invest_export["weights_unconst"] = stocks_invest_export["weights_unconst"].map('{:,.2%}'.format)

dfi.export(stocks_invest_export, "plots/selected_portfolio_characteristics.png")

# ---------------------------------------------------------------------------
# Visualize efficient Frontier
# ---------------------------------------------------------------------------

"""
Generate Minimum Variance Frontier / Investment Frontier plot by doing the following:
1. Minimize the negative value of the expected portfolio return
given a deterministic variance and the same rules as above (eg. min 2% in each stock)
as restrictions in the optimization. --> We thus retrieve the maximum return given
all restrictions for a given variance.

2. We construct "random portfolios" which fulfill all the restrictions by minimizing a function
which consists of a random array and a modulus operation which should randomize the optimization
results. We can then calculate the expected return and variance of these portfolios and add them
to the plot.
"""

# calculate yearly volatility and expected return of GMVP and MSRP constrained for plotting further below
# assumption: year = 250 trading days

GMVP_const_ER = pret(w=GMVP_const.x, ER=ER) * 250
GMVP_const_VAR = m.sqrt(pvar(w=GMVP_const.x, S=S) * 250)
MSRP_const_ER = pret(w=MSRP_const.x, ER=ER) * 250
MSRP_const_VAR = m.sqrt(pvar(w=MSRP_const.x, S=S) * 250)

# define negative portfolio return function
def pret_sim(w, ER):
    return -(w.T @ ER)

# variances to loop over
var_list = np.arange(0.1, 0.6, 0.01)
bounds = Bounds(-0.2, 0.2)

# initialize lists to store results in
MVF_var_const = []
MVF_ret_const = []

# loop over variances and maximize expected return given that portfolio variance = given variance
for i in var_list:
    max_var = i
    cons = ({"type": "eq", "fun": lambda x: np.sum(x) - 1},
            {"type": "eq", "fun": lambda x: m.sqrt((x.T @ S @ x) * 250) - max_var},
            {"type": "ineq", "fun": lambda x: -min_weight - x[10:20]},
            {"type": "ineq", "fun": lambda x: x[0:10] - min_weight})

    maximized = minimize(pret_sim, x0, method='SLSQP', args=ER,
                         options={'disp': True, 'ftol': 1e-9},
                         constraints=cons, bounds=bounds)

    # only store result if optimization was successful!
    if maximized.success:
        MVF_var_const.append(i)
        MVF_ret_const.append(maximized.fun * -250)
    else:
        continue

# ---------------------------------------------------------------------------
# Visualize efficient Frontier
# ---------------------------------------------------------------------------

random_portfolio = []

# "random" function which when optimized gives "random" portfolios
def rand_funct(x, y):
    return ((x % y) / y).sum()

# weight constraints of individual stocks
cons = ({"type": "eq", "fun": lambda x: np.sum(x) - 1},
        {"type": "ineq", "fun": lambda x: -min_weight - x[10:20]},
        {"type": "ineq", "fun": lambda x: x[0:10] - min_weight})

# simulate 100 different portfolios which fulfill the constraints
i = 0
np.random.seed(0)

while i < 100:
    y = np.random.uniform(low=0, high=1, size=N)
    portfolio_sim = minimize(rand_funct, x0, method='SLSQP', args=y,
                             constraints=cons, options={'disp': True, 'ftol': 1e-9},
                             bounds=bounds)
    i = i + 1

    # only store successful optimizations
    if portfolio_sim.success:
        random_portfolio.append(portfolio_sim.x)
    else:
        continue

MVF_randvar_const = []
MVF_randret_const = []

# calculate return and variance (yearly) from calculated "random" portfolios
for i in range(len(random_portfolio)):
    exp_return = pret(w=random_portfolio[i], ER=ER)
    exp_var = pvar(w=random_portfolio[i], S=S)
    exp_return_year = exp_return * 250
    exp_var_year = m.sqrt(exp_var * 250)

    MVF_randvar_const.append(exp_var_year)
    MVF_randret_const.append(exp_return_year)

# plot results
MVF_var_const.insert(0, GMVP_const_VAR)
MVF_ret_const.insert(0, GMVP_const_ER)
CAL_x = np.linspace(0, MSRP_const_VAR + 0.2, 50, endpoint=True)
SR_MSRP = MSRP_const_ER / MSRP_const_VAR
CAL_y = 0 + SR_MSRP * CAL_x

frontier_points = pd.DataFrame({"variance": MVF_var_const, "return": MVF_ret_const})
max_ret_index = frontier_points["return"].idxmax()

fig, ax = plt.subplots(figsize=(15, 10))
plt.plot(frontier_points.variance.iloc[0:max_ret_index + 1], frontier_points["return"].iloc[0:max_ret_index + 1])
plt.plot(frontier_points.variance.iloc[max_ret_index: len(frontier_points)],
         frontier_points["return"].iloc[max_ret_index: len(frontier_points)], color="grey", ls="--")
plt.scatter(MVF_randvar_const, MVF_randret_const)
plt.scatter(GMVP_const_VAR, GMVP_const_ER, s=70)
plt.scatter(MSRP_const_VAR, MSRP_const_ER, s=70)
plt.plot(CAL_x, CAL_y)
plt.xlim([0, 0.4])
plt.ylim([0, 0.4])
plt.title("Investment Frontier Simulated", size=25)
plt.xlabel("Volatility", size=15)
plt.ylabel("Return", size=15)
plt.legend(["Minimum Variance Frontier",
            "Investment Frontier",
            "Capital Allocation Line",
            "random portfolios",
            "GMVP",
            "MSRP"],
           prop={'size': 15})
ax.xaxis.set_major_formatter(mtick.PercentFormatter(1.0))
ax.yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
plt.xticks(size=15)
plt.yticks(size=15)
plt.savefig("plots/investmentfrontier.png")

#############################################################################
#                                                                           #
#                                #SECTION 2                                 #
#                           Calculate  returns of                           #
#                 strategy and benchmark in and out of sample               #
#                                                                           #
#############################################################################

# function which calculates the indexed performance (start = 1) of various stocks and start investment weights
def indexed_performance(start, prices, weights, end=None):
    # filter desired time range
    if end is None:
        temp = prices[prices.index >= start]
    else:
        temp = prices[(prices.index >= start) & (prices.index < end)]

    # define daily percentage changes + 1
    temp = temp.pct_change() + 1

    # replace first row with weights of each stock --> Initial Investment = 1
    temp.iloc[0] = weights

    # take cumulative return of each stock * initial investment (weight) and sum up horizontaly
    temp = temp.cumprod().sum(axis=1)
    return temp

# calculate weights of short stocks / long stocks  that they sum up to 1
weights_short = stocks_invest.weights[stocks_invest.weights < 0] / stocks_invest.weights[stocks_invest.weights < 0].sum()
weights_long = stocks_invest.weights[stocks_invest.weights > 0] / stocks_invest.weights[stocks_invest.weights > 0].sum()

# calculate the needed indexed performance series
in_sample_net = indexed_performance(start=start_backtesting, end=end_backtesting,
                                    weights=MSRP_const.x, prices=Net_Price_selected)

in_sample_gross = indexed_performance(start=start_backtesting, end=end_backtesting,
                                      weights=MSRP_const.x, prices=Gross_Price_selected)

out_sample_gross_short = indexed_performance(start=end_backtesting, weights=weights_short,
                                             prices=Gross_Price_selected[
                                                 stocks_invest[stocks_invest.weights < 0].index])

out_sample_gross_long = indexed_performance(start=end_backtesting, weights=weights_long,
                                            prices=Gross_Price_selected[stocks_invest[stocks_invest.weights > 0].index])

out_sample_gross = indexed_performance(start=end_backtesting, weights=MSRP_const.x,
                                       prices=Gross_Price_selected)

# combine all returns to data frame (one for in and one for out of sample performance)
in_sample = pd.DataFrame({"strategy_net": in_sample_net,
                          "strategy_gross": in_sample_gross},
                         index=in_sample_net.index)

out_sample = pd.DataFrame({"strategy_gross": out_sample_gross,
                           "long_gross": out_sample_gross_long,
                           "short_gross": out_sample_gross_short},
                          index=out_sample_gross.index)

# join benchmark data & calculate indexed performance of benchmark
in_sample = in_sample.join(benchmark)
out_sample = out_sample.join(benchmark)

in_sample[["benchmark_gross", "benchmark_net"]] = in_sample[["benchmark_gross", "benchmark_net"]].div(
    in_sample[["benchmark_gross", "benchmark_net"]].head(1).iloc[0])
out_sample[["benchmark_gross", "benchmark_net"]] = out_sample[["benchmark_gross", "benchmark_net"]].div(
    out_sample[["benchmark_gross", "benchmark_net"]].head(1).iloc[0])

# save in and out of sample returns
in_sample.to_csv("files/in_sample.csv")
out_sample.to_csv("files/out_sample.csv")

#############################################################################
#                                                                           #
#                            #SECTION 3                                     #
#        #Calculate in and out of sample risk & performance ratios          #
#                                                                           #
#############################################################################

# ---------------------------------------------------------------------------
# Annualized Sharp Ratio
# ---------------------------------------------------------------------------

# days = days per year (assumed 250 here)
# rf_rate = annualized risk free rate
# prices for which ratio should be alculated

def sharp_ratio(price, days, rf_rate):
    ER = price.pct_change().mean()
    SD = price.pct_change().std()
    SR = (ER - rf_rate / days) / SD * m.sqrt(days)

    return round(SR, 2)

# ---------------------------------------------------------------------------
# Yearly volatility
# ---------------------------------------------------------------------------

def yearly_vol(price, days, pct=True):
    vol = m.sqrt(price.pct_change().var() * days)

    if pct:
        return format(vol, ".2%")
    else:
        return vol

# ---------------------------------------------------------------------------
# alpha and beta
# ---------------------------------------------------------------------------

# period = period of returns for which regression should be run (eg. "1Y", "1M", "1d")
# kwargs = optional where only alpha, beta, x or y can be returned form function (keyword = param)
# rf_rate = risk free rate for the "period"

def alpha_beta(strategy, benchmark, period, rf_period, pct=True, **kwargs):
    daily_ret_strategy = strategy.pct_change().fillna(0) + 1
    daily_ret_BM = benchmark.pct_change().fillna(0) + 1

    # calculate excess returns for the period over the risk-free rate
    period_ret_strategy = daily_ret_strategy.groupby(pd.Grouper(freq=period)).prod() - 1 - rf_period
    period_ret_BM = daily_ret_BM.groupby(pd.Grouper(freq=period)).prod() - 1 - rf_period

    model = LinearRegression().fit(period_ret_BM.to_numpy().reshape((-1, 1)),
                                   period_ret_strategy.to_numpy().reshape((-1, 1)))

    beta = model.coef_[0][0]

    if pct:
        alpha = format(model.intercept_[0], ".2%")
    else:
        alpha = model.intercept_[0]

    results = {"alpha": alpha,
               "beta": beta,
               "x": period_ret_BM.to_list(),
               "y": period_ret_strategy.to_list()}

    if len(kwargs) == 0:
        return results
    else:
        return results.get(kwargs.get("param"))

# ---------------------------------------------------------------------------
# Maximum Drawdown
# ---------------------------------------------------------------------------

def maxdd(price, pct=True):
    # Value at time T divided by max value from time t = 0 to t = T (T> 0)
    diffmax = price / price.cummax()
    maxdd = (diffmax - 1).min()

    if pct:
        return format(maxdd, ".2%")
    else:
        return maxdd

# ---------------------------------------------------------------------------
# N Day unfiltered historical 1% VAR
# ---------------------------------------------------------------------------

# N as number of days (integer)
def NDAYVar(price, N, pct=True):
    daily_ret = price.pct_change().fillna(0) + 1
    daily_ret = daily_ret.reset_index(drop=True)
    nday_ret = daily_ret.groupby(daily_ret.index // N).prod() - 1
    VAR = nday_ret.quantile(0.01)

    if pct:
        return format(np.abs(VAR), ".2%")
    else:
        return np.abs(VAR)

# ---------------------------------------------------------------------------
# Expected N day return or total Return
# ---------------------------------------------------------------------------

def nday_ret(price, N=None, TR=False, pct=True):
    if TR:
        ret = price.tail(1)[0] / price.head(1)[0] - 1
    else:
        ret = price.pct_change().mean() * N
    if pct:
        return format(ret, ".2%")
    else:
        return ret

# define return series for which ratios should be calculated:
BM_in = in_sample["benchmark_gross"]
BM_out = out_sample["benchmark_gross"]
strategy_in = in_sample["strategy_gross"]
strategy_out = out_sample["strategy_gross"]

# assemble dicts with risk return metrics
ratios_BM_in = {"Avg. Yearly Return": nday_ret(BM_in, N=250),
                "Avg. Yearly Sharp Ratio": sharp_ratio(BM_in, days=250, rf_rate=0),
                "Max. Drawdown": maxdd(BM_in),
                "Alpha (monthly Returns)": "0%",
                "Beta (monthly Returns)": 1,
                "Avg. Ann. Vol": yearly_vol(BM_in, days=250),
                "5d 1% Hist. VAR": NDAYVar(BM_in, N=5)}

ratios_BM_out = {"Return YTD": nday_ret(BM_out, TR=True),
                 "Avg. Yearly Sharp Ratio": sharp_ratio(BM_out, days=250, rf_rate=0),
                 "Max. Drawdown": maxdd(BM_out),
                 "Alpha (weekly Returns)": "0%",
                 "Beta (weekly Returns)": 1,
                 "Avg. Ann. Vol": yearly_vol(BM_out, days=250),
                 "5d 1% Hist. VAR": NDAYVar(BM_out, N=5)}

ratios_strategy_in = {"Avg. Yearly Return": nday_ret(strategy_in, N=250),
                      "Avg. Yearly Sharp Ratio": sharp_ratio(strategy_in, days=250, rf_rate=0),
                      "Max. Drawdown": maxdd(strategy_in),
                      "Alpha (monthly Returns)": alpha_beta(strategy_in, BM_in, "1M", rf_period=0, param="alpha"),
                      "Beta (monthly Returns)": round(alpha_beta(strategy_in, BM_in, "1M", rf_period=0, param="beta"),
                                                      2),
                      "Avg. Ann. Vol": yearly_vol(strategy_in, days=250),
                      "5d 1% Hist. VAR": NDAYVar(strategy_in, N=5)}

ratios_strategy_out = {"Return YTD": nday_ret(strategy_out, TR=True),
                       "Avg. Yearly Sharp Ratio": sharp_ratio(strategy_out, days=250, rf_rate=0),
                       "Max. Drawdown": maxdd(strategy_out),
                       "Alpha (weekly Returns)": alpha_beta(strategy_out, BM_out, "1W", rf_period=0, param="alpha"),
                       "Beta (weekly Returns)": round(alpha_beta(strategy_out, BM_out, "1W", rf_period=0, param="beta"),
                                                      2),
                       "Avg. Ann. Vol": yearly_vol(strategy_out, days=250),
                       "5d 1% Hist. VAR": NDAYVar(strategy_out, N=5)}

risk_factors_in = pd.DataFrame({"Benchmark": ratios_BM_in, "Strategy": ratios_strategy_in})
risk_factors_out = pd.DataFrame({"Benchmark": ratios_BM_out, "Strategy": ratios_strategy_out})

dfi.export(risk_factors_out, "plots/risk_factors_out.png")
dfi.export(risk_factors_in, "plots/risk_factors_in.png")

# ---------------------------------------------------------------------------
# Plot Visual Example for in Sample Alpha/ Beta Calculation
# ---------------------------------------------------------------------------

# plot regression line
params = alpha_beta(strategy_in, BM_in, rf_period=0, period="1M", pct=False)
x = np.arange(min(params.get("x")) - 0.1, max(params.get("x")) + 0.1, 0.01)
fitted_y = params.get("alpha") + params.get("beta") * x

# assemble plot
fig, ax = plt.subplots(figsize=(15, 10))
plt.scatter(params.get("x"), params.get("y"))
plt.plot(x, fitted_y)
plt.plot(x, x)
plt.title("Monthly Excess Returns Strategy vs. Benchmark (in sample)", size=25)
ax.yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
ax.xaxis.set_major_formatter(mtick.PercentFormatter(1.0))
plt.ylabel("Monthly Excess Returns Strategy", size=15)
plt.xlabel("Monthly Excess Returns Benchmark", size=15)
plt.axhline(0, color="black", ls="--", lw=1)
plt.axvline(0, color="black", ls="--", lw=1)
plt.xticks(size=15)
plt.yticks(size=15)
plt.legend(["liner fit", "x = y"], prop={"size": 15})
plt.savefig("plots/in_sample_alphabetaplot.png")

#############################################################################
#                                                                           #
#                            #SECTION 4                                     #
#        #Calculate and plot portfolio characteristics vs. Benchmark        #
#                                                                           #
#############################################################################

# ---------------------------------------------------------------------------
# Excess Return Strategy vs. Benchmark Plot
# ---------------------------------------------------------------------------

# calculate return over out of sample period for all stocks in the benchmark index
out_sample_all = Gross_Price[Gross_Price.index > end_backtesting]
out_sample_all = out_sample_all.pct_change() + 1
out_sample_all = out_sample_all.fillna(1).cumprod()
out_return_all = out_sample_all.iloc[len(out_sample_all.index) - 1].div(out_sample_all.iloc[0]) - 1

# calculate excess return contribution as (Weight Strategy - Weigh Index) * Return
ret_contrib = stocks_index.join(stocks_invest["weights"]).fillna(0)
ret_contrib = ret_contrib[["Name", "weights", "index_weights"]]
ret_contrib["diff_weights"] = ret_contrib["weights"] - ret_contrib["index_weights"]
ret_contrib = ret_contrib.join(pd.DataFrame(out_return_all))
ret_contrib = ret_contrib.rename(columns={0: "return"})
ret_contrib["diff_return"] = ret_contrib["diff_weights"] * ret_contrib["return"]

# plot and save results
fig = plt.figure(figsize=(15, 10))
ax = fig.add_subplot(1, 1, 1)
plt.bar(ret_contrib.Name[0:10], ret_contrib.diff_return[0:10])
plt.bar(ret_contrib.Name[10:39], ret_contrib.diff_return[10:39])
plt.bar(ret_contrib.Name[39:49], ret_contrib.diff_return[39:49])
plt.legend(["Long", "Not Invested", "Short", "Currencies"])
ax.yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
plt.title("Decomposition of out of Sample excess Returns", size=22)
plt.ylabel("Excess Return Strategy vs. Benchmark", size=15)
plt.subplots_adjust(bottom=0.3)
plt.xticks(rotation=90)
plt.yticks(size=12)
plt.savefig("plots/excess_return_breakdown.png")

# ---------------------------------------------------------------------------
# Out of Sample Gross Strategy vs Benchmark Performance
# ---------------------------------------------------------------------------

fig = plt.figure(figsize=(15, 10))
plt.plot(out_sample[["strategy_gross", "benchmark_gross"]])
plt.legend(out_sample[["strategy_gross", "benchmark_gross"]].columns,
           prop={"size": 15})
plt.title("Out of Sample Strategy Performance (TR)", size=20)
plt.ylabel("Cumulative Return", size=15)
plt.yticks(size=15)
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%b-%Y'))
plt.gca().xaxis.set_major_locator(mdates.MonthLocator(interval=1))
plt.xticks(size=15)
plt.savefig("plots/outofsample_performance.png")

# ---------------------------------------------------------------------------
# In Sample Gross Strategy vs Benchmark Performance
# ---------------------------------------------------------------------------

fig = plt.figure(figsize=(15, 10))
plt.plot(in_sample[["strategy_gross", "benchmark_gross"]])
plt.legend(in_sample[["strategy_gross", "benchmark_gross"]].columns,
           prop={"size": 15})
plt.title("In Sample Strategy Performance", size=22)
plt.ylabel("Cumulative Return", size=15)
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%b-%Y'))
plt.gca().xaxis.set_major_locator(mdates.MonthLocator(interval=3))
plt.xticks(size=13)
plt.yticks(size=13)
plt.savefig("plots/insample_performance.png")

# ---------------------------------------------------------------------------
# In Sample Performance Net vs. Gross
# ---------------------------------------------------------------------------

fig = plt.figure(figsize=(15, 10))
plt.plot(in_sample[["strategy_gross", "strategy_net", "benchmark_gross", "benchmark_net"]])
plt.legend(in_sample[["strategy_gross", "strategy_net", "benchmark_gross", "benchmark_net"]].columns,
           prop={"size": 15})
plt.title("In Sample Strategy Performance (net and gross) Comparison", size=22)
plt.ylabel("Cumulative Return", size=15)
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%b-%Y'))
plt.gca().xaxis.set_major_locator(mdates.MonthLocator(interval=3))
plt.xticks(size=13)
plt.yticks(size=13)
plt.savefig("plots/insample_performance_netgross.png")

# ---------------------------------------------------------------------------
# Out of Sample Performance Decomposition Long/ Short Portfolio
# ---------------------------------------------------------------------------

fig = plt.figure(figsize=(15, 10))
plt.plot(out_sample[["strategy_gross", "benchmark_gross", "long_gross", "short_gross"]])
plt.legend(out_sample[["strategy_gross", "benchmark_gross", "long_gross", "short_gross"]].columns,
           prop={"size": 15})
plt.title("Decomposition of out of sample performance", size=22)
plt.ylabel("Cumulative Return", size=15)
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%b-%Y'))
plt.gca().xaxis.set_major_locator(mdates.MonthLocator(interval=1))
plt.xticks(size=13)
plt.yticks(size=13)
plt.savefig("plots/outofsamlpe_brekdown.png")

# ---------------------------------------------------------------------------
# Stock Weights Index vs. Constrained & Unconstrained Optimization
# ---------------------------------------------------------------------------

width = 0.3
fig, ax = plt.subplots(figsize=(15, 10))
ind = np.arange(len(stocks_invest))
ax.barh(ind, stocks_invest.weights, width, label="Strategy")
ax.barh(ind + width, stocks_invest.index_weights, width, label="Index")
ax.barh(ind + 2 * width, stocks_invest.weights_unconst, width, label="Unconstrained Strategy")
ax.set(yticks=ind + width, yticklabels=stocks_invest.Name)
ax.legend(prop={'size': 10})
ax.xaxis.set_major_formatter(mtick.PercentFormatter(1.0))
plt.title("Stock Weights Strategy vs Index", size=25)
plt.yticks(size=10)
plt.xticks(size=15, ticks=np.arange(-0.4, 0.6, 0.1))
plt.savefig("plots/strategy_weights.png")

# ---------------------------------------------------------------------------
# Plot Country Weights Index vs. Strategy
# ---------------------------------------------------------------------------

country_weights_strategy = stocks_invest[["Country", "weights"]].groupby(by="Country").sum()
country_weights_index = stocks_index[["Country", "index_weights"]].groupby(by="Country").sum()
country_weights = country_weights_index.join(country_weights_strategy).fillna(0)
country_weights = pd.DataFrame(country_weights, index=country_weights_index.index)

# assemble plot
width = 0.3
fig, ax = plt.subplots(figsize=(15, 10))
ind = np.arange(len(country_weights))
ax.barh(ind, country_weights.weights, width, label="Strategy")
ax.barh(ind + width, country_weights.index_weights, width, label="Index")
ax.set(yticks=ind + width, yticklabels=country_weights.index)
ax.legend(prop={'size': 15})
ax.xaxis.set_major_formatter(mtick.PercentFormatter(1.0))
plt.title("Country Weights Strategy vs Index", size=25)
plt.yticks(size=15)
plt.xticks(size=15)
plt.savefig("plots/country_weights.png")

# ---------------------------------------------------------------------------
# Plot Currency weights index vs. Strategy
# ---------------------------------------------------------------------------

values = stocks_index.join(stocks_invest["weights"]).fillna(0)
currency_weights_strategy = values[["Currency", "weights"]].groupby(by="Currency").sum()
currency_weights_index = values[["Currency", "index_weights"]].groupby(by="Currency").sum()

currency_weights = currency_weights_index.join(currency_weights_strategy).fillna(0)
currency_weights = currency_weights.rename(index={"GBp": "GBP"})

# assemble plot
width = 0.4
fig, ax = plt.subplots()
ind = np.arange(len(currency_weights_index))
ax.barh(ind, currency_weights.weights, width, label="Strategy")
ax.barh(ind + width, currency_weights.index_weights, width, label="Index")
ax.set(yticks=ind + width, yticklabels=currency_weights.index)
ax.legend(["Strategy", "Index"])
plt.title("Currency Weights Strategy vs Index", size=20)
ax.xaxis.set_major_formatter(mtick.PercentFormatter(1.0))
plt.yticks(size=15)
plt.xticks(size=15)
plt.savefig("plots/curency_comparison.png")

# ---------------------------------------------------------------------------
# Plot sector weights strategy vs. index
# ---------------------------------------------------------------------------

sector_weights_strategy = stocks_invest[["Sector", "weights"]].groupby(by="Sector").sum()
sector_weights_index = stocks_index[["Sector", "index_weights"]].groupby(by="Sector").sum()
sector_weights = sector_weights_index.join(sector_weights_strategy)

# assemble plot
width = 0.4
fig, ax = plt.subplots(figsize=(15, 10))
ind = np.arange(len(sector_weights))
ax.barh(ind + width, sector_weights.weights, width, label="Strategy")
ax.barh(ind, sector_weights.index_weights, width, label="Index")
ax.set(yticks=ind + width, yticklabels=sector_weights.index)
ax.legend(prop={"size": 20})
plt.title("Sector Weights Strategy vs Index", size=25)
ax.xaxis.set_major_formatter(mtick.PercentFormatter(1.0))
plt.yticks(size=15)
plt.xticks(size=15)
plt.subplots_adjust(left=0.2)
plt.savefig("plots/sector_weights.png")

# ---------------------------------------------------------------------------
# Plot performance Nasdaq100 vs. MSCI World vs. FTSE AW-HighDivYield
# ---------------------------------------------------------------------------

# download prices and clean data
prices = yf.download(tickers=["VHYL.AS", "EXXT.DE", "IWDA.AS"], start="2022-01-01", end=end_out_sample)
prices = prices["Adj Close"]
prices = prices.div(prices.iloc[0])
prices = prices.rename(
    columns={"EXXT.DE": "Nasdaq 100", "IWDA.AS": "MSCI World", "VHYL.AS": "FTSE All World High Dividend"})

# assemble plot
plt.figure(figsize=(15, 10))
plt.plot(prices)
plt.title("High Dividend Stocks vs. Technology Stocks vs. Total Market", size=25)
plt.legend(prices.columns, prop={'size': 15})
plt.ylabel("Cumulative Return", size=15)
plt.xticks(size=15)
plt.yticks(size=15)
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%b-%Y'))
plt.gca().xaxis.set_major_locator(mdates.MonthLocator(interval=1))
plt.savefig("plots/comparison.png")

# ---------------------------------------------------------------------------
# calculate change of correlation  matrix in vs out of sample
# ---------------------------------------------------------------------------

out_sample_cor = Gross_Price_selected[Gross_Price_selected.index > end_backtesting].pct_change().corr()
in_sample_cor = Gross_Price_selected[Gross_Price_selected.index < end_backtesting].pct_change().corr()

rel_cor = out_sample_cor / in_sample_cor
pd.DataFrame(rel_cor).to_csv("files/correlation_change.csv")

# ---------------------------------------------------------------------------
# calculate performance ratios for short and long portfolios & Benchmark
# ---------------------------------------------------------------------------

"""
we only include stocks which contain a forward,trailing PE ratio &
PB Ratio to calculate weighted values.
Further, as described by iShares,
we restrict the maximum value of PE ratios to 60 and of PB ratios to 25.
"""
stocks_short = stocks_invest[stocks_invest.weights < 0]
stocks_long = stocks_invest[stocks_invest.weights > 0]

# dividend yield
div_yield_long = (stocks_long.weights / stocks_long.weights.sum() * stocks_long.Yield).sum()
div_yield_short = (stocks_short.weights.abs() / stocks_short.weights.abs().sum() * stocks_short.Yield).sum()
div_yield_index = (stocks_index.index_weights / stocks_index.index_weights.sum() * stocks_index.Yield).sum()

# trailing PE Ratio, forward PE, PB Ratio

# get all stocks and join strategy weights
index = stocks_index.join(stocks_invest[["weights"]])

# 1. remove stocks where ratios are not given
index = index[index.Trailing_PE.notna()]
index = index[index.Forward_PE.notna()]

# 2. set PE ratios above 60 to 60, set PB ratios above 25 to 25
index.Trailing_PE = np.where(index.Trailing_PE > 60, 60, index.Trailing_PE)
index.Forward_PE = np.where(index.Forward_PE > 60, 60, index.Forward_PE)
index.PB_Ratio = np.where(index.PB_Ratio > 25, 25, index.PB_Ratio)

# 3. separate long and short stocks from strategy to calculate ratios separately
short = index.loc[stocks_short.index]
long = index.loc[stocks_long.index]

# calculate weighted Trailing PE
PE_long = (long.weights / long.weights.sum() * long.Trailing_PE).sum()
PE_short = (short.weights / short.weights.sum() * short.Trailing_PE).sum()
PE_index = (index.index_weights / index.index_weights.sum() * index.Trailing_PE).sum()

# calculate weighted forward PE
PE_fwd_long = (long.weights / long.weights.sum() * long.Forward_PE).sum()
PE_fwd_short = (short.weights / short.weights.sum() * short.Forward_PE).sum()
PE_fwd_index = (index.index_weights / index.index_weights.sum() * index.Forward_PE).sum()

# calculate PB Ratio
PB_long = (long.weights / long.weights.sum() * long.PB_Ratio).sum()
PB_short = (short.weights / short.weights.sum() * short.PB_Ratio).sum()
PB_index = (index.index_weights / index.index_weights.sum() * index.PB_Ratio).sum()

# assemble metrics dataframe
ratios_short = {"Yield": format(div_yield_short, ".2%"),
                "Price_Book": round(PB_short, 2),
                "Trailing_PE": round(PE_short, 2),
                "Forward_PE": round(PE_fwd_short, 2)}

ratios_long = {"Yield": format(div_yield_long, ".2%"),
               "Price_Book": round(PB_long, 2),
               "Trailing_PE": round(PE_long, 2),
               "Forward_PE": round(PE_fwd_long, 2)}

ratios_index = {"Yield": format(div_yield_index, ".2%"),
                "Price_Book": round(PB_index, 2),
                "Trailing_PE": round(PE_index, 2),
                "Forward_PE": round(PE_fwd_index, 2)}

ratios_table = pd.DataFrame({"Portfolio Short": ratios_short, "Portfolio Long": ratios_long, "Index": ratios_index})

# export table
dfi.export(ratios_table, "plots/portfolio_characteristics.png")


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

import sklearn.datasets
from sklearn import tree
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
from sklearn.metrics import confusion_matrix

# Load dataset
# (data_full, target_full) = sklearn.datasets.fetch_covtype(return_X_y=True, as_frame=True)
data_full = pd.read_csv('beans.csv')
target_full = data_full['Class']
data_full = data_full.drop('Class', axis = 1)

dt = DecisionTreeClassifier()
dt.fit(data_full, target_full)
importance = dt.feature_importances_
plt.bar([x for x in range(len(importance))], importance)

ind = np.argpartition(importance, -10)[-10:]
i_feat = data_full.columns[ind]
data_full = data_full[i_feat]

# Sample dataset (5% subset)
pct = 1
data = data_full.sample(int(len(data_full)*pct))
target = target_full[data.index]

# Split data into training and testing 70:30
split = train_test_split(data, target, test_size = 0.3)
X_train = split[0]
X_test = split[1]
y_train = split[2]
y_test = split[3]

# Pruning example
# https://scikit-learn.org/stable/auto_examples/tree/plot_cost_complexity_pruning.html#sphx-glr-auto-examples-tree-plot-cost-complexity-pruning-py
path = DecisionTreeClassifier().cost_complexity_pruning_path(X_train, y_train)
alphas, impurities = path.ccp_alphas, path.impurities

# Create trees for different alphas
dts = []
roc_aucs = []
i = 1
for alpha in alphas:
    dt = DecisionTreeClassifier(ccp_alpha=alpha)
    dt.fit(X_train, y_train)
    pred = dt.predict(X_test)
    probs = dt.predict_proba(X_test)
    roc_aucs.append(roc_auc_score(y_test, probs, multi_class='ovo'))
    dts.append(dt)

# Removing the case where tree pruned to only one node
dts = dts[:-1]
alphas = alphas[:-1]
roc_aucs = roc_aucs[:-1]

train_scores = [dt.score(X_train, y_train) for dt in dts]
test_scores = [dt.score(X_test, y_test) for dt in dts]

# Create accuracy vs alpha plots
fig, ax = plt.subplots()
ax.set_xlabel("alpha")
ax.set_ylabel("accuracy")
ax.set_title("Accuracy vs alpha for training and testing sets")
ax.plot(alphas, train_scores, marker="o", label="train", drawstyle="steps-post")
ax.plot(alphas, test_scores, marker="o", label="test", drawstyle="steps-post")
ax.legend()
plt.savefig("beans_ava.png")

plt.figure()
plt.title("ROC AUC vs. Alpha")
plt.ylabel("ROC AUC")
plt.xlabel("Alpha")
plt.plot(alphas, roc_aucs)
plt.savefig("beans_rocaucs.png")

i = np.argmax(roc_aucs)
alpha = alphas[i]
print("Max ROC AUC Alpha: ", alpha)
dtree = DecisionTreeClassifier(ccp_alpha = alpha)

# Split data into training and testing 70:30
split = train_test_split(data_full, target_full, test_size = 0.3)
X_train = split[0]
X_test = split[1]
y_train = split[2]
y_test = split[3]

dtree.fit(X_train, y_train)
score = dtree.score(X_test, y_test)
print("Accuracy: ", score)


from django.shortcuts import render, redirect
from django.http import JsonResponse
from .models import Message

def message_list(request):
    messages = Message.objects.all()
    return render(request, 'message_list.html', {'messages': messages})

def subit(request):
    if request.method == 'POST':
        author = request.user
        content = request.POST.get('content')
        message = Message(author=author, content=content)
        message.save()
        messages = Message.objects.all()
        return render(request, 'message_list.html',{'messages': messages})

def update(request):
    if request.method == 'POST':
        message_id = request.POST.get('message_id')
        new_content = request.POST.get('new_content')

        message = Message.objects.get(id=message_id)
        message.content = new_content
        message.save()

        messages = Message.objects.all()
        return render(request, 'message_list.html', {'messages': messages})

def delete_message(request):
    if request.method == 'POST':
        message_id = request.POST.get('message_id')

        message = Message.objects.get(id=message_id)
        message.content = "訊息已被 " + message.author + " 已收回"
        message.change = 0
        message.save()

        response = {'author': message.author}  # 將 author 包含在回應中
        return JsonResponse(response)


from numbers import Number
from contextlib import contextmanager

class Comparator:

    default_delta_value = 0.0000001

    def __init__(self, value, delta=None):
        if delta is None:
            self.delta = Comparator.default_delta_value
        else:
            self.delta = delta
        self.value = value

    def __eq__(self, other_value):
        # Could use math.isclose as well
        return other_value - self.delta <= self.value <= other_value + self.delta

    def __repr__(self):
        return f"{self.__class__.__name__}({self.value}, delta={self.delta})"

    def __add__(self, other):
        if isinstance(other, Number):
            return Comparator(self.value + other, delta=self.delta)
        elif isinstance(other, Comparator):
            return Comparator(
                self.value + other.value, delta=max(self.delta, other.delta)
            )
        return NotImplemented

    __radd__ = __add__

    def __sub__(self, other):
        if isinstance(other, Number):
            return Comparator(self.value - other, delta=self.delta)
        elif isinstance(other, Comparator):
            return Comparator(
                self.value - other.value, delta=max(self.delta, other.delta)
            )
        return NotImplemented

    def __rsub__(self, other):
        if isinstance(other, Number):
            return Comparator(other - self.value, delta=self.delta)
        elif isinstance(other, Comparator):
            return Comparator(
                other.value - self.value, delta=max(self.delta, other.delta)
            )
        return NotImplemented

    # I think it is more correct to have this as a class method
    @classmethod
    @contextmanager
    def default_delta(cls, delta):
        old_default_delta = cls.default_delta_value
        try:
            cls.default_delta_value = delta
            yield
        finally:
            cls.default_delta_value = old_default_delta

    # Solution:
    #  @contextmanager
    #  def default_delta(delta):
    #      old_default_delta = Comparator.default_delta_value
    #      try:
    #          Comparator.default_delta_value = delta
    #          yield
    #      finally:
    #          Comparator.default_delta_value = old_default_delta


import math
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from voicefixer.vocoder.config import Config

# From xin wang of nii
class SineGen(torch.nn.Module):
    """Definition of sine generator
    SineGen(samp_rate, harmonic_num = 0,
            sine_amp = 0.1, noise_std = 0.003,
            voiced_threshold = 0,
            flag_for_pulse=False)

    samp_rate: sampling rate in Hz
    harmonic_num: number of harmonic overtones (default 0)
    sine_amp: amplitude of sine-wavefrom (default 0.1)
    noise_std: std of Gaussian noise (default 0.003)
    voiced_thoreshold: F0 threshold for U/V classification (default 0)
    flag_for_pulse: this SinGen is used inside PulseGen (default False)

    Note: when flag_for_pulse is True, the first time step of a voiced
        segment is always sin(np.pi) or cos(0)
    """

    def __init__(
        self,
        samp_rate=24000,
        harmonic_num=0,
        sine_amp=0.1,
        noise_std=0.003,
        voiced_threshold=0,
        flag_for_pulse=False,
    ):
        super(SineGen, self).__init__()
        self.sine_amp = sine_amp
        self.noise_std = noise_std
        self.harmonic_num = harmonic_num
        self.dim = self.harmonic_num + 1
        self.sampling_rate = samp_rate
        self.voiced_threshold = voiced_threshold
        self.flag_for_pulse = flag_for_pulse

    def _f02uv(self, f0):
        # generate uv signal
        uv = torch.ones_like(f0)
        uv = uv * (f0 > self.voiced_threshold)
        return uv

    def _f02sine(self, f0_values):
        """f0_values: (batchsize, length, dim)
        where dim indicates fundamental tone and overtones
        """
        # convert to F0 in rad. The interger part n can be ignored
        # because 2 * np.pi * n doesn't affect phase
        rad_values = (f0_values / self.sampling_rate) % 1

        # initial phase noise (no noise for fundamental component)
        rand_ini = torch.rand(
            f0_values.shape[0], f0_values.shape[2], device=f0_values.device
        )
        rand_ini[:, 0] = 0
        rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini

        # instantanouse phase sine[t] = sin(2*pi \sum_i=1 ^{t} rad)
        if not self.flag_for_pulse:
            # for normal case

            # To prevent torch.cumsum numerical overflow,
            # it is necessary to add -1 whenever \sum_k=1^n rad_value_k > 1.
            # Buffer tmp_over_one_idx indicates the time step to add -1.
            # This will not change F0 of sine because (x-1) * 2*pi = x *2*pi
            tmp_over_one = torch.cumsum(rad_values, 1) % 1
            tmp_over_one_idx = (tmp_over_one[:, 1:, :] - tmp_over_one[:, :-1, :]) < 0
            cumsum_shift = torch.zeros_like(rad_values)
            cumsum_shift[:, 1:, :] = tmp_over_one_idx * -1.0

            sines = torch.sin(
                torch.cumsum(rad_values + cumsum_shift, dim=1) * 2 * np.pi
            )
        else:
            # If necessary, make sure that the first time step of every
            # voiced segments is sin(pi) or cos(0)
            # This is used for pulse-train generation

            # identify the last time step in unvoiced segments
            uv = self._f02uv(f0_values)
            uv_1 = torch.roll(uv, shifts=-1, dims=1)
            uv_1[:, -1, :] = 1
            u_loc = (uv < 1) * (uv_1 > 0)

            # get the instantanouse phase
            tmp_cumsum = torch.cumsum(rad_values, dim=1)
            # different batch needs to be processed differently
            for idx in range(f0_values.shape[0]):
                temp_sum = tmp_cumsum[idx, u_loc[idx, :, 0], :]
                temp_sum[1:, :] = temp_sum[1:, :] - temp_sum[0:-1, :]
                # stores the accumulation of i.phase within
                # each voiced segments
                tmp_cumsum[idx, :, :] = 0
                tmp_cumsum[idx, u_loc[idx, :, 0], :] = temp_sum

            # rad_values - tmp_cumsum: remove the accumulation of i.phase
            # within the previous voiced segment.
            i_phase = torch.cumsum(rad_values - tmp_cumsum, dim=1)

            # get the sines
            sines = torch.cos(i_phase * 2 * np.pi)
        return sines

    def forward(self, f0):
        """sine_tensor, uv = forward(f0)
        input F0: tensor(batchsize=1, length, dim=1)
                  f0 for unvoiced steps should be 0
        output sine_tensor: tensor(batchsize=1, length, dim)
        output uv: tensor(batchsize=1, length, 1)
        """

        with torch.no_grad():
            f0_buf = torch.zeros(f0.shape[0], f0.shape[1], self.dim, device=f0.device)
            # fundamental component
            f0_buf[:, :, 0] = f0[:, :, 0]
            for idx in np.arange(self.harmonic_num):
                # idx + 2: the (idx+1)-th overtone, (idx+2)-th harmonic
                f0_buf[:, :, idx + 1] = f0_buf[:, :, 0] * (idx + 2)

            # generate sine waveforms
            sine_waves = self._f02sine(f0_buf) * self.sine_amp

            # generate uv signal
            # uv = torch.ones(f0.shape)
            # uv = uv * (f0 > self.voiced_threshold)
            uv = self._f02uv(f0)

            # noise: for unvoiced should be similar to sine_amp
            #        std = self.sine_amp/3 -> max value ~ self.sine_amp
            # .       for voiced regions is self.noise_std
            noise_amp = uv * self.noise_std + (1 - uv) * self.sine_amp / 3
            noise = noise_amp * torch.randn_like(sine_waves)

            # first: set the unvoiced part to 0 by uv
            # then: additive noise
            sine_waves = sine_waves * uv + noise
        return sine_waves, uv, noise

class LowpassBlur(nn.Module):
    """perform low pass filter after upsampling for anti-aliasing"""

    def __init__(self, channels=128, filt_size=3, pad_type="reflect", pad_off=0):
        super(LowpassBlur, self).__init__()
        self.filt_size = filt_size
        self.pad_off = pad_off
        self.pad_sizes = [
            int(1.0 * (filt_size - 1) / 2),
            int(np.ceil(1.0 * (filt_size - 1) / 2)),
        ]
        self.pad_sizes = [pad_size + pad_off for pad_size in self.pad_sizes]
        self.off = 0
        self.channels = channels

        if self.filt_size == 1:
            a = np.array(
                [
                    1.0,
                ]
            )
        elif self.filt_size == 2:
            a = np.array([1.0, 1.0])
        elif self.filt_size == 3:
            a = np.array([1.0, 2.0, 1.0])
        elif self.filt_size == 4:
            a = np.array([1.0, 3.0, 3.0, 1.0])
        elif self.filt_size == 5:
            a = np.array([1.0, 4.0, 6.0, 4.0, 1.0])
        elif self.filt_size == 6:
            a = np.array([1.0, 5.0, 10.0, 10.0, 5.0, 1.0])
        elif self.filt_size == 7:
            a = np.array([1.0, 6.0, 15.0, 20.0, 15.0, 6.0, 1.0])

        filt = torch.Tensor(a)
        filt = filt / torch.sum(filt)
        self.register_buffer("filt", filt[None, None, :].repeat((self.channels, 1, 1)))

        self.pad = get_pad_layer_1d(pad_type)(self.pad_sizes)

    def forward(self, inp):
        if self.filt_size == 1:
            return inp
        return F.conv1d(self.pad(inp), self.filt, groups=inp.shape[1])

def get_pad_layer_1d(pad_type):
    if pad_type in ["refl", "reflect"]:
        PadLayer = nn.ReflectionPad1d
    elif pad_type in ["repl", "replicate"]:
        PadLayer = nn.ReplicationPad1d
    elif pad_type == "zero":
        PadLayer = nn.ZeroPad1d
    else:
        print("Pad type [%s] not recognized" % pad_type)
    return PadLayer

class MovingAverageSmooth(torch.nn.Conv1d):
    def __init__(self, channels, window_len=3):
        """Initialize Conv1d module."""
        super(MovingAverageSmooth, self).__init__(
            in_channels=channels,
            out_channels=channels,
            kernel_size=1,
            groups=channels,
            bias=False,
        )

        torch.nn.init.constant_(self.weight, 1.0 / window_len)
        for p in self.parameters():
            p.requires_grad = False

    def forward(self, data):
        return super(MovingAverageSmooth, self).forward(data)

class Conv1d(torch.nn.Conv1d):
    """Conv1d module with customized initialization."""

    def __init__(self, *args, **kwargs):
        """Initialize Conv1d module."""
        super(Conv1d, self).__init__(*args, **kwargs)

    def reset_parameters(self):
        """Reset parameters."""
        torch.nn.init.kaiming_normal_(self.weight, nonlinearity="relu")
        if self.bias is not None:
            torch.nn.init.constant_(self.bias, 0.0)

class Stretch2d(torch.nn.Module):
    """Stretch2d module."""

    def __init__(self, x_scale, y_scale, mode="nearest"):
        """Initialize Stretch2d module.
        Args:
            x_scale (int): X scaling factor (Time axis in spectrogram).
            y_scale (int): Y scaling factor (Frequency axis in spectrogram).
            mode (str): Interpolation mode.
        """
        super(Stretch2d, self).__init__()
        self.x_scale = x_scale
        self.y_scale = y_scale
        self.mode = mode

    def forward(self, x):
        """Calculate forward propagation.
        Args:
            x (Tensor): Input tensor (B, C, F, T).
        Returns:
            Tensor: Interpolated tensor (B, C, F * y_scale, T * x_scale),
        """
        return F.interpolate(
            x, scale_factor=(self.y_scale, self.x_scale), mode=self.mode
        )

class Conv2d(torch.nn.Conv2d):
    """Conv2d module with customized initialization."""

    def __init__(self, *args, **kwargs):
        """Initialize Conv2d module."""
        super(Conv2d, self).__init__(*args, **kwargs)

    def reset_parameters(self):
        """Reset parameters."""
        self.weight.data.fill_(1.0 / np.prod(self.kernel_size))
        if self.bias is not None:
            torch.nn.init.constant_(self.bias, 0.0)

class UpsampleNetwork(torch.nn.Module):
    """Upsampling network module."""

    def __init__(
        self,
        upsample_scales,
        nonlinear_activation=None,
        nonlinear_activation_params={},
        interpolate_mode="nearest",
        freq_axis_kernel_size=1,
        use_causal_conv=False,
    ):
        """Initialize upsampling network module.
        Args:
            upsample_scales (list): List of upsampling scales.
            nonlinear_activation (str): Activation function name.
            nonlinear_activation_params (dict): Arguments for specified activation function.
            interpolate_mode (str): Interpolation mode.
            freq_axis_kernel_size (int): Kernel size in the direction of frequency axis.
        """
        super(UpsampleNetwork, self).__init__()
        self.use_causal_conv = use_causal_conv
        self.up_layers = torch.nn.ModuleList()
        for scale in upsample_scales:
            # interpolation layer
            stretch = Stretch2d(scale, 1, interpolate_mode)
            self.up_layers += [stretch]

            # conv layer
            assert (
                freq_axis_kernel_size - 1
            ) % 2 == 0, "Not support even number freq axis kernel size."
            freq_axis_padding = (freq_axis_kernel_size - 1) // 2
            kernel_size = (freq_axis_kernel_size, scale * 2 + 1)
            if use_causal_conv:
                padding = (freq_axis_padding, scale * 2)
            else:
                padding = (freq_axis_padding, scale)
            conv = Conv2d(1, 1, kernel_size=kernel_size, padding=padding, bias=False)
            self.up_layers += [conv]

            # nonlinear
            if nonlinear_activation is not None:
                nonlinear = getattr(torch.nn, nonlinear_activation)(
                    **nonlinear_activation_params
                )
                self.up_layers += [nonlinear]

    def forward(self, c):
        """Calculate forward propagation.
        Args:
            c : Input tensor (B, C, T).
        Returns:
            Tensor: Upsampled tensor (B, C, T'), where T' = T * prod(upsample_scales).
        """
        c = c.unsqueeze(1)  # (B, 1, C, T)
        for f in self.up_layers:
            if self.use_causal_conv and isinstance(f, Conv2d):
                c = f(c)[..., : c.size(-1)]
            else:
                c = f(c)
        return c.squeeze(1)  # (B, C, T')

class ConvInUpsampleNetwork(torch.nn.Module):
    """Convolution + upsampling network module."""

    def __init__(
        self,
        upsample_scales=[3, 4, 5, 5],
        nonlinear_activation="ReLU",
        nonlinear_activation_params={},
        interpolate_mode="nearest",
        freq_axis_kernel_size=1,
        aux_channels=80,
        aux_context_window=0,
        use_causal_conv=False,
    ):
        """Initialize convolution + upsampling network module.
        Args:
            upsample_scales (list): List of upsampling scales.
            nonlinear_activation (str): Activation function name.
            nonlinear_activation_params (dict): Arguments for specified activation function.
            mode (str): Interpolation mode.
            freq_axis_kernel_size (int): Kernel size in the direction of frequency axis.
            aux_channels (int): Number of channels of pre-convolutional layer.
            aux_context_window (int): Context window size of the pre-convolutional layer.
            use_causal_conv (bool): Whether to use causal structure.
        """
        super(ConvInUpsampleNetwork, self).__init__()
        self.aux_context_window = aux_context_window
        self.use_causal_conv = use_causal_conv and aux_context_window > 0
        # To capture wide-context information in conditional features
        kernel_size = (
            aux_context_window + 1 if use_causal_conv else 2 * aux_context_window + 1
        )
        # NOTE(kan-bayashi): Here do not use padding because the input is already padded
        self.conv_in = Conv1d(
            aux_channels, aux_channels, kernel_size=kernel_size, bias=False
        )
        self.upsample = UpsampleNetwork(
            upsample_scales=upsample_scales,
            nonlinear_activation=nonlinear_activation,
            nonlinear_activation_params=nonlinear_activation_params,
            interpolate_mode=interpolate_mode,
            freq_axis_kernel_size=freq_axis_kernel_size,
            use_causal_conv=use_causal_conv,
        )

    def forward(self, c):
        """Calculate forward propagation.
        Args:
            c : Input tensor (B, C, T').
        Returns:
            Tensor: Upsampled tensor (B, C, T),
                where T = (T' - aux_context_window * 2) * prod(upsample_scales).
        Note:
            The length of inputs considers the context window size.
        """
        c_ = self.conv_in(c)
        c = c_[:, :, : -self.aux_context_window] if self.use_causal_conv else c_
        return self.upsample(c)

class DownsampleNet(nn.Module):
    def __init__(self, input_size, output_size, upsample_factor, hp=None, index=0):
        super(DownsampleNet, self).__init__()
        self.input_size = input_size
        self.output_size = output_size
        self.upsample_factor = upsample_factor
        self.skip_conv = nn.Conv1d(input_size, output_size, kernel_size=1)
        self.index = index
        layer = nn.Conv1d(
            input_size,
            output_size,
            kernel_size=upsample_factor * 2,
            stride=upsample_factor,
            padding=upsample_factor // 2 + upsample_factor % 2,
        )

        self.layer = nn.utils.weight_norm(layer)

    def forward(self, inputs):
        B, C, T = inputs.size()
        res = inputs[:, :, :: self.upsample_factor]
        skip = self.skip_conv(res)

        outputs = self.layer(inputs)
        outputs = outputs + skip

        return outputs

class UpsampleNet(nn.Module):
    def __init__(self, input_size, output_size, upsample_factor, hp=None, index=0):

        super(UpsampleNet, self).__init__()
        self.up_type = Config.up_type
        self.use_smooth = Config.use_smooth
        self.use_drop = Config.use_drop
        self.input_size = input_size
        self.output_size = output_size
        self.upsample_factor = upsample_factor
        self.skip_conv = nn.Conv1d(input_size, output_size, kernel_size=1)
        self.index = index
        if self.use_smooth:
            window_lens = [5, 5, 4, 3]
            self.window_len = window_lens[index]

        if self.up_type != "pn" or self.index < 3:
            # if self.up_type != "pn":
            layer = nn.ConvTranspose1d(
                input_size,
                output_size,
                upsample_factor * 2,
                upsample_factor,
                padding=upsample_factor // 2 + upsample_factor % 2,
                output_padding=upsample_factor % 2,
            )
            self.layer = nn.utils.weight_norm(layer)
        else:
            self.layer = nn.Sequential(
                nn.ReflectionPad1d(1),
                nn.utils.weight_norm(
                    nn.Conv1d(input_size, output_size * upsample_factor, kernel_size=3)
                ),
                nn.LeakyReLU(),
                nn.ReflectionPad1d(1),
                nn.utils.weight_norm(
                    nn.Conv1d(
                        output_size * upsample_factor,
                        output_size * upsample_factor,
                        kernel_size=3,
                    )
                ),
                nn.LeakyReLU(),
                nn.ReflectionPad1d(1),
                nn.utils.weight_norm(
                    nn.Conv1d(
                        output_size * upsample_factor,
                        output_size * upsample_factor,
                        kernel_size=3,
                    )
                ),
                nn.LeakyReLU(),
            )

        if hp is not None:
            self.org = Config.up_org
            self.no_skip = Config.no_skip
        else:
            self.org = False
            self.no_skip = True

        if self.use_smooth:
            self.mas = nn.Sequential(
                # LowpassBlur(output_size, self.window_len),
                MovingAverageSmooth(output_size, self.window_len),
                # MovingAverageSmooth(output_size, self.window_len),
            )

    def forward(self, inputs):

        if not self.org:
            inputs = inputs + torch.sin(inputs)
            B, C, T = inputs.size()
            res = inputs.repeat(1, self.upsample_factor, 1).view(B, C, -1)
            skip = self.skip_conv(res)
            if self.up_type == "repeat":
                return skip

        outputs = self.layer(inputs)
        if self.up_type == "pn" and self.index > 2:
            B, c, l = outputs.size()
            outputs = outputs.view(B, -1, l * self.upsample_factor)

        if self.no_skip:
            return outputs

        if not self.org:
            outputs = outputs + skip

        if self.use_smooth:
            outputs = self.mas(outputs)

        if self.use_drop:
            outputs = F.dropout(outputs, p=0.05)

        return outputs

class ResStack(nn.Module):
    def __init__(self, channel, kernel_size=3, resstack_depth=4, hp=None):
        super(ResStack, self).__init__()

        self.use_wn = Config.use_wn
        self.use_shift_scale = Config.use_shift_scale
        self.channel = channel

        def get_padding(kernel_size, dilation=1):
            return int((kernel_size * dilation - dilation) / 2)

        if self.use_shift_scale:
            self.scale_conv = nn.utils.weight_norm(
                nn.Conv1d(
                    channel, 2 * channel, kernel_size=kernel_size, dilation=1, padding=1
                )
            )

        if not self.use_wn:
            self.layers = nn.ModuleList(
                [
                    nn.Sequential(
                        nn.LeakyReLU(),
                        nn.utils.weight_norm(
                            nn.Conv1d(
                                channel,
                                channel,
                                kernel_size=kernel_size,
                                dilation=3 ** (i % 10),
                                padding=get_padding(kernel_size, 3 ** (i % 10)),
                            )
                        ),
                        nn.LeakyReLU(),
                        nn.utils.weight_norm(
                            nn.Conv1d(
                                channel,
                                channel,
                                kernel_size=kernel_size,
                                dilation=1,
                                padding=get_padding(kernel_size, 1),
                            )
                        ),
                    )
                    for i in range(resstack_depth)
                ]
            )
        else:
            self.wn = WaveNet(
                in_channels=channel,
                out_channels=channel,
                cin_channels=-1,
                num_layers=resstack_depth,
                residual_channels=channel,
                gate_channels=channel,
                skip_channels=channel,
                # kernel_size=5,
                # dilation_rate=3,
                causal=False,
                use_downup=False,
            )

    def forward(self, x):
        if not self.use_wn:
            for layer in self.layers:
                x = x + layer(x)
        else:
            x = self.wn(x)

        if self.use_shift_scale:
            m_s = self.scale_conv(x)
            m_s = m_s[:, :, :-1]

            m, s = torch.split(m_s, self.channel, dim=1)
            s = F.softplus(s)

            x = m + s * x[:, :, 1:]  # key!!!
            x = F.pad(x, pad=(1, 0), mode="constant", value=0)

        return x

class WaveNet(nn.Module):
    def __init__(
        self,
        in_channels=1,
        out_channels=1,
        num_layers=10,
        residual_channels=64,
        gate_channels=64,
        skip_channels=64,
        kernel_size=3,
        dilation_rate=2,
        cin_channels=80,
        hp=None,
        causal=False,
        use_downup=False,
    ):
        super(WaveNet, self).__init__()

        self.in_channels = in_channels
        self.causal = causal
        self.num_layers = num_layers
        self.out_channels = out_channels
        self.gate_channels = gate_channels
        self.residual_channels = residual_channels
        self.skip_channels = skip_channels
        self.cin_channels = cin_channels
        self.kernel_size = kernel_size
        self.use_downup = use_downup

        self.front_conv = nn.Sequential(
            nn.Conv1d(
                in_channels=self.in_channels,
                out_channels=self.residual_channels,
                kernel_size=3,
                padding=1,
            ),
            nn.ReLU(),
        )
        if self.use_downup:
            self.downup_conv = nn.Sequential(
                nn.Conv1d(
                    in_channels=self.residual_channels,
                    out_channels=self.residual_channels,
                    kernel_size=3,
                    stride=2,
                    padding=1,
                ),
                nn.ReLU(),
                nn.Conv1d(
                    in_channels=self.residual_channels,
                    out_channels=self.residual_channels,
                    kernel_size=3,
                    stride=2,
                    padding=1,
                ),
                nn.ReLU(),
                UpsampleNet(self.residual_channels, self.residual_channels, 4, hp),
            )

        self.res_blocks = nn.ModuleList()
        for n in range(self.num_layers):
            self.res_blocks.append(
                ResBlock(
                    self.residual_channels,
                    self.gate_channels,
                    self.skip_channels,
                    self.kernel_size,
                    dilation=dilation_rate**n,
                    cin_channels=self.cin_channels,
                    local_conditioning=(self.cin_channels > 0),
                    causal=self.causal,
                    mode="SAME",
                )
            )
        self.final_conv = nn.Sequential(
            nn.ReLU(),
            Conv(self.skip_channels, self.skip_channels, 1, causal=self.causal),
            nn.ReLU(),
            Conv(self.skip_channels, self.out_channels, 1, causal=self.causal),
        )

    def forward(self, x, c=None):
        return self.wavenet(x, c)

    def wavenet(self, tensor, c=None):

        h = self.front_conv(tensor)
        if self.use_downup:
            h = self.downup_conv(h)
        skip = 0
        for i, f in enumerate(self.res_blocks):
            h, s = f(h, c)
            skip += s
        out = self.final_conv(skip)
        return out

    def receptive_field_size(self):
        num_dir = 1 if self.causal else 2
        dilations = [2 ** (i % self.num_layers) for i in range(self.num_layers)]
        return (
            num_dir * (self.kernel_size - 1) * sum(dilations)
            + 1
            + (self.front_channels - 1)
        )

    def remove_weight_norm(self):
        for f in self.res_blocks:
            f.remove_weight_norm()

class Conv(nn.Module):
    def __init__(
        self,
        in_channels,
        out_channels,
        kernel_size,
        dilation=1,
        causal=False,
        mode="SAME",
    ):
        super(Conv, self).__init__()

        self.causal = causal
        self.mode = mode
        if self.causal and self.mode == "SAME":
            self.padding = dilation * (kernel_size - 1)
        elif self.mode == "SAME":
            self.padding = dilation * (kernel_size - 1) // 2
        else:
            self.padding = 0
        self.conv = nn.Conv1d(
            in_channels,
            out_channels,
            kernel_size,
            dilation=dilation,
            padding=self.padding,
        )
        self.conv = nn.utils.weight_norm(self.conv)
        nn.init.kaiming_normal_(self.conv.weight)

    def forward(self, tensor):
        out = self.conv(tensor)
        if self.causal and self.padding is not 0:
            out = out[:, :, : -self.padding]
        return out

    def remove_weight_norm(self):
        nn.utils.remove_weight_norm(self.conv)

class ResBlock(nn.Module):
    def __init__(
        self,
        in_channels,
        out_channels,
        skip_channels,
        kernel_size,
        dilation,
        cin_channels=None,
        local_conditioning=True,
        causal=False,
        mode="SAME",
    ):
        super(ResBlock, self).__init__()
        self.causal = causal
        self.local_conditioning = local_conditioning
        self.cin_channels = cin_channels
        self.mode = mode

        self.filter_conv = Conv(
            in_channels, out_channels, kernel_size, dilation, causal, mode
        )
        self.gate_conv = Conv(
            in_channels, out_channels, kernel_size, dilation, causal, mode
        )
        self.res_conv = nn.Conv1d(out_channels, in_channels, kernel_size=1)
        self.skip_conv = nn.Conv1d(out_channels, skip_channels, kernel_size=1)
        self.res_conv = nn.utils.weight_norm(self.res_conv)
        self.skip_conv = nn.utils.weight_norm(self.skip_conv)

        if self.local_conditioning:
            self.filter_conv_c = nn.Conv1d(cin_channels, out_channels, kernel_size=1)
            self.gate_conv_c = nn.Conv1d(cin_channels, out_channels, kernel_size=1)
            self.filter_conv_c = nn.utils.weight_norm(self.filter_conv_c)
            self.gate_conv_c = nn.utils.weight_norm(self.gate_conv_c)

    def forward(self, tensor, c=None):
        h_filter = self.filter_conv(tensor)
        h_gate = self.gate_conv(tensor)

        if self.local_conditioning:
            h_filter += self.filter_conv_c(c)
            h_gate += self.gate_conv_c(c)

        out = torch.tanh(h_filter) * torch.sigmoid(h_gate)

        res = self.res_conv(out)
        skip = self.skip_conv(out)
        if self.mode == "SAME":
            return (tensor + res) * math.sqrt(0.5), skip
        else:
            return (tensor[:, :, 1:] + res) * math.sqrt(0.5), skip

    def remove_weight_norm(self):
        self.filter_conv.remove_weight_norm()
        self.gate_conv.remove_weight_norm()
        nn.utils.remove_weight_norm(self.res_conv)
        nn.utils.remove_weight_norm(self.skip_conv)
        nn.utils.remove_weight_norm(self.filter_conv_c)
        nn.utils.remove_weight_norm(self.gate_conv_c)

@torch.jit.script
def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
    n_channels_int = n_channels[0]
    in_act = input_a + input_b
    t_act = torch.tanh(in_act[:, :n_channels_int])
    s_act = torch.sigmoid(in_act[:, n_channels_int:])
    acts = t_act * s_act
    return acts

@torch.jit.script
def fused_res_skip(tensor, res_skip, n_channels):
    n_channels_int = n_channels[0]
    res = res_skip[:, :n_channels_int]
    skip = res_skip[:, n_channels_int:]
    return (tensor + res), skip

class ResStack2D(nn.Module):
    def __init__(self, channels=16, kernel_size=3, resstack_depth=4, hp=None):
        super(ResStack2D, self).__init__()
        channels = 16
        kernel_size = 3
        resstack_depth = 2
        self.channels = channels

        def get_padding(kernel_size, dilation=1):
            return int((kernel_size * dilation - dilation) / 2)

        self.layers = nn.ModuleList(
            [
                nn.Sequential(
                    nn.LeakyReLU(),
                    nn.utils.weight_norm(
                        nn.Conv2d(
                            1,
                            self.channels,
                            kernel_size,
                            dilation=(1, 3 ** (i)),
                            padding=(1, get_padding(kernel_size, 3 ** (i))),
                        )
                    ),
                    nn.LeakyReLU(),
                    nn.utils.weight_norm(
                        nn.Conv2d(
                            self.channels,
                            self.channels,
                            kernel_size,
                            dilation=(1, 3 ** (i)),
                            padding=(1, get_padding(kernel_size, 3 ** (i))),
                        )
                    ),
                    nn.LeakyReLU(),
                    nn.utils.weight_norm(nn.Conv2d(self.channels, 1, kernel_size=1)),
                )
                for i in range(resstack_depth)
            ]
        )

    def forward(self, tensor):
        x = tensor.unsqueeze(1)
        for layer in self.layers:
            x = x + layer(x)
        x = x.squeeze(1)

        return x

class FiLM(nn.Module):
    """
    feature-wise linear modulation
    """

    def __init__(self, input_dim, attribute_dim):
        super().__init__()
        self.input_dim = input_dim
        self.generator = nn.Conv1d(
            attribute_dim, input_dim * 2, kernel_size=3, padding=1
        )

    def forward(self, x, c):
        """
        x: (B, input_dim, seq)
        c: (B, attribute_dim, seq)
        """
        c = self.generator(c)
        m, s = torch.split(c, self.input_dim, dim=1)

        return x * s + m

class FiLMConv1d(nn.Module):
    """
    Conv1d with FiLMs in between
    """

    def __init__(self, in_size, out_size, attribute_dim, ins_norm=True, loop=1):
        super().__init__()
        self.loop = loop
        self.mlps = nn.ModuleList(
            [nn.Conv1d(in_size, out_size, kernel_size=3, padding=1)]
            + [
                nn.Conv1d(out_size, out_size, kernel_size=3, padding=1)
                for i in range(loop - 1)
            ]
        )
        self.films = nn.ModuleList([FiLM(out_size, attribute_dim) for i in range(loop)])
        self.ins_norm = ins_norm
        if self.ins_norm:
            self.norm = nn.InstanceNorm1d(attribute_dim)

    def forward(self, x, c):
        """
        x: (B, input_dim, seq)
        c: (B, attribute_dim, seq)
        """
        if self.ins_norm:
            c = self.norm(c)
        for i in range(self.loop):
            x = self.mlps[i](x)
            x = F.relu(x)
            x = self.films[i](x, c)

        return x


import os
import sys
import glob
import random
import logging
import argparse
from os import path, makedirs, environ
from card_generator import create_card
from spotipy import SpotifyClientCredentials, Spotify, SpotifyException

logging.basicConfig(stream=sys.stderr, level=logging.WARNING)

auth_manager = SpotifyClientCredentials(
    environ["SPOTIFY_CLIENT_ID"], environ["SPOTIFY_CLIENT_SECRET"]
)
spotify = Spotify(auth_manager=auth_manager)
PLAYLIST_URL = (
    "https://open.spotify.com/playlist/5tl7iMYPGspVvMgryoA2HS?si=3812eee2d17a4cc5"
)
# playlist = spotify.playlist(PLAYLIST_URL)

# NUM_SONGS = playlist["tracks"]["total"]

PLAYLISTS = [
    "https://open.spotify.com/playlist/02Ze53FXXBgaSTfB9jpV1N?si=1a29aeb0e6b94621",  # 70s 1
    "https://open.spotify.com/playlist/2ZXxE8nsKuXplbOGVXRmk7?si=3508c19bb14343b0",  # 80s 1
    "https://open.spotify.com/playlist/4akSDZd9ppJx6LEhpWUXnJ?si=dce9d73d30714749",  # 90s 1
    "https://open.spotify.com/playlist/4WSK9EZdrSmVn94oh9fC0F?si=489fe486218b462c",  # 00s 1
    "https://open.spotify.com/playlist/340iasv2MTZsqAJBZHvScl?si=da6837376dd94468",  # 00s 2
    "https://open.spotify.com/playlist/6m9nAnMWZZ37nKqb8VxgT3?si=1ab09f9777c6417d",  # 10s 1
    "https://open.spotify.com/playlist/18i1CLpGUR1F0zl4IEWsOm?si=548426d4703d4b24",  # 10s 2
    "https://open.spotify.com/playlist/312n27j1xsU0Ee1NkyDdFw?si=0cca0803da4f44b2",  # 10s 3
]

def get_song_list(playlist_url: str) -> list:
    """Retrieves a Spotify playlist

    Args:
        playlist_url: the url of the desired playlist

    Returns:
        a list of tuples (title, artist) for the entire playlist
    """
    song_list = []
    results = spotify.playlist_tracks(playlist_url)
    tracks = results["items"]
    while results["next"]:
        results = spotify.next(results)
        tracks.extend(results["items"])

    for song in tracks:
        artists = ""
        for artist in song["track"]["artists"]:
            artists += artist["name"] + ", "
        artists = artists[:-2]

        song_list.append((song["track"]["name"], artists))
    return song_list

def generate_24_numbers(playlist: int = None) -> list:
    """Generates 24 unique numbers within the range of the playlist

    Returns:
        a list of numbers
    """
    url = PLAYLISTS[playlist] if playlist is not None else PLAYLIST_URL
    num_songs = spotify.playlist(url)["tracks"]["total"]

    nums = []
    div, mod = divmod(num_songs, 5)

    for i in range(5):
        nums.extend(
            random.sample(
                range(i * div + min(i, mod), (i + 1) * div + min(i + 1, mod)),
                5 if i != 2 else 4,
            )
        )

    return nums

def split_list(list_: list, n: int) -> list:
    """ "Splits a list into n (roughly) equal lists

    Args:
        list_: desired list to split
        n: number of desired sublists
    Returns:
        list of n sublists
    """
    div, mod = divmod(len(list_), n)

    return [
        list_[i * div + min(i, mod) : (i + 1) * div + min(i + 1, mod)] for i in range(n)
    ]

def add_custom_playlist(url: str) -> bool:
    try:
        spotify.playlist(PLAYLIST_URL)
    except SpotifyException:
        return False

    PLAYLISTS.append(url)
    return True

def generate_card(seed: int = None, playlist: int = None) -> int:
    """Generates a bingo card

    Args:
        seed: seed to use for card generation

    Returns:
        the seed of the generated bingo card
    """
    if seed:
        random.seed(int(seed))
    else:
        seed = random.randint(1000000, 99999999)
        logging.info(f"Seed: {seed}")
        random.seed(seed)

    nums = generate_24_numbers(playlist)
    logging.info(f"{nums}")

    song_list = get_song_list(
        PLAYLISTS[playlist] if playlist is not None else PLAYLIST_URL
    )

    # Shuffle list in a repeatable manner (i.e. shuffle the same way every time)
    random.Random(1).shuffle(song_list)

    card_songs = []
    for num in nums:
        card_songs.append(song_list[num])

    if not path.exists("output/"):
        makedirs("output/")

    # Remove previous generated bingo cards
    files = glob.glob("output/*.jpg")
    for f in files:
        os.remove(f)

    create_card(card_songs).save(f"output/bingo_card-{seed}.jpg")
    return int(seed)

if __name__ == "__main__":
    logging.getLogger().setLevel(logging.INFO)

    parser = argparse.ArgumentParser(description="Generate seed and grid.")
    parser.add_argument("--load", dest="seed", action="store")

    args = parser.parse_args()

    seed = generate_card(args.seed)


from flask import Blueprint, render_template, redirect, url_for, request
from flask_login import current_user, login_required
from app import db
from models import Item

selling = Blueprint('selling', __name__)

@selling.route('/create-listing')
@login_required
def listing():
    return render_template('order/listing.html')

@selling.route('/create-listing', methods=['POST'])
@login_required
def listing_post():
    itemname = request.form.get('itemname')
    price = request.form.get('price')
    imageurl = request.form.get('imageurl')
    description = request.form.get('description')
    qty = request.form.get('qty')

    item = Item(itemname=itemname, merchant_id=current_user.id, price=price, imageurl=imageurl, description=description, qty=qty)
    db.session.add(item)
    db.session.commit()

    return redirect(url_for('user_profile.profile'))

@selling.route('/inventory/item/<item_id>')
@login_required
def view_item(item_id):
    item = Item.query.filter_by(id=item_id).first()

    if (current_user.id == item.merchant_id):
        return render_template('order/inventory.html',
            itemname=item.itemname,
            imageurl=item.imageurl,
            description=item.description,
            price="{:,.2f}".format(item.price),
            qty=item.qty
        )
    else:
        return render_template('permission_error.html')

@selling.route('/inventory/item/<item_id>/edit')
@login_required
def edit_item(item_id):
    item = Item.query.filter_by(id=item_id).first()

    if (current_user.id == item.merchant_id):
        return render_template('order/inventory.html',
            itemname=item.itemname,
            imageurl=item.imageurl,
            description=item.description,
            price="{:,.2f}".format(item.price),
            qty=item.qty
        )
    else:
        return render_template('permission_error.html')


import matplotlib.pyplot as plt
import os
import sys
import csv
import tensorflow as tf
import numpy as np
from util import score_linear_regression, score_logistic_regression, pretty
import pandas as pd
import argparse

pd.set_option('display.width', 1000)
np.set_printoptions(suppress=True)

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

np.random.seed(0)

# ################################################## Args
parser = argparse.ArgumentParser(description="Model Metrixs")
parser.add_argument('--sname', type=str, help="script name", required=True)
parser.add_argument('--model', type=str, help="model", default='logistic')
args, unparsed = parser.parse_known_args()
# ##################################################

csvprefix = "./log/"
tfcsvprefix = "./log/tf-" + args.sname + "/tf"
rttcsvprefix = "./log/rtt-" + args.sname + "/rtt"

y_real_file = tfcsvprefix + "-real-Y.csv"
y_pred_file_tf = tfcsvprefix + "-pred-Y.csv"
y_pred_file_rtt = rttcsvprefix + "-pred-Y.csv"

Y = pd.read_csv(y_real_file, sep=',', header=None, names=['L'])
predYtf = pd.read_csv(y_pred_file_tf, sep=',', header=None, names=['L'])
predYrtt = pd.read_csv(y_pred_file_rtt, sep=',', header=None, names=['L'])

print(args.model)
if args.model == 'logistic':
    emetrixs = score_logistic_regression(
        predYtf.to_numpy(), Y.to_numpy(), tag='tensorflow')
    print(pretty(emetrixs))

    emetrixs = score_logistic_regression(
        predYrtt.to_numpy(), Y.to_numpy(), tag='rosetta')
    print(pretty(emetrixs))

    # hist
    plt.title("tensorflow rosetta")
    plt.xlabel("Probability")
    plt.ylabel("Frequency")
    plt.hist(predYtf.to_numpy(), bins=50, label="TensorFlow")
    plt.hist(predYrtt.to_numpy(), bins=50, label="Rosetta")
    plt.legend()
    plt.savefig(csvprefix+"/" + args.sname + "-hist.png")
    plt.clf()

if args.model == 'linear':
    emetrixs = score_linear_regression(
        predYtf.to_numpy(), Y.to_numpy(), tag='tensorflow')
    print(pretty(emetrixs))

    emetrixs = score_linear_regression(
        predYrtt.to_numpy(), Y.to_numpy(), tag='rosetta')
    print(pretty(emetrixs))


#Problem: https://www.hackerrank.com/challenges/s10-mcq-5/problem

from itertools import permutations

deck = [(suit, rank) for suit in ['a', 'b', 'c', 'd'] for rank in [*range(1,14)]]
all_options = [*permutations(deck, 2)]
correct_options = [card for card in all_options if card[0][0] == card[1][0]]
solution = len(correct_options)  / len(all_options)
print(round(solution, 4))

10_Days_of_Statistics/Day 3: Cards of the Same Suit.py


numbers = open("AOC_txt_18").read().split("\n")

def add(a, b):
    return '[' + a + ',' + b + ']'

def magnitude(x):
    if type(x) == list:
        return 3 * magnitude(x[0]) + 2 * magnitude(x[1])
    else:
        return x

def str_to_snb(string):
    pile = []
    for elt in string:
        if elt.isdigit():
            pile.append(int(elt))
        elif elt == ']':
            b = pile.pop()
            a = pile.pop()
            pile.append([a, b])
    return pile[0]

def split(n):
    return n // 2, n // 2 + n % 2

def stack_to_str(stack):
    res = ''
    for k in range(len(stack)):
        res += str(stack[k])
        if k < len(stack) - 1:
            if stack[k + 1] != ']' and stack[k] != '[':
                res += ','
    return res

def reduce(s):
    stackp = []
    stack = []
    for k in range(len(s)):
        if s[k].isdigit():
            if s[k + 1].isdigit():
                stackp.append(int(s[k] + s[k + 1]))
            elif not s[k - 1].isdigit():
                stackp.append(int(s[k]))
        elif s[k] != ',':
            stackp.append(s[k])
    stackp.reverse()
    p = 0
    r = False
    i = -1
    while stackp:
        a = stackp.pop()
        i += 1
        if a == '[':
            p += 1
            stack.append(a)
        elif a == ']':
            if p <= 4:
                p -= 1
                stack.append(a)
            else:
                stackp.reverse()
                y = stack.pop()
                x = stack.pop()
                stack.pop()
                for k in range(i - 4, -1, -1):
                    if type(stack[k]) == int:
                        stack[k] += x
                        r = True
                        break
                for k in range(len(stackp)):
                    if type(stackp[k]) == int:
                        stackp[k] += y
                        r = True
                        break
                stack = stack + [0] + stackp
                break
        else:
            stack.append(a)
    if not r:
        for k in range(len(stack)):
            if type(stack[k]) == int:
                if stack[k] >= 10:
                    splited = split(stack[k])
                    stack[k] = '['
                    stack.insert(k+1,splited[0])
                    stack.insert(k+2,splited[1])
                    stack.insert(k+3,']')
                    r = True
                    break
    s = stack_to_str(stack)
    if r:
        return reduce(s)
    else:
        return s

# PART 1
# res = numbers[0]
# for number in numbers[1:]:
#     res = reduce(add(res,number))
#
# print(magnitude(str_to_snb(res)))

# PART 2
# m = 0
# for n1 in numbers:
#     for n2 in numbers:
#         if n1 != n2:
#             m = max(magnitude(str_to_snb(reduce(add(n1,n2)))),magnitude(str_to_snb(reduce(add(n2,n1)))),m)
# print(m)


from flask import Flask
from flask_sqlalchemy import SQLAlchemy
import json
from configs import app

db = SQLAlchemy(app)

class Recipe(db.Model):
    __tablename__ = 'recipes'
    recipe_id = db.Column(db.String, primary_key=True)
    recipe_name = db.Column(db.String, nullable=False)
    recipe_type = db.Column(db.String, nullable=False)
    ingredients = db.Column(db.String, nullable=False)
    instructions = db.Column(db.String, nullable=False)

    def json(self):
        return{
        'recipe_id': self.recipe_id,
        'recipe_name': self.recipe_name,
        'recipe_type': self.recipe_type,
        'ingredients': self.ingredients,
        'instructions': self.instructions
        }

    def get_all_recipes():
        return [Recipe.json(recipes) for recipes in Recipe.query.all()]

    def delete_recipe(_recipe_id):
        is_successful = Recipe.query.filter_by(recipe_id=_recipe_id).delete()
        db.session.commit()
        return bool(is_successful)

    def update_recipe_instructions(_recipe_id, _instructions):
        recipe_to_update = Recipe.query.filter_by(recipe_id=_recipe_id).first()
        recipe_to_update.instructions = _instructions
        db.session.commit()

    def update_recipe_ingredients(_recipe_id, _ingredients):
        recipe_to_update = Recipe.query.filter_by(recipe_id=_recipe_id).first()
        recipe_to_update.ingredients = _ingredients
        db.session.commit()

    def update_recipe_name(_recipe_id, _recipe_name):
        recipe_to_update = Recipe.query.filter_by(recipe_id=_recipe_id).first()
        recipe_to_update.recipe_name = _recipe_name
        db.session.commit()

    def add_recipe(_recipe_id, _recipe_name, _recipe_type, _ingredients, _instructions):
        new_recipe = Recipe(

            recipe_id=_recipe_id,
            recipe_name=_recipe_name,
            recipe_type=_recipe_type,
            ingredients=_ingredients,
            instructions=_instructions
            )

        db.session.add(new_recipe)
        db.session.commit()

    def __repr__(self):
        recipe_object = {
        'recipe_id': self.recipe_id,
        'recipe_name': self.recipe_name,
        'recipe_type': self.recipe_type,
        'ingredients': self.ingredients,
        'instructions': self.instructions
        }

        return json.dumps(recipe_object)


import re

def cellline2lucence_query(cellline):
    clean_cellline = cellline.split("/")[0].lower()
    elements = re.findall("[a-z]+|\d+", clean_cellline)

    full_name = "".join(elements)
    all_options = " AND ".join(elements)
    simple_options = clean_cellline.replace("-", " AND ")
    query = f"{full_name} OR ({all_options}) OR ({simple_options})"

    return query


# -*- coding: utf-8 -*-
"""Helper functions to load and save CSV data.

This contains a helper function for loading and saving CSV files.

"""
import csv

def load_csv(csvpath):
    """Reads the CSV file from path provided.

    Args:
        csvpath (Path): The csv file path.

    Returns:
        A list of lists that contains the rows of data from the CSV file.

    """
    with open(csvpath, "r") as csvfile:
        data = []
        csvreader = csv.reader(csvfile, delimiter=",")

        # Skip the CSV Header
        next(csvreader)

        # Read the CSV data
        for row in csvreader:
            data.append(row)
    return data

def save_csv(csvpath,qualifying_loans,header=None):
    """Writes the CSV file to the one named qualifying_loans.

    Args:
        csvpath (Path): The csv file path.

    Prints:
        Confirmation that the date has been saved a CSV file with all the qualifying loans informatin.

    """
    with open(csvpath, 'w', newline='') as csvfile:
        csvwriter = csv.writer(csvfile, delimiter=',')
        if header:

            # Write the header row first
            csvwriter.writerow(header)

        # Write data rows
        csvwriter.writerows(qualifying_loans)

        # If user wants to save CSV file, print the following statement after CSV file is saved.
        print("Thank you for submitting your information. The data has been saved.")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
@date: 2018-Today
@author: jasper.bathmann@ufz.de
"""
from lxml import etree
import importlib.util

class Plant:

    def __init__(self,
                 x,
                 y,
                 species,
                 plant_id,
                 initial_geometry=False,
                 group_name=""):
        self.plant_id = plant_id
        self.species = species
        self.plants = []
        self.x = x
        self.y = y
        self.survival = 1
        self.group_name = group_name
        ## This initialization is only required if networks (root grafts) are
        # simulated
        self.iniNetwork()
        if species == "Avicennia":
            from PopulationLib.Species import Avicennia
            self.geometry, self.parameter = Avicennia.createPlant()
        elif "/" in species:
            try:
                spec = importlib.util.spec_from_file_location("", species)
                foo = importlib.util.module_from_spec(spec)
                spec.loader.exec_module(foo)
                self.geometry, self.parameter = foo.createPlant()
            except FileNotFoundError:
                raise FileNotFoundError("The file " + species +
                                        " does not exist.")
            except AttributeError:
                raise AttributeError("The file " + species + " is not " +
                                     "correctly defining a plant species. "
                                     "Please review the file.")
        else:
            raise KeyError("Species " + species + " unknown!")
        if initial_geometry:
            self.geometry["r_crown"] = initial_geometry["r_crown"]
            self.geometry["r_root"] = initial_geometry["r_root"]
            self.geometry["r_stem"] = initial_geometry["r_stem"]
            self.geometry["h_stem"] = initial_geometry["h_stem"]
        self.growth_concept_information = {}

    def getPosition(self):
        return self.x, self.y

    def getGeometry(self):
        return self.geometry

    def setGeometry(self, geometry):
        self.geometry = geometry

    def getGrowthConceptInformation(self):
        return self.growth_concept_information

    def setGrowthConceptInformation(self, growth_concept_information):
        self.growth_concept_information = growth_concept_information

    def getParameter(self):
        return self.parameter

    def getSurvival(self):
        return self.survival

    def setSurvival(self, survival):
        self.survival = survival

    def getId(self):
        return self.plant_id

    ## This function initializes a dictionary containing parameters required
    # to build a network of grafted plants
    def iniNetwork(self):
        self.network = {}
        ## Counter to track or define the time required for root graft
        # formation, if -1 no root graft formation takes place at the moment
        self.network['rgf'] = -1
        ## List with the names of plants (plant_name) with which an root graft
        # is currently being formed
        self.network['potential_partner'] = []
        # List with the names of plants (plant_name) with which it is connected
        self.network['partner'] = []
        self.network['groupID'] = []
        self.network['node_degree'] = 0
        self.network['water_absorbed'] = []
        self.network['water_available'] = []
        self.network['water_exchanged'] = []
        ## List with lengths of grafted roots (proportional to r_root of
        # adjacent plants
        self.network['weight_gr'] = 0
        self.network['psi_osmo'] = []
        # List with minimum grafted root radius, only for rgf variant V2
        self.network['r_gr_min'] = []
        self.network['r_gr_rgf'] = []
        self.network['l_gr_rgf'] = []
        self.network['variant'] = None

    def getNetwork(self):
        return self.network

    def setNetwork(self, network):
        self.network = network


from db.run_sql import run_sql

from models.album import Album
from models.artist import Artist

from repositories import artist_repository

def save(album):
    sql = "INSERT INTO album (name, genre, artist_id) VALUES (%s, %s, %s) RETURNING *"
    values = [album.name, album.genre, album.artist.id]
    results = run_sql(sql, values)
    id = results[0]['id']
    album.id = id
    return album

def delete_all():
    sql = "DELETE FROM album"
    run_sql(sql)

def find(id):
    album = None
    sql = "SELECT * from album WHERE id = %s"
    values = [id]
    results = run_sql(sql, values)

    if results:
        result = results[0]
        artist = artist_repository.find(result["artist_id"])
        album = Album(result['name'], result['genre'], artist, result['id'])
    return album

def select_all():
    albums = []
    sql = "SELECT * FROM album"
    results = run_sql(sql)

    for row in results:
        artist = artist_repository.find(row['artist_id'])
        album = Album(row['name'], row['genre'], artist, row['id'])
        albums.append(album)
    return albums

def delete(id):
    sql = 'DELETE FROM album WHERE id = %s'
    values = [id]
    run_sql(sql, values)