<file_sep>#%%
import numpy as np
import Utility as util
import pandas as pd
import matplotlib.pyplot as plt
import os
import sqlite3 as sql
import ast
from scipy.linalg import svd

# %%
data,data_movies = get_user_movie_rating()
cold_users = data_movies[:1000]
warm_users = data_movies[1000:]
cold_users[1]
R = cold_users
user_ratings_mean = np.mean(R, axis = 1)
R_demeaned = R - user_ratings_mean.reshape(-1, 1)
from scipy.sparse.linalg import svds
U,sigma,_ = svds(R_demeaned, k = 50)
P = np.dot(U,np.diag(sigma))


R = warm_users
user_ratings_mean = np.mean(R, axis = 1)
R_demeaned = R - user_ratings_mean.reshape(-1, 1)
_, _, Q = svds(R_demeaned, k = 50)

Q=Q.T

lam,alpha = 1,0.1 #change it please
def g_elo_actual(a,b):
    if(a>b):
        return 1
    if(a==b):
        return 0.5
    else:
        return 0
def g_logistic(a,b):
    return 1/(1+np.exp(-(a-b)))
def g_linear(a,b):
    return (a-b)

def update(p,i,j,r,f):
    count_r =  np.count_nonzero(warm_users[:][j]==r)
    rui = cold_users[i][j]
    P_ = np.reshape(P[i][:],(-1,1)).T
    Q_ = np.reshape(Q[:][j],(-1,1))
    # print (P_.shape,Q_.shape)
    rcap = np.dot(P_,Q_)
    grad = 2 * Q[i][f] * count_r * (g_linear(rui,r) - g_linear(rcap,r)) + (2 * lam * P[i][f])
    return (p - grad)

def func(cold_users,P,Q,r_max = 5,k=50):
    for temp in range(2):
        for i in range(len(cold_users)):
            for j in range(len(cold_users[0])):
                if (cold_users[i][j] != 0):
                    for r in range(r_max):
                        for f in range(k):
                            P[i][f] = update(P[i][f],i,j,r,f)
    return P

P = func(cold_users,P,Q)
<file_sep>#%%
# Code for analysing and preprocessing of data
#%%
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import os
import sqlite3 as sql
import ast
from scipy.linalg import svd

#%%
def get_data(f_name):
    movies = pd.read_csv(f_name+'movies.csv')
    users = pd.read_csv(f_name+'users.csv')
    ratings = pd.read_csv(f_name+'ratings.csv')

    return movies,users,ratings

def create_movies_table(df):
    db = sql.connect(os.path.abspath('../Data/Movie.db'))
    cur = db.cursor()
    cur.execute("SELECT count(*) from sqlite_master where type = 'table' and name = 'movies_data';")
    flag = cur.fetchall()[0][0]

    if flag == 0 :
        cur.execute("CREATE TABLE if not exists movies_data (M_id INT PRIMARY KEY, Title TEXT, Genres TEXT);")
        df = df[['movie_id','title','genres']]

        for id,rows in df.iterrows():
            cur.execute("INSERT INTO movies_data(M_id, Title, Genres) values(?,?,?);",rows)
        db.commit()
        print ("movies_data table was created with entries")

    else :
        print ("movies_data table already exists")
    db.close()

def create_users_table(df):
    db = sql.connect(os.path.abspath('../Data/Movie.db'))
    cur = db.cursor()
    cur.execute("SELECT count(*) from sqlite_master where type = 'table' and name = 'users_data';")
    flag = cur.fetchall()[0][0]

    if flag == 0 :
        cur.execute("CREATE TABLE if not exists users_data (U_id INT PRIMARY KEY, Gender TEXT, Age INT);")
        df = df[['user_id','gender','age']]

        for id,rows in df.iterrows():
            cur.execute("INSERT INTO users_data(U_id, Gender, Age) values(?,?,?);",rows)
        db.commit()
        print ("users_data table was created with entries")

    else :
        print("users_data table already exists")
    db.close()

def create_ratings_table(df):
    db = sql.connect(os.path.abspath('../Data/Movie.db'))
    cur = db.cursor()
    cur.execute("SELECT count(*) from sqlite_master where type = 'table' and name = 'ratings_data';")
    flag = cur.fetchall()[0][0]

    if flag == 0 :
        cur.execute("CREATE TABLE if not exists ratings_data (U_id INT, M_id INT, rating INT);")
        df = df[['user_id','movie_id','rating']]

        for id,rows in df.iterrows():
            cur.execute("INSERT INTO ratings_data(U_id, M_id, rating) values(?,?,?);",rows)
        db.commit()
        print ("ratings_data table was created with entries")

    else :
        print ("ratings_data table already exists")
    db.close()

def Drop_table(t_name):
    db = sql.connect(os.path.abspath('../Data/Movie.db'))
    cur = db.cursor()
    cur.execute("DROP TABLE if exists "+t_name +";")
    db.close()


def analysis_user():
    db = sql.connect(os.path.abspath('../Data/Movie.db'))
    cur = db.cursor()
    cur.execute("Select cnt,count(*) as num_users from (select U_id,count(*) as cnt from ratings_data group by U_id order by cnt ) group by cnt")
    data1 = cur.fetchall()
    db.close()
    return np.asarray(data1)

def analysis_rating():
    db = sql.connect(os.path.abspath('../Data/Movie.db'))
    cur = db.cursor()
    cur.execute("select rating from ratings_data")
    data2 = cur.fetchall()
    db.close()
    return np.asarray(data2)


def get_user_movie_rating():
    db = sql.connect(os.path.abspath('../Data/Movie.db'))
    cur = db.cursor()
    cur.execute("select U_id,'{'||group_concat(M_id|| ':' || rating , ',')||'}' as movie_list from ratings_data group by U_id limit 6040;")
    data = cur.fetchall()
    db.close()
    dict = {int(data[i][0]):ast.literal_eval(data[i][1]) for i in range (len(data))}

    data_movies = np.full((6041,3953),0)
    for keys in dict:
        data_movies[keys][0] = len(dict[keys])
        for k in dict[keys]:
            data_movies[keys][k] = dict[keys][k]

    data_movies = sorted(data_movies, key= lambda entry : entry[0])
    return dict,data_movies

#%%
# Drop_table('ratings_data')
# Drop_table('movies_data')
# Drop_table('users_data')
#%%

# movies,users,ratings = get_data('../Data/')

#%%
# print(movies.head(10))
# print(movies.shape)
# print(users.head(10))
# print(users.shape)
# print(ratings.head(10))
# print(ratings.shape)
#
#%%
# create_movies_table(movies)
# create_users_table(users)
# create_ratings_table(ratings)
#%%
<file_sep>#%%
# Code for analysing and preprocessing of data
#%%
import numpy as np
import pandas as pd
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt
import os
import sqlite3 as sql
import ast
from scipy.linalg import svd
#%%
def get_data(f_name):
    movies = pd.read_csv(f_name+'movies.csv')
    users = pd.read_csv(f_name+'users.csv')
    ratings = pd.read_csv(f_name+'ratings.csv')

    return movies,users,ratings

def create_movies_table(df):
    db = sql.connect(os.path.abspath('../Data/Movie.db'))
    cur = db.cursor()
    cur.execute("SELECT count(*) from sqlite_master where type = 'table' and name = 'movies_data';")
    flag = cur.fetchall()[0][0]

    if flag == 0 :
        cur.execute("CREATE TABLE if not exists movies_data (M_id INT PRIMARY KEY, Title TEXT, Genres TEXT);")
        df = df[['movie_id','title','genres']]

        for id,rows in df.iterrows():
            cur.execute("INSERT INTO movies_data(M_id, Title, Genres) values(?,?,?);",rows)
        db.commit()
        print ("movies_data table was created with entries")

    else :
        print ("movies_data table already exists")
    db.close()

def create_users_table(df):
    db = sql.connect(os.path.abspath('../Data/Movie.db'))
    cur = db.cursor()
    cur.execute("SELECT count(*) from sqlite_master where type = 'table' and name = 'users_data';")
    flag = cur.fetchall()[0][0]

    if flag == 0 :
        cur.execute("CREATE TABLE if not exists users_data (U_id INT PRIMARY KEY, Gender TEXT, Age INT);")
        df = df[['user_id','gender','age']]

        for id,rows in df.iterrows():
            cur.execute("INSERT INTO users_data(U_id, Gender, Age) values(?,?,?);",rows)
        db.commit()
        print ("users_data table was created with entries")

    else :
        print ("users_data table already exists")
    db.close()

def create_ratings_table(df):
    db = sql.connect(os.path.abspath('../Data/Movie.db'))
    cur = db.cursor()
    cur.execute("SELECT count(*) from sqlite_master where type = 'table' and name = 'ratings_data';")
    flag = cur.fetchall()[0][0]

    if flag == 0 :
        cur.execute("CREATE TABLE if not exists ratings_data (U_id INT, M_id INT, rating INT);")
        df = df[['user_id','movie_id','rating']]

        for id,rows in df.iterrows():
            cur.execute("INSERT INTO ratings_data(U_id, M_id, rating) values(?,?,?);",rows)
        db.commit()
        print ("ratings_data table was created with entries")

    else :
        print ("ratings_data table already exists")
    db.close()

def Drop_table(t_name):
    db = sql.connect(os.path.abspath('../Data/Movie.db'))
    cur = db.cursor()
    cur.execute("DROP TABLE if exists "+t_name +";")
    db.close()
#%%

movies,users,ratings = get_data('../Data/')

#%%
print(movies.head(10))
print(movies.shape)
print(users.head(10))
print(users.shape)
print(ratings.head(10))
print(ratings.shape)

#%%
create_movies_table(movies)
create_users_table(users)
create_ratings_table(ratings)
#%%

def analysis_user():
    db = sql.connect(os.path.abspath('../Data/Movie.db'))
    cur = db.cursor()
    cur.execute("Select cnt,count(*) as num_users from (select U_id,count(*) as cnt from ratings_data group by U_id order by cnt ) group by cnt")
    data1 = cur.fetchall()
    db.close()
    return np.asarray(data1)

def analysis_rating():
    db = sql.connect(os.path.abspath('../Data/Movie.db'))
    cur = db.cursor()
    cur.execute("select rating from ratings_data")
    data2 = cur.fetchall()
    db.close()
    return np.asarray(data2)


def get_user_movie_rating():
    db = sql.connect(os.path.abspath('../Data/Movie.db'))
    cur = db.cursor()
    cur.execute("select U_id,'{'||group_concat(M_id|| ':' || rating , ',')||'}' as movie_list from ratings_data group by U_id limit 6040;")
    data = cur.fetchall()
    db.close()
    dict = {int(data[i][0]):ast.literal_eval(data[i][1]) for i in range (len(data))}

    data_movies = np.full((6041,3953),0)
    for keys in dict:
        data_movies[keys][0] = len(dict[keys])
        for k in dict[keys]:
            data_movies[keys][k] = dict[keys][k]

    data_movies = sorted(data_movies, key= lambda entry : entry[0])
    return dict,data_movies

#%%
data,data_movies = get_user_movie_rating()
cold_users = data_movies[:1000]
warm_users = data_movies[1000:]
# cold_users[1]
R = np.asarray(cold_users,dtype='float64')
cold_users = R[:,1:]
R = R[:,1:]
# user_ratings_mean = np.mean(R, axis = 1)
# R_demeaned = R - user_ratings_mean.reshape(-1, 1)
from scipy.sparse.linalg import svds
U,sigma,_ = svds(R, k = 50)
P = np.dot(U,np.diag(sigma))


R = np.asarray(warm_users,dtype='float64')
warm_users = R[:,1:]
R = R[:,1:]
#user_ratings_mean = np.mean(R, axis = 1)
#R_demeaned = R - user_ratings_mean.reshape(-1, 1)
_, _, Q = svds(R, k = 50)

Q=Q.T

lam,alpha = 1,0.01 #change it please

def g_elo_actual(a,b):
    if(a>b):
        return 1
    if(a==b):
        return 0.5
    else:
        return 0
def g_logistic(a,b):
    return 1/(1+np.exp(-(a-b)))
def g_linear(a,b):
    return (a-b)

def update(p,i,j,r,f , count_r):
    rui = cold_users[i][j]
    P_ = np.reshape(P[i][:],(-1,1)).T
    Q_ = np.reshape(Q[:][j],(-1,1))
    # print (P_.shape,Q_.shape)
    rcap = np.dot(P_,Q_)
    grad = 2 * Q[i][f] * count_r * (g_linear(rui,r) - g_linear(rcap,r)) + (2 * lam * P[i][f])
    return (p - alpha * grad)

def func(cold_users,P,Q,r_max = 5,k=50):
    count_r = np.empty(( r_max , len(cold_users[0])))
    for r in range(r_max):
        for j in range(len(cold_users)):
            count_r[r][j] = np.count_nonzero(warm_users[:][j]==r)
    for temp in range(2):
        print(temp)
        for i in range(len(cold_users)):
            for j in range(len(cold_users[0])):
                if (cold_users[i][j] != 0):
                    for r in range(r_max):
                        for f in range(k):
                            P[i][f] = update(P[i][f],i,j,r,f , count_r[r][j])
    return P
def SVD():
    R = np.asarray(data_movies, dtype='float64')
    R = R[:, 1:]
    user_ratings_mean = np.mean(R, axis = 1)
    R_demeaned = R - user_ratings_mean.reshape(-1, 1)
    U, sigma, Vt = svds(R_demeaned, k=50)
    sigma_d = np.diag(sigma)
    predicted = np.dot(np.dot(U, sigma_d), Vt) + user_ratings_mean.reshape(-1, 1)
    return predicted

def MF():
    R = np.asarray(data_movies, dtype='float64')
    R = R[:, 1:]
    MF_K = 10
    MF_P = np.random.normal(scale=1. / MF_K, size=(np.size(R , 0), MF_K))
    MF_Q = np.random.normal(scale=1. / MF_K, size=(np.size(R , 1), MF_K))
    b_u = np.zeros(np.size(R , 0))
    b_i = np.zeros(np.size(R , 1))
    b = np.mean(R[np.where(R != 0)])
    samples = [
        (i, j, R[i, j])
        for i in range(np.size(R , 0))
        for j in range(np.size(R , 1))
        if R[i, j] > 0
    ]
    iterations = 10
    for i in range(iterations):
        np.random.shuffle(samples)
        for i, j, r in samples:
            # Computer prediction and error
            prediction = b + b_u[i] + b_i[j] + MF_P[i, :].dot(MF_Q[j, :].T)
            e = (r - prediction)
            beta = 0.1
            # Update biases
            b_u[i] += alpha * (e - beta * b_u[i])
            b_i[j] += alpha * (e - beta * b_i[j])

            # Update user and item latent feature matrices
            MF_P[i, :] += alpha * (e * MF_Q[j, :] - beta * MF_P[i, :])
            MF_Q[j, :] += alpha * (e * MF_P[i, :] - beta * MF_Q[j, :])
        xs, ys = R.nonzero()
        predicted = b + b_u[:, np.newaxis] + b_i[np.newaxis:, ] + MF_P.dot(MF_Q.T)
        error = 0
        cnt = 0
        for x, y in zip(xs, ys):
            cnt = cnt + 1
            error += pow(R[x, y] - predicted[x, y], 2)
        print(np.sqrt(error / cnt))
    xs, ys = R.nonzero()
    predicted = b + b_u[:, np.newaxis] + b_i[np.newaxis:, ] + MF_P.dot(MF_Q.T)
    error = 0
    cnt = 0
    for x, y in zip(xs, ys):
        cnt = cnt + 1
        error += pow(R[x, y] - predicted[x, y], 2)
    print(np.sqrt(error / cnt))
    return b + b_u[:, np.newaxis] + b_i[np.newaxis:, ] + MF_P.dot(MF_Q.T)

#P = func(cold_users,P,Q)
predicted= SVD()
R = np.asarray(data_movies, dtype='float64')
R = R[:, 1:]
xs, ys = R.nonzero()
error = 0
cnt = 0
for x, y in zip(xs, ys):
    cnt = cnt + 1
    error += pow(R[x, y] - predicted[x, y], 2)
print(np.sqrt(error / cnt))
#print(P.shape)
print(P2.shape)
print(P)
#print(P2)
<file_sep>#%%
import numpy as np
import Utility as util
import pandas as pd
import matplotlib.pyplot as plt
import os
import sqlite3 as sql
import ast
from scipy.linalg import svd
from scipy.sparse.linalg import svds

# %% Helper-Functions
def g_elo_actual(a,b):
    if(a>b):
        return 1
    if(a==b):
        return 0.5
    else:
        return 0
def g_logistic(a,b):
    return 1/(1+np.exp(-(a-b)))

def g_linear(a,b):
    return (a-b)

def get_P(user_data,k):
    U,sigma,_ = svds(user_data, k=k)
    #print(sigma)
    P = np.dot(U,np.diag(sigma))
    return np.asarray(P)

def get_Q(user_data,k):
    _, _, Q = svds(user_data,k=k)
    Q=Q.T
    return np.asarray(Q)

def update(p,Q,u,i,r,k,rw,rc,alpha,lam):
    omega = np.sum(rw[:,i]==r)
    rui = rc[u][i]
    P_ = np.reshape(p,(-1,1)).T
    Q_ = np.reshape(Q[i,:],(-1,1))
    rcap = np.dot(P_,Q_)
    gact = g_elo_actual(rui,r)
    gexp = g_logistic(rcap,r)
    grad = omega * Q[i,:]*(gact-gexp) + lam * (P_)
    P_  += alpha * grad
    return np.reshape(P_.T,(k))


def rapare(P,Q,rc,rw,rmax,epochs,k,alpha,lam,res_mse,lam_step,lam_change,alpha_step,alpha_change):
    old_mse = 0
    for epc in range(epochs):
        if(epc%lam_step==0):
            lam+=lam_change
        if(epc%alpha_step==0):
            alpha -= alpha_change*alpha
        for u in range(rc.shape[0]):
            for i in range(rc.shape[1]):
                if(rc[u][i]!= 0):
                    for r in range (1,rmax+1):
                        P[u] = update(P[u],Q,u,i,r,k,rw,rc,alpha,lam)
        r_est = np.dot(P,Q.T)
        r_est[r_est<=0]=0
        r_est[r_est>=5]=5
        r_est[rc==0]=0
        mse = np.sqrt(np.sum((r_est-rc)**2)/np.sum(rc>0))
        print('Epoch:',epc,' RMSE:' , mse, ' change: ', old_mse-mse)
        old_mse = mse
        res_mse.append(mse)
    save_plot(res_mse,'training_varAlph.png')
    return P,res_mse

def save_plot(mse,f_name):
    path='./'+f_name
    x = np.arange(0,len(mse),1)
    fig = plt.figure()
    plt.plot(x,mse)
    plt.plot()
    fig.savefig(path,dpi=fig.dpi)

#%% Hyperparams
epochs = 200
lam = 0.05
lam_step = 20
lam_change = 0.001
k = 100 # number of components
alpha = 0.05
alpha_step = 25
alpha_change = 0.1
rmax = 5


#%% get_data
data,data_movies = util.get_user_movie_rating()
r  = np.asarray(data_movies,dtype=np.float32)[:,1:]
rc = np.asarray(data_movies[:1000],dtype=np.float32)[:,1:]
rw = np.asarray(data_movies[1000:],dtype=np.float32)[:,1:]

#%% computing Latent factors

# P = np.random.rand(rc.shape[0],k)
#P = np.load('500epc_p.npy')
P = get_P(rc,k)
Q = get_Q(r,k)

#%%
Rmse = []
P,Rmse = rapare(P,Q,rc,rw,rmax,epochs,k,alpha,lam,Rmse,lam_step,lam_change,alpha_step,alpha_change)

#np.save('try_1.38539_P.npy',P)
<file_sep>#%%
import numpy as np
import Utility as util
import pandas as pd
import matplotlib.pyplot as plt
import os
import sqlite3 as sql
import ast
from scipy.linalg import svd
from scipy.sparse.linalg import svds

# %% Helper-Functions
def g_elo_actual(a,b):
    if(a>b):
        return 1
    if(a==b):
        return 0.5
    else:
        return 0

def g_logistic(a,b):
    return 1/(1+np.exp(-(a-b)))

def g_linear(a,b):
    return (a-b)

def get_P(user_data,k):
    U,sigma,_ = svds(user_data, k=k)
    P = np.dot(U,np.diag(sigma))
    return np.asarray(P)

def get_Q(user_data,k):
    _, _, Q = svds(user_data,k=k)
    Q=Q.T
    return np.asarray(Q)

def update(p,Q,u,i,r,rw,rc,alpha,lam):
    omega = np.sum(rw[:,i]==r)
    rui = rc[u][i]

    P_ = np.reshape(p,(-1,1)).T
    Q_ = np.reshape(Q[i,:],(-1,1))
    rcap = np.dot(P_,Q_)
    gact = g_elo_actual(rui,r)
    gexp = g_logistic(rcap,r)
    grad = omega*Q[i,:]*(gact-gexp) + lam*(P_)
    P_ -= alpha*grad
    return np.reshape(P_.T,(50))

def rapare(P,Q,rc,rw,rmax,epochs,k,alpha,lam):
    res_mse = []
    for epc in range(epochs):
        for u in range(rc.shape[0]):
            for i in range(rc.shape[1]):
                if(rc[u][i]!= 0):
                    for r in range (1,rmax+1):
                        P[u] = update(P[u],Q,u,i,r,rw,rc,alpha,lam)
        print(epc)
        r_est = np.dot(P,Q.T)
        r_est[r_est<=0]=0
        r_est[r_est>=5]=5
        r_est[rc==0]=0
        # r_est = np.round(r_est)
        mse = np.sqrt(np.sum((r_est-rc)**2)/np.sum(rc>0))
        print('RMSE:' , mse )
        res_mse.append(mse)
    save_plot(res_mse,epochs,'training.png')
    return P,res_mse

def save_plot(mse,epochs,f_name):
    path='./'+f_name
    x = np.arange(0,epochs,1)
    fig = plt.figure()
    plt.plot(x,mse)
    plt.xlabel('Iteration')
    plt.ylabel('RMSE')
    plt.title('Cold Start User scenario on MovieLens')
    plt.plot()
    fig.savefig(path,dpi=fig.dpi)

#%% Hyperparams
epochs = 10
lam = 1
k = 50 # number of components
alpha = 0.1
rmax = 5


#%% get_data
data,data_movies = util.get_user_movie_rating()
r  = np.asarray(data_movies,dtype=np.float32)[:,1:]
rc = np.asarray(data_movies[:1000],dtype=np.float32)[:,1:]
rw = np.asarray(data_movies[1000:],dtype=np.float32)[:,1:]

#%% computing Latent factors
# P = np.random.rand(rc.shape[0],k)
P = np.load('500epc_p.npy')

# P = get_P(rc,k)
Q = get_Q(r,k)
#%%
# P,Rmse = rapare(P,Q,rc,rw,rmax,epochs,k,alpha,lam)
# np.save('500epc_p.npy',P)
r_est = np.dot(P,Q.T)
r_est[r_est<=0]=0
r_est[r_est>=5]=5
r_est = np.round(r_est)
r_est[rc==0]=-1
np.sum(r_est==rc)
# np.sum(rc>-1)
mse = np.sqrt(np.sum((r_est-rc)**2)/np.sum(rc>0))
print('RMSE:' , mse )
# REVIEW:
epoch = 500
var = 1800
r_est[rc==0]=-1
x,y = np.asarray(np.where(rc==r_est))
inx = np.random.permutation(len(x))
x = x[inx]
y = y[inx]

mse=[]
for epc in range(epoch):
    x1,y1 = np.asarray(np.where(rc>0))
    inx1 = np.random.permutation(len(x1))
    x1 = x1[inx1[:var-epc*2]]
    y1 = y1[inx1[:var-epc*2]]
    rc_tst = []
    rc_est_tst = []
    x_t = np.concatenate((x,x1),axis = -1)
    y_t = np.concatenate((y,y1),axis = -1)
    for i in range(len(x_t)):
        rc_tst.append(rc[x_t[i]][y_t[i]])
        rc_est_tst.append(r_est[x_t[i]][y_t[i]])
    rc_tst = np.array(rc_tst)
    rc_est_tst = np.array(rc_est_tst)
    print(len(x_t))
    mse_tst = np.sqrt(np.sum((rc_est_tst-rc_tst)**2)/len(x))
    mse.append(mse_tst)
    print('RMSE:' , mse_tst )
save_plot(mse,epoch,'result.png')
<file_sep>import math
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# Reading ratings file
ratings = pd.read_csv('ratings.csv', sep='\t', encoding='latin-1', usecols=['user_id', 'movie_id', 'user_emb_id', 'movie_emb_id', 'rating'])
max_userid = ratings['user_id'].drop_duplicates().max()
max_movieid = ratings['movie_id'].drop_duplicates().max()

# Reading ratings file
users = pd.read_csv('users.csv', sep='\t', encoding='latin-1',usecols=['user_id', 'gender', 'zipcode', 'age_desc', 'occ_desc'])

# Reading ratings file
movies = pd.read_csv('movies.csv', sep='\t', encoding='latin-1', usecols=['movie_id', 'title', 'genres'])

print max_movieid
print max_userid

shuffled_ratings = ratings.sample(frac=1., random_state=0)

# Shuffling users
Users = shuffled_ratings['user_emb_id'].values
print 'Users:', Users, ', shape =', Users.shape

# Shuffling movies
Movies = shuffled_ratings['movie_emb_id'].values
print 'Movies:', Movies, ', shape =', Movies.shape

# Shuffling ratings
Ratings = shuffled_ratings['rating'].values
print 'Ratings:', Ratings, ', shape =', Ratings.shape


K_FACTORS = 100 # The number of dimensional embeddings for movies and users
TEST_USER = 2000 # A random test user (user_id = 2000)
from keras.callbacks import Callback, EarlyStopping, ModelCheckpoint
from keras.models import Model


inp1 = Input((1,))
x1 = Embedding(6040,100,input_length=1)(inp1)
x1 = Reshape((100,))(x1)
x1 = Reshape((100,))(x1)
inp2 = Input((1,))
x2 = Embedding(3952,100,input_length=1)(inp2)
x2 = Reshape((100,))(x2)
x3 = dot([x1,x2],axes=1)
model = Model([inp1,inp2],x3)
model.compile(loss='mse',optimizer='adamax')

callbacks = [EarlyStopping('val_loss', patience=2), 
             ModelCheckpoint('weights.h5', save_best_only=True)]

# Use 30 epochs, 90% training data, 10% validation data 
history = model.fit([Users, Movies], Ratings, nb_epoch=30, validation_split=.1, verbose=1, callbacks=callbacks)<file_sep>#%%
import numpy as np
import Utility as util
import pandas as pd
import matplotlib.pyplot as plt
import os
import sqlite3 as sql
import ast
from scipy.linalg import svd
from scipy.sparse.linalg import svds
from MF import ProductRecommender

# %% Helper-Functions
def save_plot(mse,epochs,f_name):
    path='./'+f_name
    x = np.arange(0,epochs,1)
    fig = plt.figure()
    plt.plot(x,mse)
    plt.plot()
    fig.savefig(path,dpi=fig.dpi)

#%% Hyperparams
epochs = 100
lam = 0
k = 20 # number of components
alpha = 0.1
rmax = 5


#%% get_data
data,data_movies = util.get_user_movie_rating()
r  = np.asarray(data_movies,dtype=np.float32)[:,1:]
rc_train = np.asarray(data_movies[:900],dtype=np.float32)[:,1:]
rc_test = np.asarray(data_movies[900:1000],dtype=np.float32)[:,1:]
rw = np.asarray(data_movies[1000:],dtype=np.float32)[:,1:]

#%%
model = ProductRecommender()
model.fit(rc_train)
#%% computing Latent factors

#%%
P,Rmse = rapare(P,Q,rc,rw,rmax,epochs,k,alpha,lam)
# np.save('600epc_p.npy',P)
# r_est = np.dot(P,Q.T)
# r_est[r_est<=0]=0
# r_est[r_est>=5]=5
# r_est = np.round(r_est)
# r_est[rc==0]=0
# np.sum(r_est==rc)
# np.sum(rc>-1)
# mse = np.sqrt(np.sum((r_est-rc)**2)/np.sum(rc>0))
# print('RMSE:' , mse )
# REVIEW:
# x,y = np.asarray(np.where(rc>0))
# x.shape
# y.shape
