Tensorflow regression predicting 1 for all inputs
I'm trying to predict a UPDRS score (regression) from the <PERSON> dataset. I wanted to start with a fairly basic structure and the add in more layers/techniques to improve performance but no matter what I do the network constantly predicts 1 for every input.
I've been banging my head against this problem for a couple of days now with no success. Originally I was using RELU and thought it was a case of dead neurons but using leaky RELU and monitoring the outputs I can see them firing.
Changing the network structure or training steps always results with the same horrible MSE of 896.39105. This leads me to believe the mistake is somewhere in the normalization but when I check the intermediate values they all seem fine.
Can anyone tell me where I've messed up?
import tensorflow as tf
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import normalize
from sklearn.preprocessing import StandardScaler
def leaky_relu(x, alpha):
return tf.nn.relu(x) - alpha * tf.nn.relu(-x)
raw = pd.read_csv('parkinsons_updrs.data')
print("CSV loaded successfully")
#Split into features and targets
features = raw.drop(['subject#','test_time','motor_UPDRS','total_UPDRS'], axis = 1).as_matrix()
targets = raw['total_UPDRS'].get_values()
scale = StandardScaler(with_mean=0, with_std=1)
scale.fit(features, targets)
scaled_feats = scale.transform(features)
print("pre-processing complete")
#Split data into test/train
data_train, data_test, targets_train, targets_test = train_test_split(scaled_feats,targets,test_size = 0.2,random_state=123)
print("Train/Test split complete")
#Need to reshape to match placeholder shape
targets_train = targets_train.reshape(targets_train.shape[0],-1)
targets_test = targets_test.reshape(targets_test.shape[0],-1)
#Network Structure
X = tf.placeholder(tf.float32, [None,features.shape[1]])
y = tf.placeholder(tf.float32, [None,1])
prob = tf.placeholder(tf.float32)
w1 = tf.Variable(tf.random_normal([features.shape[1],100]))
w2 = tf.Variable(tf.random_normal([100,1]))
b1 = tf.Variable(tf.random_normal([100]))
b2 = tf.Variable(tf.random_normal([1]))
layer1 = tf.nn.dropout(leaky_relu(tf.add(tf.matmul(X,w1),b1),0.2),prob)
out = tf.nn.dropout(tf.nn.tanh(tf.add(tf.matmul(layer1,w2),b2)),prob)
loss = tf.reduce_mean(tf.square(out - y))
train_step = tf.train.AdamOptimizer(0.001).minimize(loss)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
print("Training...")
minibatch = False
batchsize = 512
for i in range(1000):
if minibatch:
for x in range(0,targets_train.shape[0],batchsize):
sess.run(train_step, feed_dict={X:data_train[x:x+batchsize],y:targets_train[x:x+batchsize],prob:0.5})
else:
sess.run(train_step, feed_dict={X:data_train,y:targets_train,prob:0.5})
if i%25==0:
acc = sess.run(loss, feed_dict={X: data_train, y: targets_train,prob:1})
print ("Training accuracy: %. 392