Ok, I've managed to get a decent improvement. Currently producing an MSE of 20 for training and 30 for test. This is after converting all layers to relu and increasing their size.
The problem appeared the activation function on the output. I was under the impression that a sigmoid output layer was common practice.
I was also normalizing before the test/train split and leaking information into the test set! (though this likely wasn't the cause of my problem)
Here is my 'improved' code:
import pandas as pd
import numpy as np
from scipy import stats
from sklearn.cross_validation import train_test_split
from sklearn import preprocessing
raw = pd.read_csv('parkinsons_updrs.data')
print("CSV loaded successfully")
events = raw.shape[0]
#drop events where any feature has a zscore > 3 for that feature
raw = raw[(np.abs(stats.zscore(raw)) < 3).all(axis=1)]
print("Removed " + str(events-raw.shape[0]) + " outlier events")
features = raw.drop(['subject#','test_time','motor_UPDRS','total_UPDRS'], axis = 1)
targets = raw['total_UPDRS'].get_values()
print("pre-processing complete")
#Split data into test/train
data_train, data_test, targets_train, targets_test = train_test_split(features,targets,test_size = 0.33,random_state=123)
print("Train/Test split complete")
#Normalize data to mean 0 and std 1
#We normalize after the split to avoid leaking information to the training set
data_train = (data_train-data_train.mean())/data_train.std()
data_test = (data_test-data_test.mean())/data_test.std()
targets_train = targets_train.reshape(targets_train.shape[0],-1)
targets_test = targets_test.reshape(targets_test.shape[0],-1)
#Network Structure
X = tf.placeholder(tf.float32, [None,18])
y = tf.placeholder(tf.float32, [None,1])
prob = tf.placeholder_with_default(1.0, shape=())
w1 = tf.Variable(tf.random_normal([18,72]))
w2 = tf.Variable(tf.random_normal([72,36]))
w3 = tf.Variable(tf.random_normal([36,1]))
b1 = tf.Variable(tf.random_normal([72]))
b2 = tf.Variable(tf.random_normal([36]))
b3 = tf.Variable(tf.random_normal([1]))
layer1 = tf.nn.dropout(tf.nn.relu(tf.add(tf.matmul(X,w1),b1)),prob)
layer2 = tf.nn.dropout(tf.nn.relu(tf.add(tf.matmul(layer1,w2),b2)),prob)
out = tf.nn.relu(tf.add(tf.matmul(layer2,w3),b3))
loss = tf.losses.mean_squared_error(y, out)
train_step = tf.train.AdamOptimizer(0.005).minimize(loss)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
print("Training...")
minibatch = True
batchsize = 512
for i in range(10000):
if minibatch:
for x in range(0,targets_train.shape[0],batchsize):
sess.run(train_step, feed_dict={X:data_train[x:x+batchsize],y:targets_train[x:x+batchsize],prob:0.5})
else:
sess.run(train_step, feed_dict={X:data_train,y:targets_train,prob:0.5})
if i%25==0:
acc = sess.run(loss, feed_dict={X: data_train, y: targets_train})
print ("Training accuracy: %. 392