Thanks for reminding me to post the codes.
BTW, the train/valid split is 6k/1k.
Here is the Trainer definition:
class Trainer(nn.Module):
	def __init__(self, hidden_size, embed_dim, device, optim_cfg, n_layer=2, n_classes=5):
		super(Trainer, self).__init__()
		self.clsf = Clsf(hidden_size, 5).to(device)
		self.encoder = Encoder(hidden_size, embed_dim, device).to(device)
		self.device = device 
		self.cfg = optim_cfg 
		self.loss_fn = nn.CrossEntropyLoss().to(device) 

	def create_optimizer(self):
		self.model_params = list(self.encoder.parameters()) + list(self.clsf.parameters())
		if "optimizer_type" not in self.cfg:
			# default Adam 
			self.optimizer = optim.Adam(self.model_params, 
					   lr=self.cfg['lr'], 
					   weight_decay=self.cfg['weight_decay'])
		else:
			opt_type = self.cfg["optimizer_type"]
			if opt_type == "SGD":
				self.optimizer = torch.optim.SGD(
						self.model_params, lr=self.cfg['lr'],
						momentum=self.cfg.get("momentum", 0.9),
						weight_decay=self.cfg.get("weight_decay", 0.0))
			elif opt_type == "Adadelta":
				self.optimizer = torch.optim.Adadelta(self.model_params, lr=self.cfg['lr'])
			elif opt_type == "RMSprop":
				self.optimizer = torch.optim.RMSprop(self.model_params, lr=self.cfg['lr'])
			else:
				raise NotImplementedError(
					"Not supported optimizer [{}]".format(opt_type))
		print("CREATED optimizer ")

	def forward(self, batch, mode="train"):
		gold_labels, batch_output = self.encoder(batch)
		self.enc_out = batch_output
		self.optimizer.zero_grad() 
		s1s, s2s, s3s, s4s, s5s = [], [], [], [], [] 
		for _b, b in enumerate(batch_output):
			s1, s2, s3, s4, s5 = b 
			s1s.append(torch.mean(s1, dim=1)) 
			s2s.append(torch.mean(s2,dim=1)) 
			s3s.append(torch.mean(s3,dim=1)) 
			s4s.append(torch.mean(s4, dim=1)) 
			s5s.append(torch.mean(s5, dim=1))  

		x1 = torch.cat(s1s) 
		x2 = torch.cat(s2s) 
		x3 = torch.cat(s3s) 
		x4 = torch.cat(s4s) 
		x5 = torch.cat(s5s) 

		self.mlp_in = torch.cat([x1, x1-x2, x1-x3, x1-x4, x1-x5], dim=-1) 
		clsf_out = self.clsf(self.mlp_in) 
		self.out = F.softmax(clsf_out, dim=-1)  
		loss = self.loss_fn(self.out, gold_labels.to(self.device))
		pred = torch.argmax(clsf_out, dim=1)
		correct = (pred == gold_labels.to(device)).sum().item()
		loss.backward()
		self.optimizer.step()
		return loss.item(), correct  

	def validate(self, testdb):
		# eval model 
		#self.eval() 
		ks = list(testdb.all_sent_data.keys())
		accs = 0 
		cnt = 0 
		for k in ks:
			sample =[testdb[k]]
			if None in sample:
				continue 
			data_batch = my_collate_fn(sample, self.device, testdb.wordvecs)
			gold_tensors, pad_adj_matrices, anchors, stories_tensor, stories_vecs, allens  = data_batch
			cnt +=1 
			with torch.no_grad():
				gold_labels, batch_output = self.encoder(data_batch)
				s1s, s2s, s3s, s4s, s5s = [], [], [], [], [] 
				for _b, b in enumerate(batch_output):
					s1, s2, s3, s4, s5 = b 
					s1s.append(torch.mean(s1, dim=1)) 
					s2s.append(torch.mean(s2, dim=1)) 
					s3s.append(torch.mean(s3, dim=1)) 
					s4s.append(torch.mean(s4, dim=1)) 
					s5s.append(torch.mean(s5, dim=1)) 

				x1 = torch.cat(s1s) 
				x2 = torch.cat(s2s) 
				x3 = torch.cat(s3s) 
				x4 = torch.cat(s4s) 
				x5 = torch.cat(s5s) 

				mlp_in = torch.cat([x1, x1-x2, x1-x3, x1-x4, x1-x5], dim=-1) 
				clsf_out = self.clsf(mlp_in) 
				self.out = F.softmax(clsf_out, dim=-1)  
				pred = torch.argmax(self.out).item()
				if pred == gold_tensors.item():
					accs +=1 
		return accs / cnt 

And here is my training loop:
data_dir = "../data/"
dset = Dataset(data_dir, "train", "../../glove.6B.100d.txt") 
testdset = Dataset(data_dir, "test", "../../glove.6B.100d.txt") 
device= torch.device("cpu")
bsize = 32
val_every, val_after = 2, 2 
optim_cfg = {"lr":1e-4, "weight_decay":0.9}
net = Trainer(100,100, device,optim_cfg)
net.create_optimizer() 
ks = list(dset.all_sent_data.keys())
bad_batch = json.load(open("bad_batch.json","r")) 
n_iters = int(len(ks) / bsize) 
best_acc = 0 
for ep in range(0,20):
	now = time.time()
	losses = [] 
	accs = [] 
	ks_perm = np.random.permutation(ks) 
	for _iter in range(n_iters):
		batch = [] 
		batch = [dset[k] for k in ks_perm[_iter*bsize : (_iter+1)*bsize] if (None not in dset[k]) and (k not in bad_batch)]
		data_batch = my_collate_fn(batch,device, dset.wordvecs)
		loss, correct = net(data_batch)
		losses.append(loss)
		accs.append(correct)
	end = time.time() 
	mins = (end - now) / 60 
	print("TRAIN EPOCH {} LOSS {} ACC {} TIME {}".format(ep, np.mean(losses),np.mean(accs), mins))
	if (ep > val_after) and (ep % val_every == 0):
		net.eval()
		acc = net.validate(testdset)
		print("VALIDATING EPOCH {} ACC {}".format(ep, acc))
		if acc > best_acc:
			best_acc = acc  
		net.train()