I found that indexing is very slow for backpropagation.
import time

import numpy as np
import torch
from torch.autograd import Variable

_num_units = 1000

def FC_layer(inputs, num_units=_num_units):
	input_size = inputs.size()[0]
	W = Variable(torch.rand(input_size, num_units), requires_grad=True)
	ret = torch.matmul(inputs, W)
	
	# Avoid large number
	# ret = torch.sigmoid(ret)
	return ret

def stupid_FC_layer(inputs, num_units=_num_units):
	input_size = inputs.size()[0]
	W = Variable(torch.rand(input_size, num_units), requires_grad=True)
	ret = Variable(torch.rand(num_units))

	splited_W = torch.split(W, 1, dim=1)
	for i in range(num_units):
	    ret[num_units - 1 - i] = torch.matmul(inputs, splited_W[i])
	
	# Avoid large number
	# ret = torch.sigmoid(ret)
	return ret

def another_stupid_FC_layer(inputs, num_units=_num_units):
	input_size = inputs.size()[0]
	W = Variable(torch.rand(input_size, num_units), requires_grad=True)

	ret_list = []
	splited_W = torch.split(W, 1, dim=1)
	for i in range(num_units):
	    ret_list.append(torch.matmul(inputs, splited_W[i]))
	ret = torch.cat(ret_list)

	# Avoid large number
	# ret = torch.sigmoid(ret)
	return ret

if __name__ == "__main__":
	num_layers = 2

	print("=" * 80)
	print("Common FC layer Test")
	print("-" * 80)
	print("Construct and Forward Time:")
	T0 = time.time()
	inputs = Variable(torch.rand(_num_units))
	for i in range(num_layers):
	    inputs = FC_layer(inputs)
	inputs = FC_layer(inputs, 1)
	T1 = time.time()
	print('Elapsed %.3f ms' % ((T1 - T0) * 1000.0))
	print("-" * 80)
	print("Backward Time:")
	T0 = time.time()
	inputs.backward()
	T1 = time.time()
	print('Elapsed %.3f ms' % ((T1 - T0) * 1000.0))

	print("=" * 80)
	print("Stupid FC layer Test")
	print("-" * 80)
	print("Construct and Forward Time:")
	T0 = time.time()
	inputs = Variable(torch.rand(_num_units))
	for i in range(num_layers):
	    inputs = stupid_FC_layer(inputs)
	inputs = stupid_FC_layer(inputs, 1)
	T1 = time.time()
	print('Elapsed %.3f ms' % ((T1 - T0) * 1000.0))
	print("-" * 80)
	print("Backward Time:")
	T0 = time.time()
	inputs.backward()
	T1 = time.time()
	print('Elapsed %.3f ms' % ((T1 - T0) * 1000.0))

	print("=" * 80)
	print("Another Stupid FC layer Test")
	print("-" * 80)
	print("Construct and Forward Time:")
	T0 = time.time()
	inputs = Variable(torch.rand(_num_units))
	for i in range(num_layers):
	    inputs = another_stupid_FC_layer(inputs)
	inputs = another_stupid_FC_layer(inputs, 1)
	T1 = time.time()
	print('Elapsed %.3f ms' % ((T1 - T0) * 1000.0))
	print("-" * 80)
	print("Backward Time:")
	T0 = time.time()
	inputs.backward()
	T1 = time.time()
	print('Elapsed %.3f ms' % ((T1 - T0) * 1000.0))

It shows that …
================================================================================
Common FC layer Test
--------------------------------------------------------------------------------
Construct and Forward Time:
Elapsed 17.241 ms
--------------------------------------------------------------------------------
Backward Time:
Elapsed 2.436 ms
================================================================================
Stupid FC layer Test
--------------------------------------------------------------------------------
Construct and Forward Time:
Elapsed 137.253 ms
--------------------------------------------------------------------------------
Backward Time:
Elapsed 2317.587 ms
================================================================================
Another Stupid FC layer Test
--------------------------------------------------------------------------------
Construct and Forward Time:
Elapsed 123.154 ms
--------------------------------------------------------------------------------
Backward Time:
Elapsed 1749.845 ms

I am using python 3.6, pytorch 0.2.0