Thanks @SimonW
I test BN with 4 modes (with or w/o Affine, train or eval) and find that BN uses different bwd calculation for TRAIN/EVAL mode, regardless of the affine weights. See code below.
import torch
import torch.backends.cudnn as cudnn


if __name__ == '__main__':
    print("CUDNN Version: {}".format(cudnn.version()))
    cudnn.enabled = True
    cudnn.benchmark = True
    print("############## Check BN with Frozen Param")
    input_tensor = torch.rand(1, 64, 100, 100).cuda() * 100
    weight_init = torch.ones(64)
    bias_init = torch.zeros(64)
    mean_init = torch.rand(64) * 100
    var_init = torch.rand(64) * 100

    # torch.nn.BatchNorm2d(num_features, eps=1e-05, momentum=0.1, affine=True)
    BN0 = torch.nn.BatchNorm2d(64, affine=True)
    state_dict_0 = BN0.state_dict()
    state_dict_0['weight'].copy_(weight_init)
    state_dict_0['bias'].copy_(bias_init)
    state_dict_0['running_mean'].copy_(mean_init)
    state_dict_0['running_var'].copy_(var_init)
    BN0 = BN0.cuda()

    BN0.weight.requires_grad = False
    BN0.bias.requires_grad = False
    BN0.train()
    input_tensor_0 = input_tensor.clone()
    input_var_0 = torch.autograd.Variable(input_tensor_0, requires_grad=True)
    output_var_0 = BN0(input_var_0)
    loss_0 = output_var_0.sum()
    loss_0.backward()

    BN1 = torch.nn.BatchNorm2d(64, affine=True)
    state_dict_1 = BN1.state_dict()
    state_dict_1['weight'].copy_(weight_init)
    state_dict_1['bias'].copy_(bias_init)
    state_dict_1['running_mean'].copy_(mean_init)
    state_dict_1['running_var'].copy_(var_init)
    BN1 = BN1.cuda()

    BN1.train()
    input_tensor_1 = input_tensor.clone()
    input_var_1 = torch.autograd.Variable(input_tensor_1, requires_grad=True)
    output_var_1 = BN1(input_var_1)
    loss_1 = output_var_1.sum()
    loss_1.backward()

    BN2 = torch.nn.BatchNorm2d(64, affine=True)
    state_dict_2 = BN2.state_dict()
    state_dict_2['weight'].copy_(weight_init)
    state_dict_2['bias'].copy_(bias_init)
    state_dict_2['running_mean'].copy_(mean_init)
    state_dict_2['running_var'].copy_(var_init)
    BN2 = BN2.cuda()

    BN2.eval()
    input_tensor_2 = input_tensor.clone()
    input_var_2 = torch.autograd.Variable(input_tensor_2, requires_grad=True)
    output_var_2 = BN2(input_var_2)
    loss_2 = output_var_2.sum()
    loss_2.backward()

    BN3 = torch.nn.BatchNorm2d(64, affine=False)
    state_dict_3 = BN3.state_dict()
    state_dict_3['running_mean'].copy_(mean_init)
    state_dict_3['running_var'].copy_(var_init)
    BN3 = BN3.cuda()

    BN3.train()
    input_tensor_3 = input_tensor.clone()
    input_var_3 = torch.autograd.Variable(input_tensor_3, requires_grad=True)
    output_var_3 = BN3(input_var_3)
    loss_3 = output_var_3.sum()
    loss_3.backward()

    BN4 = torch.nn.BatchNorm2d(64, affine=False)
    state_dict_4 = BN4.state_dict()
    state_dict_4['running_mean'].copy_(mean_init)
    state_dict_4['running_var'].copy_(var_init)
    BN4 = BN4.cuda()

    BN4.eval()
    input_tensor_4 = input_tensor.clone()
    input_var_4 = torch.autograd.Variable(input_tensor_4, requires_grad=True)
    output_var_4 = BN4(input_var_4)
    loss_4 = output_var_4.sum()
    loss_4.backward()

    print((input_var_2.grad - input_var_1.grad).abs().max().data[0])
    print((input_var_4.grad - input_var_3.grad).abs().max().data[0])
    print((input_var_4.grad - input_var_2.grad).abs().max().data[0])
    print((input_var_3.grad - input_var_1.grad).abs().max().data[0])
    print((input_var_1.grad - input_var_0.grad).abs().max().data[0])

The result is
CUDNN Version: 6021
############## Check BN with Frozen Param
0.8908671140670776
0.8908671140670776
0.0
1.646934677523859e-08
0.0

Could you please show me how BN module use diffenrent bwd calculation in dependence on TRAIN/EVAL mode. I can not find it in the code.
Regards,