Thanks @ptrblck, I tested as you suggested (on V100).
Sorry that the modified codes below are not well organized.
I first save the state_dict and input in the disk as follows,
import torch
import torch.nn as nn
import apex
from apex import amp

def init_model(in_channel=8, out_channel=4, kernel_size=(3, 1, 1), padding=(1, 0, 0), opt_level='O0'):
    model = nn.Conv3d(in_channel, out_channel, kernel_size=kernel_size, padding=padding, bias=False)
    model = model.cuda()
    parameters = model.parameters()
    optimizer = torch.optim.SGD(parameters,
                                    lr=0.1,
                                    )

    model, optimizer = amp.initialize(model, optimizer,
                                      opt_level=opt_level,
                                      keep_batchnorm_fp32=None if opt_level=='O1' else True,
                                      loss_scale=None
                                      )
    return model, optimizer

def compare_opt_level(in_channel, out_channel, kernel_size, padding, opt1='O0', opt2='O2'):
    input = torch.randn(size=(4, in_channel, 8, 16, 16)).cuda()

    n0, _ = init_model(in_channel, out_channel, kernel_size=kernel_size, padding=padding, opt_level=opt1)
    init_param = torch.randn_like(n0.weight)
    n0.weight = nn.Parameter(init_param)

    v0 = n0(input)

    save_dict = {'input': input.cpu(), 'state_dict':n0.state_dict()}
    torch.save(save_dict, 'ckpt_{}_{}_{}.pt'.format('_'.join(map(str, kernel_size)), in_channel, out_channel))

if __name__=='__main__':
    torch.random.manual_seed(1)
    ic, oc = 8, 4
    compare_opt_level(ic, oc, kernel_size=(3, 1, 1), padding=(1, 0, 0))
    compare_opt_level(ic, oc, kernel_size=(3, 3, 3), padding=(1, 1, 1))
    compare_opt_level(ic, oc, kernel_size=(1, 1, 1), padding=(0, 0, 0))
    compare_opt_level(ic, oc, kernel_size=(1, 3, 3), padding=(0, 1, 1))
    
    ic, oc = 512, 128
    compare_opt_level(ic, oc, kernel_size=(3, 1, 1), padding=(1, 0, 0), opt1='O0', opt2='O2')
    compare_opt_level(ic, oc, kernel_size=(3, 3, 3), padding=(1, 1, 1), opt1='O0', opt2='O2')
    compare_opt_level(ic, oc, kernel_size=(1, 1, 1), padding=(0, 0, 0), opt1='O0', opt2='O2')
    compare_opt_level(ic, oc, kernel_size=(1, 3, 3), padding=(0, 1, 1), opt1='O0', opt2='O2')

Then, save the output of the model with different opt-level separately with following code,
import argparse
import torch
import torch.nn as nn
import apex
from apex import amp

def init_model(in_channel=8, out_channel=4, kernel_size=(3, 1, 1), padding=(1, 0, 0), opt_level='O0', ckpt=None):
    model = nn.Conv3d(in_channel, out_channel, kernel_size=kernel_size, padding=padding, bias=False)
    model.load_state_dict(ckpt['state_dict'])
    model = model.cuda()
    parameters = model.parameters()
    optimizer = torch.optim.SGD(parameters,
                                    lr=0.1,
                                    )

    model, optimizer = amp.initialize(model, optimizer,
                                      opt_level=opt_level,
                                      keep_batchnorm_fp32=None if opt_level=='O1' else True,
                                      loss_scale=None
                                      )
    return model, optimizer

def get_result(in_channel, out_channel, kernel_size, padding, opt_level='O0'):
    checkpoint = torch.load('ckpt_{}_{}_{}.pt'.format('_'.join(map(str, kernel_size)), in_channel, out_channel))
    input = checkpoint['input'].cuda()

    n0, _ = init_model(in_channel, out_channel, kernel_size=kernel_size, padding=padding, opt_level=opt_level,
                       ckpt = checkpoint)
    v0 = n0(input)

    return v0


if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('--opt_level', type=str)
    args = parser.parse_args()

    result_dict = {}
    ic, oc = 8, 4
    ks = [[3, 1, 1], [3, 3, 3], [1, 1, 1], [1, 3, 3]]
    for k in ks:
        r = get_result(ic, oc, kernel_size=k, padding=[(i-1)//2 for i in k], opt_level=args.opt_level)
        result_dict['{}_{}_{}'.format('_'.join(map(str, k)), ic, oc)] = r

    ic, oc = 512, 128
    for k in ks:
        r = get_result(ic, oc, kernel_size=k, padding=[(i-1)//2 for i in k], opt_level=args.opt_level)
        result_dict['{}_{}_{}'.format('_'.join(map(str, k)), ic, oc)] = r

    torch.save(result_dict, 'result_{}'.format(args.opt_level))

Then compare the result files as follows,
import torch

if __name__ == '__main__':
    opt1 = 'O0'
    opt2 = 'O2'
    dict_O0 = torch.load('./result_'+opt1)
    dict_O2 = torch.load('./result_'+opt2)

    for k in sorted(dict_O0.keys()):

        v0 = dict_O0[k]
        v2 = dict_O2[k]
        k1, k2, k3, in_channel, out_channel = k.split('_')
        kernel_size = (k1, k2, k3)
        print('/////////////////////////////')
        print('Compare {}/{}, in/out channel {}/{}, kernel size {}'.format(
            opt1, opt2, in_channel, out_channel, kernel_size))
        print(torch.sqrt((v0 - v2) ** 2).mean(dim=(1, 2, 3, 4)))

And I got the similar result as before.
Compare O0/O2, in/out channel 512/128, kernel size ('1', '1', '1')
tensor([0.0064, 0.0064, 0.0064, 0.0064], device='cuda:0',
       grad_fn=<MeanBackward2>)
/////////////////////////////
Compare O0/O2, in/out channel 8/4, kernel size ('1', '1', '1')
tensor([0.0008, 0.0009, 0.0009, 0.0009], device='cuda:0',
       grad_fn=<MeanBackward2>)
/////////////////////////////
Compare O0/O2, in/out channel 512/128, kernel size ('1', '3', '3')
tensor([0.0184, 0.0184, 0.0184, 0.0183], device='cuda:0',
       grad_fn=<MeanBackward2>)
/////////////////////////////
Compare O0/O2, in/out channel 8/4, kernel size ('1', '3', '3')
tensor([0.0023, 0.0023, 0.0023, 0.0023], device='cuda:0',
       grad_fn=<MeanBackward2>)
/////////////////////////////
Compare O0/O2, in/out channel 512/128, kernel size ('3', '1', '1')
tensor([34.9032, 34.8360, 34.9023, 34.8901], device='cuda:0',
       grad_fn=<MeanBackward2>)
/////////////////////////////
Compare O0/O2, in/out channel 8/4, kernel size ('3', '1', '1')
tensor([0.0016, 0.0016, 0.0016, 0.0016], device='cuda:0',
       grad_fn=<MeanBackward2>)
/////////////////////////////
Compare O0/O2, in/out channel 512/128, kernel size ('3', '3', '3')
tensor([0.0303, 0.0303, 0.0304, 0.0303], device='cuda:0',
       grad_fn=<MeanBackward2>)
/////////////////////////////
Compare O0/O2, in/out channel 8/4, kernel size ('3', '3', '3')
tensor([0.0043, 0.0042, 0.0043, 0.0042], device='cuda:0',
       grad_fn=<MeanBackward2>)

By the way, if I shouldn’t call half(), then how can I use some torch functions such as torch.randn() inside a module? I got an type error without half() in pytorch v1.1. Will it be different in v1.4?