Hello Zhou,
I have same problems in my work and tried your approach.
In case of yours, changing data type to double might help. There is similar 44 of yours.
I changed your code a bit like below.
# data type
# data_type = np.float32
data_type = np.float64

# Different weights for testing
# weight = np.random.randint(1, 1000, (3, 3, 3, 6)).astype(data_type)
# weight = 0.008 * np.random.randint(1, 1000, (3, 3, 3, 6)).astype(data_type)
weight = 10.12 * np.random.randint(1, 1000, (3, 3, 3, 6)).astype(data_type)

inputs = tf.Variable(1.5 * np.ones((1, 10, 10, 3), dtype=data_type))

# Tensorflow
tf_model = tf.contrib.slim.conv2d(inputs, 6, [3, 3], stride=1,
                                  weights_initializer=tf.constant_initializer(weight),
                                  biases_initializer=tf.constant_initializer(0),
                                  activation_fn=None)
with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    tf_output = sess.run(tf_model)

# Pytorch
weights = torch.from_numpy(weight).permute((3, 2, 0, 1))
biases = torch.from_numpy(np.zeros(6).astype(data_type))

inputs = torch.from_numpy(1.5 * np.ones((1, 3, 10, 10), dtype=data_type))
torch_model = nn.Conv2d(3, 6, kernel_size=3, stride=1, padding=1)
torch_model.weight = nn.Parameter(weights)
torch_model.bias = nn.Parameter(biases)
torch_output = torch_model(inputs)

# Compare results

print("diff max: ", (tf_output - torch_output.permute((0, 2, 3, 1)).data.numpy()).max())
print("diff norm: ", np.linalg.norm(tf_output - torch_output.permute((0, 2, 3, 1)).data.numpy()))

Here is the result.
Result in case np.float32
diff max:  0.015625
diff norm:  0.286411

Result in case np.float64
diff max:  2.9103830456733704e-11
diff norm:  7.428606633388099e-10