After so many days, I still didn’t find any solutions to this. I tried all the same code in a old machine, no problem. Then now I tried again in another new machine, same problem happens.
my setup.py
import os
# os.environ['CC'] = 'g++'
# os.environ['CXX'] = 'g++'
from setuptools import setup
import sys
import re
from distutils import sysconfig
from distutils.core import setup, Extension

# if sys.platform == 'linux' or sys.platform == 'darwin':
#   sysconfig.get_config_var(None)  # to fill up _config_vars
#   d = sysconfig._config_vars
#   for x in ['OPT', 'CFLAGS', 'PY_CFLAGS', 'PY_CORE_CFLAGS', 'CONFIGURE_CFLAGS', 'LDSHARED']:
#     d[x] = re.sub(' -g ', ' ', d[x])
#     d[x] = re.sub('^-g ', '',  d[x])
#     d[x] = re.sub(' -g$', '',  d[x])
from torch.utils.cpp_extension import BuildExtension, CppExtension



setup(
    name = 'op_conv_ext',
    ext_modules=[
        CppExtension(name='op_conv_ext', sources=['extension/conv3x3s1.cpp'],
       extra_compile_args={'cxx': ['-O3', '-pthread', '-mavx2', '-mfma', '-funroll-loops', '-fomit-frame-pointer', '-finline-small-functions', '-finline-functions-called-once', 
           '-finline-functions', '-ffp-contract=on', "-mno-vzeroupper", '-m64', '-g', '-DNDEBUG[-march=native] [-mtune=native]' ]},
       extra_link_args=['-O3', '-pthread', '-mavx2', '-mfma', '-funroll-loops', 
           '-fomit-frame-pointer', '-finline-small-functions', '-finline-functions-called-once', 
           '-finline-functions', '-ffp-contract=on', "-mno-vzeroupper", '-m64', '-g', '-DNDEBUG[-march=native] [-mtune=native]' ],
    )],
    cmdclass={
        'build_ext': BuildExtension
    })


The output from python3 setup.py is
x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/home/zyx/.local/lib/python3.6/site-packages/torch/include -I/home/zyx/.local/lib/python3.6/site-packages/torch/include/torch/csrc/api/include -I/home/zyx/.local/lib/python3.6/site-packages/torch/include/TH -I/home/zyx/.local/lib/python3.6/site-packages/torch/include/THC -I/usr/include/python3.6m -c extension/conv3x3s1.cpp -o build/temp.linux-x86_64-3.6/extension/conv3x3s1.o -O3 -pthread -mavx2 -mfma -funroll-loops -fomit-frame-pointer -finline-small-functions -finline-functions-called-once -finline-functions -ffp-contract=on -mno-vzeroupper -m64 -g -DNDEBUG[-march=native] [-mtune=native] -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=op_conv_ext -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11
x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.6/extension/conv3x3s1.o -o build/lib.linux-x86_64-3.6/op_conv_ext.cpython-36m-x86_64-linux-gnu.so -O3 -pthread -mavx2 -mfma -funroll-loops -fomit-frame-pointer -finline-small-functions -finline-functions-called-once -finline-functions -ffp-contract=on -mno-vzeroupper -m64 -g -DNDEBUG[-march=native] [-mtune=native]

Flags in the source file is the same.
Now the performance invoked from python is 35ms, that invoked from c++ which is built by g++ directly is 25ms.