<p dir="auto"><strong>System information</strong></p>
<ul dir="auto">
<li>Have I written custom code (as opposed to using a stock example script provided in TensorFlow): No -- using <a href="https://github.com/tensorflow/custom-op">https://github.com/tensorflow/custom-op</a> (But it breaks for addons too)</li>
<li>OS Platform and Distribution (e.g., Linux Ubuntu 16.04): Ubuntu16.04</li>
<li>TensorFlow installed from (source or binary): Binary</li>
<li>TensorFlow version (use command below): tf-nightly &amp; tf-nighty-2.0-preview</li>
</ul>
<p dir="auto"><strong>Describe the current behavior</strong><br>
Currently when I build a custom op in the <code class="notranslate">tensorflow/tensorflow:custom-op-ubuntu16</code> docker image using the defined steps I get an install-able pip package <code class="notranslate">tensorflow_zero_out-0.0.1-cp27-cp27mu-linux_x86_64.whl</code></p>
<p dir="auto">This works fine, however if I repair that wheel to be manylinux2010 compliant, then <code class="notranslate">tf.load_op_library</code> will fail to find the custom-op.</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="python -c &quot;import tensorflow as tf; print(dir(tf.load_op_library('manylinux/tensorflow_zero_out/python/ops/_zero_out_ops.so')))&quot;

['LIB_HANDLE', 'OP_LIST', 'ZeroOut', '_IS_TENSORFLOW_PLUGIN', 
'_InitOpDefLibrary', '__builtins__', '__doc__', '__name__', '__package__', 
'_collections', '_common_shapes', '_context', '_core', '_dispatch', '_doc_controls', 
'_dtypes', '_errors', '_execute', '_kwarg_only', '_op_def_lib', '_op_def_library', 
'_op_def_pb2', '_op_def_registry', '_ops', '_pywrap_tensorflow', '_six', 
'_tensor_shape', 'deprecated_endpoints', 'tf_export', 'zero_out',
 'zero_out_eager_fallback']"><pre class="notranslate"><code class="notranslate">python -c "import tensorflow as tf; print(dir(tf.load_op_library('manylinux/tensorflow_zero_out/python/ops/_zero_out_ops.so')))"

['LIB_HANDLE', 'OP_LIST', 'ZeroOut', '_IS_TENSORFLOW_PLUGIN', 
'_InitOpDefLibrary', '__builtins__', '__doc__', '__name__', '__package__', 
'_collections', '_common_shapes', '_context', '_core', '_dispatch', '_doc_controls', 
'_dtypes', '_errors', '_execute', '_kwarg_only', '_op_def_lib', '_op_def_library', 
'_op_def_pb2', '_op_def_registry', '_ops', '_pywrap_tensorflow', '_six', 
'_tensor_shape', 'deprecated_endpoints', 'tf_export', 'zero_out',
 'zero_out_eager_fallback']
</code></pre></div>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="python -c &quot;import tensorflow as tf;print(dir(tf.load_op_library('manylinux2010/tensorflow_zero_out/python/ops/_zero_out_ops.so')))&quot;

['LIB_HANDLE', 'OP_LIST', '_IS_TENSORFLOW_PLUGIN', 
'_InitOpDefLibrary', '__builtins__', '__doc__', '__name__', '__package__', 
'_collections', '_common_shapes', '_context', '_core', 
'_dispatch', '_doc_controls', '_dtypes', '_errors', '_execute', '_kwarg_only', 
'_op_def_lib', '_op_def_library', '_op_def_pb2', '_op_def_registry', '_ops', 
'_pywrap_tensorflow', '_six', '_tensor_shape', 'deprecated_endpoints', 'tf_export']"><pre class="notranslate"><code class="notranslate">python -c "import tensorflow as tf;print(dir(tf.load_op_library('manylinux2010/tensorflow_zero_out/python/ops/_zero_out_ops.so')))"

['LIB_HANDLE', 'OP_LIST', '_IS_TENSORFLOW_PLUGIN', 
'_InitOpDefLibrary', '__builtins__', '__doc__', '__name__', '__package__', 
'_collections', '_common_shapes', '_context', '_core', 
'_dispatch', '_doc_controls', '_dtypes', '_errors', '_execute', '_kwarg_only', 
'_op_def_lib', '_op_def_library', '_op_def_pb2', '_op_def_registry', '_ops', 
'_pywrap_tensorflow', '_six', '_tensor_shape', 'deprecated_endpoints', 'tf_export']
</code></pre></div>
<p dir="auto">Notice  <code class="notranslate">'zero_out'</code> &amp;  <code class="notranslate">'zero_out_eager_fallback'</code> are not found in the loaded library for manylinux2010</p>
<p dir="auto"><strong>Code to reproduce the issue</strong></p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="git clone https://github.com/tensorflow/custom-op.git &amp;&amp; cd custom-op
docker run -it --rm -v ${PWD}:/workspace -w /workspace tensorflow/tensorflow:custom-op-ubuntu16 /bin/bash

pip install tf-nightly
./configure.sh
bazel build build_pip_pkg
bazel-bin/build_pip_pkg artifacts

# Installed auditwheel is too old for manylinux2010
pip3 install --upgrade auditwheel

# Libtensorflow framework needs to be on LD path
export LD_LIBRARY_PATH=&quot;/usr/local/lib/python2.7/dist-packages/tensorflow_core&quot;

# Repair logs look more or less okay
auditwheel -v repair --plat manylinux2010_x86_64 artifacts/tensorflow_zero_out-0.0.1-cp27-cp27mu-linux_x86_64.whl &amp;&gt; repair.txt"><pre class="notranslate"><code class="notranslate">git clone https://github.com/tensorflow/custom-op.git &amp;&amp; cd custom-op
docker run -it --rm -v ${PWD}:/workspace -w /workspace tensorflow/tensorflow:custom-op-ubuntu16 /bin/bash

pip install tf-nightly
./configure.sh
bazel build build_pip_pkg
bazel-bin/build_pip_pkg artifacts

# Installed auditwheel is too old for manylinux2010
pip3 install --upgrade auditwheel

# Libtensorflow framework needs to be on LD path
export LD_LIBRARY_PATH="/usr/local/lib/python2.7/dist-packages/tensorflow_core"

# Repair logs look more or less okay
auditwheel -v repair --plat manylinux2010_x86_64 artifacts/tensorflow_zero_out-0.0.1-cp27-cp27mu-linux_x86_64.whl &amp;&gt; repair.txt
</code></pre></div>
<p dir="auto"><strong>Other info / logs</strong><br>
Here are the auditwheel repair logs:<br>
<a href="https://github.com/tensorflow/tensorflow/files/3521714/repair.txt">repair.txt</a></p>
<p dir="auto">Here are the readelf inspections of the so files:<br>
<a href="https://github.com/tensorflow/tensorflow/files/3521717/readelf.txt">readelf.txt</a><br>
<a href="https://github.com/tensorflow/tensorflow/files/3521718/readelf-manylinux2010.txt">readelf-manylinux2010.txt</a></p>
<p dir="auto">Here are the so files:<br>
<a href="https://github.com/tensorflow/tensorflow/files/3521726/so-files.zip">so-files.zip</a></p>
<p dir="auto">cc <a class="user-mention notranslate" data-hovercard-type="user" data-hovercard-url="/users/perfinion/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self" href="https://github.com/perfinion">@perfinion</a> <a class="user-mention notranslate" data-hovercard-type="user" data-hovercard-url="/users/gunan/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self" href="https://github.com/gunan">@gunan</a> <a class="user-mention notranslate" data-hovercard-type="user" data-hovercard-url="/users/yifeif/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self" href="https://github.com/yifeif">@yifeif</a></p>
<p dir="auto">--------------------------EDIT--------------------<br>
Here are the extracted whl directories which will work with the python <code class="notranslate">tf.load_op_library</code> commands from above. (Manylinux2010 repair makes it so the custom op depends on a newly copied libtensorflow_framework.so which is part of the new whl):<br>
<a href="https://github.com/tensorflow/tensorflow/files/3522649/custom-op-dirs.zip">custom-op-dirs.zip</a></p>
<p dir="auto">It seems if I have cpp custom op in a python function and I pass the python function to tf.data.Dataset.map it will crush.<br>
If I only call this python function outside, It will be ok.<br>
I've spend a whole afternoon to find the bug. I'm really mad about this bug.</p>
<p dir="auto">Have I written custom code (as opposed to using a stock example script provided in TensorFlow):yes<br>
OS Platform and Distribution (e.g., Linux Ubuntu 16.04): linux ubuntu 18.04<br>
TensorFlow installed from (source or binary):binary<br>
TensorFlow version (use command below): 2.0b1<br>
Python version:3.6<br>
CUDA/cuDNN version:10/7.4<br>
GPU model and memory:7.5/24gb</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="
import tensorflow as tf
import pdb
extr_module = tf.load_op_library('./build/libextr_module.so')
res = extr_module.test_bug() # ok

def aaa(filename):
    res = extr_module.test_bug() # Segmentation fault (core dumped)

    return tf.zeros([1], tf.float32)
    
dataset = tf.data.TextLineDataset(['aaa']).map(aaa)
"><pre class="notranslate"><code class="notranslate">
import tensorflow as tf
import pdb
extr_module = tf.load_op_library('./build/libextr_module.so')
res = extr_module.test_bug() # ok

def aaa(filename):
    res = extr_module.test_bug() # Segmentation fault (core dumped)

    return tf.zeros([1], tf.float32)
    
dataset = tf.data.TextLineDataset(['aaa']).map(aaa)

</code></pre></div>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="#include &quot;tensorflow/core/framework/op_kernel.h&quot;
#include &quot;tensorflow/core/framework/register_types.h&quot;
#include &quot;tensorflow/core/framework/tensor.h&quot;
#include &quot;tensorflow/core/framework/tensor_shape.h&quot;
#include &quot;tensorflow/core/framework/register_types.h&quot;
#include &quot;tensorflow/core/framework/op.h&quot;
#include &quot;tensorflow/core/framework/shape_inference.h&quot;
#include &quot;tensorflow/core/util/work_sharder.h&quot;

#include &lt;iostream&gt;
#include &lt;cmath&gt;

using namespace tensorflow;

REGISTER_OP(&quot;TestBug&quot;)
    .Output(&quot;dummy: float&quot;)
    .SetShapeFn([](::tensorflow::shape_inference::InferenceContext* c) {
        c-&gt;set_output(0, c-&gt;MakeShape({1}));
        return Status::OK();
    });

class TestBugOp : public OpKernel
{
public:
explicit TestBugOp(OpKernelConstruction* context)
        : OpKernel(context)
{

}

void Compute(OpKernelContext* context) override
{
    Tensor* dummy = NULL;
    OP_REQUIRES_OK(context, context-&gt;allocate_output(0, {1},
                                                     &amp;dummy));
}
};

REGISTER_KERNEL_BUILDER(
  Name(&quot;TestBug&quot;).Device(DEVICE_CPU),
  TestBugOp
);"><pre class="notranslate"><code class="notranslate">#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/register_types.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/register_types.h"
#include "tensorflow/core/framework/op.h"
#include "tensorflow/core/framework/shape_inference.h"
#include "tensorflow/core/util/work_sharder.h"

#include &lt;iostream&gt;
#include &lt;cmath&gt;

using namespace tensorflow;

REGISTER_OP("TestBug")
    .Output("dummy: float")
    .SetShapeFn([](::tensorflow::shape_inference::InferenceContext* c) {
        c-&gt;set_output(0, c-&gt;MakeShape({1}));
        return Status::OK();
    });

class TestBugOp : public OpKernel
{
public:
explicit TestBugOp(OpKernelConstruction* context)
        : OpKernel(context)
{

}

void Compute(OpKernelContext* context) override
{
    Tensor* dummy = NULL;
    OP_REQUIRES_OK(context, context-&gt;allocate_output(0, {1},
                                                     &amp;dummy));
}
};

REGISTER_KERNEL_BUILDER(
  Name("TestBug").Device(DEVICE_CPU),
  TestBugOp
);
</code></pre></div>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(extr_module)


# compiler flags
SET(CMAKE_CXX_FLAGS &quot;${CMAKE_CXX_FLAGS} -std=c++11 -O2 ${OpenMP_CXX_FLAGS} -Wall -fPIC -D_GLIBCXX_USE_CXX11_ABI=0 -DGOOGLE_CUDA=1&quot;)

# TensorFlow dependencies
EXECUTE_PROCESS(COMMAND python3 -c &quot;import os; os.environ['TF_CPP_MIN_LOG_LEVEL']='3'; import tensorflow as tf; print(tf.sysconfig.get_include(), end='', flush=True)&quot;  OUTPUT_VARIABLE TF_INC)

EXECUTE_PROCESS(COMMAND python3 -c &quot;import os; os.environ['TF_CPP_MIN_LOG_LEVEL']='3'; import tensorflow as tf; print(tf.sysconfig.get_lib(), end='', flush=True)&quot;  OUTPUT_VARIABLE TF_LIB)


MESSAGE(STATUS &quot;Found TF_INC: &quot; ${TF_INC})
#MESSAGE(STATUS &quot;Found TF_INC_EXTERNAL: &quot; ${TF_INC}/external/nsync/public)
MESSAGE(STATUS &quot;Found TF_LIB: &quot; ${TF_LIB})


INCLUDE_DIRECTORIES(${TF_INC})
#INCLUDE_DIRECTORIES(${TF_INC}/external/nsync/public)
LINK_DIRECTORIES(${TF_LIB})


ADD_LIBRARY(extr_module SHARED
  testbug.cc
)

TARGET_LINK_LIBRARIES(extr_module tensorflow_framework)"><pre class="notranslate"><code class="notranslate">
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(extr_module)


# compiler flags
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -O2 ${OpenMP_CXX_FLAGS} -Wall -fPIC -D_GLIBCXX_USE_CXX11_ABI=0 -DGOOGLE_CUDA=1")

# TensorFlow dependencies
EXECUTE_PROCESS(COMMAND python3 -c "import os; os.environ['TF_CPP_MIN_LOG_LEVEL']='3'; import tensorflow as tf; print(tf.sysconfig.get_include(), end='', flush=True)"  OUTPUT_VARIABLE TF_INC)

EXECUTE_PROCESS(COMMAND python3 -c "import os; os.environ['TF_CPP_MIN_LOG_LEVEL']='3'; import tensorflow as tf; print(tf.sysconfig.get_lib(), end='', flush=True)"  OUTPUT_VARIABLE TF_LIB)


MESSAGE(STATUS "Found TF_INC: " ${TF_INC})
#MESSAGE(STATUS "Found TF_INC_EXTERNAL: " ${TF_INC}/external/nsync/public)
MESSAGE(STATUS "Found TF_LIB: " ${TF_LIB})


INCLUDE_DIRECTORIES(${TF_INC})
#INCLUDE_DIRECTORIES(${TF_INC}/external/nsync/public)
LINK_DIRECTORIES(${TF_LIB})


ADD_LIBRARY(extr_module SHARED
  testbug.cc
)

TARGET_LINK_LIBRARIES(extr_module tensorflow_framework)
</code></pre></div>