<p dir="auto">It seems that the local cache feature introduced in <code class="notranslate">v3.1.10</code> breaks the propagation of exceptions via <code class="notranslate">celery.result.AsyncResult.get</code> if the result is serialized with JSON (and possibly others, except <code class="notranslate">pickle</code>).</p>
<p dir="auto">To reproduce:</p>
<div class="highlight highlight-source-python notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="# test.py

import sys
import traceback

from celery import Celery

app = Celery('test')
class Config:
    BROKER_URL = 'amqp://guest:guest@127.0.0.1:5672/celery'
    CELERY_RESULT_BACKEND = 'amqp'
    CELERY_RESULT_SERIALIZER = 'json'
app.config_from_object(Config)

@app.task
def exception_raising_task(to_raise=None):
    if to_raise is not None:
        raise RuntimeError(to_raise)
    return True

if __name__ == '__main__':
    result = exception_raising_task.delay()
    result.get()    # returns True

    result = exception_raising_task.delay('test')
    try:
        result.get()    # raises TypeError at celery.backends.base:exception_to_python
                        # displayed by print_exception() below
    except Exception as e:
        traceback.print_exception(*sys.exc_info())
    finally:
        result.get()    # correctly raises RuntimeError propagated from the task"><pre class="notranslate"><span class="pl-c"># test.py</span>

<span class="pl-k">import</span> <span class="pl-s1">sys</span>
<span class="pl-k">import</span> <span class="pl-s1">traceback</span>

<span class="pl-k">from</span> <span class="pl-s1">celery</span> <span class="pl-k">import</span> <span class="pl-v">Celery</span>

<span class="pl-s1">app</span> <span class="pl-c1">=</span> <span class="pl-v">Celery</span>(<span class="pl-s">'test'</span>)
<span class="pl-k">class</span> <span class="pl-v">Config</span>:
    <span class="pl-v">BROKER_URL</span> <span class="pl-c1">=</span> <span class="pl-s">'amqp://guest:guest@127.0.0.1:5672/celery'</span>
    <span class="pl-v">CELERY_RESULT_BACKEND</span> <span class="pl-c1">=</span> <span class="pl-s">'amqp'</span>
    <span class="pl-v">CELERY_RESULT_SERIALIZER</span> <span class="pl-c1">=</span> <span class="pl-s">'json'</span>
<span class="pl-s1">app</span>.<span class="pl-en">config_from_object</span>(<span class="pl-v">Config</span>)

<span class="pl-en">@<span class="pl-s1">app</span>.<span class="pl-s1">task</span></span>
<span class="pl-k">def</span> <span class="pl-en">exception_raising_task</span>(<span class="pl-s1">to_raise</span><span class="pl-c1">=</span><span class="pl-c1">None</span>):
    <span class="pl-k">if</span> <span class="pl-s1">to_raise</span> <span class="pl-c1">is</span> <span class="pl-c1">not</span> <span class="pl-c1">None</span>:
        <span class="pl-k">raise</span> <span class="pl-v">RuntimeError</span>(<span class="pl-s1">to_raise</span>)
    <span class="pl-k">return</span> <span class="pl-c1">True</span>

<span class="pl-k">if</span> <span class="pl-s1">__name__</span> <span class="pl-c1">==</span> <span class="pl-s">'__main__'</span>:
    <span class="pl-s1">result</span> <span class="pl-c1">=</span> <span class="pl-s1">exception_raising_task</span>.<span class="pl-en">delay</span>()
    <span class="pl-s1">result</span>.<span class="pl-en">get</span>()    <span class="pl-c"># returns True</span>

    <span class="pl-s1">result</span> <span class="pl-c1">=</span> <span class="pl-s1">exception_raising_task</span>.<span class="pl-en">delay</span>(<span class="pl-s">'test'</span>)
    <span class="pl-k">try</span>:
        <span class="pl-s1">result</span>.<span class="pl-en">get</span>()    <span class="pl-c"># raises TypeError at celery.backends.base:exception_to_python</span>
                        <span class="pl-c"># displayed by print_exception() below</span>
    <span class="pl-k">except</span> <span class="pl-v">Exception</span> <span class="pl-k">as</span> <span class="pl-s1">e</span>:
        <span class="pl-s1">traceback</span>.<span class="pl-en">print_exception</span>(<span class="pl-c1">*</span><span class="pl-s1">sys</span>.<span class="pl-en">exc_info</span>())
    <span class="pl-k">finally</span>:
        <span class="pl-s1">result</span>.<span class="pl-en">get</span>()    <span class="pl-c"># correctly raises RuntimeError propagated from the task</span></pre></div>
<p dir="auto">The exact traceback of the broken propagation is:</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="&gt;&gt;&gt; result = exception_raising_task.delay('test')
&gt;&gt;&gt; result.get()
Traceback (most recent call last):
  File &quot;&lt;stdin&gt;&quot;, line 1, in &lt;module&gt;
  File &quot;/opt/celery-dev/local/lib/python2.7/site-packages/celery-3.1.10-py2.7.egg/celery/result.py&quot;, line 171, in get
    self._get_task_meta()  # update self._cache
  File &quot;/opt/celery-dev/local/lib/python2.7/site-packages/celery-3.1.10-py2.7.egg/celery/result.py&quot;, line 332, in _get_task_meta
    return self._set_cache(meta)
  File &quot;/opt/celery-dev/local/lib/python2.7/site-packages/celery-3.1.10-py2.7.egg/celery/result.py&quot;, line 339, in _set_cache
    d['result'] = self.backend.exception_to_python(d['result'])
  File &quot;/opt/celery-dev/local/lib/python2.7/site-packages/celery-3.1.10-py2.7.egg/celery/backends/base.py&quot;, line 171, in exception_to_python
    from_utf8(exc['exc_type']), __name__)(exc['exc_message'])
TypeError: sequence index must be integer, not 'str'"><pre class="notranslate"><code class="notranslate">&gt;&gt;&gt; result = exception_raising_task.delay('test')
&gt;&gt;&gt; result.get()
Traceback (most recent call last):
  File "&lt;stdin&gt;", line 1, in &lt;module&gt;
  File "/opt/celery-dev/local/lib/python2.7/site-packages/celery-3.1.10-py2.7.egg/celery/result.py", line 171, in get
    self._get_task_meta()  # update self._cache
  File "/opt/celery-dev/local/lib/python2.7/site-packages/celery-3.1.10-py2.7.egg/celery/result.py", line 332, in _get_task_meta
    return self._set_cache(meta)
  File "/opt/celery-dev/local/lib/python2.7/site-packages/celery-3.1.10-py2.7.egg/celery/result.py", line 339, in _set_cache
    d['result'] = self.backend.exception_to_python(d['result'])
  File "/opt/celery-dev/local/lib/python2.7/site-packages/celery-3.1.10-py2.7.egg/celery/backends/base.py", line 171, in exception_to_python
    from_utf8(exc['exc_type']), __name__)(exc['exc_message'])
TypeError: sequence index must be integer, not 'str'
</code></pre></div>
<p dir="auto">Upon inspection, this seems to be caused by a duplicate call of <code class="notranslate">celery.result.AsyncResult._set_cache</code> in <code class="notranslate">celery.result.AsyncResult._get_task_meta</code>, which could be left there by a mistake:</p>
<div class="highlight highlight-source-python notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="# celery/result.py

    def _get_task_meta(self):
        if self._cache is None:
            meta = self.backend.get_task_meta(self.id)
            if meta:
                state = meta['status']
                if state == states.SUCCESS or state in states.PROPAGATE_STATES:
                    self._set_cache(meta)   # &lt;--- this call substitutes meta['result'] field, so the following will fail
                    return self._set_cache(meta)
            return meta
        return self._cache"><pre class="notranslate"><span class="pl-c"># celery/result.py</span>

    <span class="pl-k">def</span> <span class="pl-en">_get_task_meta</span>(<span class="pl-s1">self</span>):
        <span class="pl-k">if</span> <span class="pl-s1">self</span>.<span class="pl-s1">_cache</span> <span class="pl-c1">is</span> <span class="pl-c1">None</span>:
            <span class="pl-s1">meta</span> <span class="pl-c1">=</span> <span class="pl-s1">self</span>.<span class="pl-s1">backend</span>.<span class="pl-en">get_task_meta</span>(<span class="pl-s1">self</span>.<span class="pl-s1">id</span>)
            <span class="pl-k">if</span> <span class="pl-s1">meta</span>:
                <span class="pl-s1">state</span> <span class="pl-c1">=</span> <span class="pl-s1">meta</span>[<span class="pl-s">'status'</span>]
                <span class="pl-k">if</span> <span class="pl-s1">state</span> <span class="pl-c1">==</span> <span class="pl-s1">states</span>.<span class="pl-v">SUCCESS</span> <span class="pl-c1">or</span> <span class="pl-s1">state</span> <span class="pl-c1">in</span> <span class="pl-s1">states</span>.<span class="pl-v">PROPAGATE_STATES</span>:
                    <span class="pl-s1">self</span>.<span class="pl-en">_set_cache</span>(<span class="pl-s1">meta</span>)   <span class="pl-c"># &lt;--- this call substitutes meta['result'] field, so the following will fail</span>
                    <span class="pl-k">return</span> <span class="pl-s1">self</span>.<span class="pl-en">_set_cache</span>(<span class="pl-s1">meta</span>)
            <span class="pl-k">return</span> <span class="pl-s1">meta</span>
        <span class="pl-k">return</span> <span class="pl-s1">self</span>.<span class="pl-s1">_cache</span></pre></div>
<p dir="auto">So, the problem should be fixed by removing the line highlighted above (works for my local environment).</p>
<p dir="auto"><code class="notranslate">celery inspect conf</code> command always returns empty <code class="notranslate">BROKER_TRANSPORT_OPTIONS</code> even when it seems set properly with <code class="notranslate">celery report</code> command.</p>
<p dir="auto">I'm not sure this setting is really working on each active workers.  But I see some 1h+ tasks are running on duplicately on two workers.</p>
<p dir="auto">Output of <code class="notranslate">celery report</code></p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="redash@02f67972c8fb:/app$ celery report -A redash.worker        

software -&gt; celery:3.1.23 (Cipater) kombu:3.0.37 py:2.7.12
            billiard:3.3.0.23 redis:2.10.5
platform -&gt; system:Linux arch:64bit, ELF imp:CPython
loader   -&gt; celery.loaders.app.AppLoader
settings -&gt; transport:redis results:redis://***

CELERY_ACKS_LATE: True
CELERYBEAT_SCHEDULE: {
    'cleanup_query_results': {   'schedule': datetime.timedelta(0, 300),
                                 'task': 'redash.tasks.cleanup_query_results'},
    'cleanup_tasks': {   'schedule': datetime.timedelta(0, 300),
                         'task': 'redash.tasks.cleanup_tasks'},
    'refresh_queries': {   'schedule': datetime.timedelta(0, 30),
                           'task': 'redash.tasks.refresh_queries'},
    'refresh_schemas': {   'schedule': datetime.timedelta(0, 1800),
                           'task': 'redash.tasks.refresh_schemas'},
    'version_check': {   'schedule': &lt;crontab: 14 1 * * * (m/h/d/dM/MY)&gt;,
                         'task': 'redash.tasks.version_check'}}
BROKER_TRANSPORT_OPTIONS: {
    'visibility_timeout': 16200}
CELERY_TASK_RESULT_EXPIRES: 43200
CELERY_TIMEZONE: 'UTC'
CELERYD_PREFETCH_MULTIPLIER: 1
CELERY_RESULT_BACKEND: 'redis://***'"><pre class="notranslate"><code class="notranslate">redash@02f67972c8fb:/app$ celery report -A redash.worker        

software -&gt; celery:3.1.23 (Cipater) kombu:3.0.37 py:2.7.12
            billiard:3.3.0.23 redis:2.10.5
platform -&gt; system:Linux arch:64bit, ELF imp:CPython
loader   -&gt; celery.loaders.app.AppLoader
settings -&gt; transport:redis results:redis://***

CELERY_ACKS_LATE: True
CELERYBEAT_SCHEDULE: {
    'cleanup_query_results': {   'schedule': datetime.timedelta(0, 300),
                                 'task': 'redash.tasks.cleanup_query_results'},
    'cleanup_tasks': {   'schedule': datetime.timedelta(0, 300),
                         'task': 'redash.tasks.cleanup_tasks'},
    'refresh_queries': {   'schedule': datetime.timedelta(0, 30),
                           'task': 'redash.tasks.refresh_queries'},
    'refresh_schemas': {   'schedule': datetime.timedelta(0, 1800),
                           'task': 'redash.tasks.refresh_schemas'},
    'version_check': {   'schedule': &lt;crontab: 14 1 * * * (m/h/d/dM/MY)&gt;,
                         'task': 'redash.tasks.version_check'}}
BROKER_TRANSPORT_OPTIONS: {
    'visibility_timeout': 16200}
CELERY_TASK_RESULT_EXPIRES: 43200
CELERY_TIMEZONE: 'UTC'
CELERYD_PREFETCH_MULTIPLIER: 1
CELERY_RESULT_BACKEND: 'redis://***'
</code></pre></div>
<p dir="auto">Output of <code class="notranslate">celery inspect conf</code></p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="redash@02f67972c8fb:/app$ celery inspect conf -A redash.worker
-&gt; celery@6d971ca8001a: OK
    {
        &quot;ADMINS&quot;: [], 
        &quot;BROKER_CONNECTION_MAX_RETRIES&quot;: 100, 
        &quot;BROKER_CONNECTION_RETRY&quot;: true, 
        &quot;BROKER_CONNECTION_TIMEOUT&quot;: 4, 
        &quot;BROKER_FAILOVER_STRATEGY&quot;: null, 
        &quot;BROKER_HEARTBEAT&quot;: null, 
        &quot;BROKER_HEARTBEAT_CHECKRATE&quot;: 3.0, 
        &quot;BROKER_HOST&quot;: null, 
        &quot;BROKER_LOGIN_METHOD&quot;: null, 
        &quot;BROKER_PASSWORD&quot;: &quot;********&quot;, 
        &quot;BROKER_POOL_LIMIT&quot;: 10, 
        &quot;BROKER_PORT&quot;: null, 
        &quot;BROKER_TRANSPORT&quot;: null, 
        &quot;BROKER_TRANSPORT_OPTIONS&quot;: {}, "><pre class="notranslate"><code class="notranslate">redash@02f67972c8fb:/app$ celery inspect conf -A redash.worker
-&gt; celery@6d971ca8001a: OK
    {
        "ADMINS": [], 
        "BROKER_CONNECTION_MAX_RETRIES": 100, 
        "BROKER_CONNECTION_RETRY": true, 
        "BROKER_CONNECTION_TIMEOUT": 4, 
        "BROKER_FAILOVER_STRATEGY": null, 
        "BROKER_HEARTBEAT": null, 
        "BROKER_HEARTBEAT_CHECKRATE": 3.0, 
        "BROKER_HOST": null, 
        "BROKER_LOGIN_METHOD": null, 
        "BROKER_PASSWORD": "********", 
        "BROKER_POOL_LIMIT": 10, 
        "BROKER_PORT": null, 
        "BROKER_TRANSPORT": null, 
        "BROKER_TRANSPORT_OPTIONS": {}, 
</code></pre></div>
<h2 dir="auto">Steps to reproduce</h2>
<ul dir="auto">
<li>Set <code class="notranslate">BROKER_TRANSPORT_OPTIONS</code> with config file or in your code.</li>
<li>Run <code class="notranslate">celery inspect conf</code></li>
</ul>
<h2 dir="auto">Expected behavior</h2>
<ul dir="auto">
<li>Returns proper not empty <code class="notranslate">BROKER_TRANSPORT_OPTIONS</code></li>
</ul>
<h2 dir="auto">Actual behavior</h2>
<ul dir="auto">
<li>Returns empty <code class="notranslate">BROKER_TRANSPORT_OPTIONS</code></li>
</ul>