<p dir="auto">I'm trying distributed training with pytorch, and I encountered the following  error.</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="  File &quot;/home.local/kamo/.anyenv/envs/pyenv/versions/torch2/lib/python3.6/site-packages/torch/nn/modules/module.py&quot;, line 259, in __call__
    result = self.forward(*input, **kwargs)
  File &quot;/home.local/kamo/.anyenv/envs/pyenv/versions/torch2/lib/python3.6/site-packages/torch/nn/parallel/distributed.py&quot;, line 154, in forward
    self._sync_params()
  File &quot;/home.local/kamo/.anyenv/envs/pyenv/versions/torch2/lib/python3.6/site-packages/torch/nn/parallel/distributed.py&quot;, line 183, in _sync_params
    flat_buffers = _flatten_tensors(buffers)
  File &quot;/home.local/kamo/.anyenv/envs/pyenv/versions/torch2/lib/python3.6/site-packages/torch/_utils.py&quot;, line 111, in _flatten_tensors
    flat = tensors[0].new(size)
IndexError: list index out of range"><pre class="notranslate"><code class="notranslate">  File "/home.local/kamo/.anyenv/envs/pyenv/versions/torch2/lib/python3.6/site-packages/torch/nn/modules/module.py", line 259, in __call__
    result = self.forward(*input, **kwargs)
  File "/home.local/kamo/.anyenv/envs/pyenv/versions/torch2/lib/python3.6/site-packages/torch/nn/parallel/distributed.py", line 154, in forward
    self._sync_params()
  File "/home.local/kamo/.anyenv/envs/pyenv/versions/torch2/lib/python3.6/site-packages/torch/nn/parallel/distributed.py", line 183, in _sync_params
    flat_buffers = _flatten_tensors(buffers)
  File "/home.local/kamo/.anyenv/envs/pyenv/versions/torch2/lib/python3.6/site-packages/torch/_utils.py", line 111, in _flatten_tensors
    flat = tensors[0].new(size)
IndexError: list index out of range
</code></pre></div>
<p dir="auto">Reproduce code:</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="# main.py
import os

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torch.autograd import Variable
import torch.distributed as dist
import torch.utils.data.distributed


class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
        self.conv2_drop = nn.Dropout2d()
        self.fc1 = nn.Linear(320, 50)
        self.fc2 = nn.Linear(50, 10)

    def forward(self, x):
        x = F.relu(F.max_pool2d(self.conv1(x), 2))
        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
        x = x.view(-1, 320)
        x = F.relu(self.fc1(x))
        x = F.dropout(x, training=self.training)
        x = self.fc2(x)
        return F.log_softmax(x)

def main():
    BACKEND = os.environ['BACKEND']
    WORLD_SIZE = os.environ['WORLD_SIZE']
    INIT_METHOD = os.getenv('INIT_METHOD', 'env://')

    dist.init_process_group(
            init_method=INIT_METHOD,
            backend=BACKEND, world_size=int(WORLD_SIZE))
    train_dataset = datasets.MNIST(root='./torch', train=True, download=True,
                                   transform=transforms.Compose([
                                       transforms.ToTensor(),
                                       transforms.Normalize((0.1307,),
                                                            (0.3081,))]))
    train_sampler =\
        torch.utils.data.distributed.DistributedSampler(train_dataset)
    train_loader = torch.utils.data.DataLoader(
        train_dataset, sampler=train_sampler,
        batch_size=64, shuffle=False, num_workers=0, pin_memory=False)

    model = Net()
    model.cuda()
    model = torch.nn.parallel.DistributedDataParallel(model)
    optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5)

    train_sampler.set_epoch(0)
    model.train()
    for data, target in train_loader:
        data, target = data.cuda(), target.cuda()
        data, target = Variable(data), Variable(target)
        optimizer.zero_grad()
        output = model(data)
        loss = F.nll_loss(output, target)
        loss.backward()
        optimizer.step()

if __name__ == '__main__':
    import argparse
    p = argparse.ArgumentParser()
    main()"><pre class="notranslate"><code class="notranslate"># main.py
import os

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torch.autograd import Variable
import torch.distributed as dist
import torch.utils.data.distributed


class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
        self.conv2_drop = nn.Dropout2d()
        self.fc1 = nn.Linear(320, 50)
        self.fc2 = nn.Linear(50, 10)

    def forward(self, x):
        x = F.relu(F.max_pool2d(self.conv1(x), 2))
        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
        x = x.view(-1, 320)
        x = F.relu(self.fc1(x))
        x = F.dropout(x, training=self.training)
        x = self.fc2(x)
        return F.log_softmax(x)

def main():
    BACKEND = os.environ['BACKEND']
    WORLD_SIZE = os.environ['WORLD_SIZE']
    INIT_METHOD = os.getenv('INIT_METHOD', 'env://')

    dist.init_process_group(
            init_method=INIT_METHOD,
            backend=BACKEND, world_size=int(WORLD_SIZE))
    train_dataset = datasets.MNIST(root='./torch', train=True, download=True,
                                   transform=transforms.Compose([
                                       transforms.ToTensor(),
                                       transforms.Normalize((0.1307,),
                                                            (0.3081,))]))
    train_sampler =\
        torch.utils.data.distributed.DistributedSampler(train_dataset)
    train_loader = torch.utils.data.DataLoader(
        train_dataset, sampler=train_sampler,
        batch_size=64, shuffle=False, num_workers=0, pin_memory=False)

    model = Net()
    model.cuda()
    model = torch.nn.parallel.DistributedDataParallel(model)
    optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5)

    train_sampler.set_epoch(0)
    model.train()
    for data, target in train_loader:
        data, target = data.cuda(), target.cuda()
        data, target = Variable(data), Variable(target)
        optimizer.zero_grad()
        output = model(data)
        loss = F.nll_loss(output, target)
        loss.backward()
        optimizer.step()

if __name__ == '__main__':
    import argparse
    p = argparse.ArgumentParser()
    main()
</code></pre></div>
<p dir="auto">Launching shell script:</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="#!/bin/sh

distributed_set_up() {
  export TEMP_DIR=&quot;$(mktemp -d)&quot;
  rm -rf &quot;$TEMP_DIR/&quot;*
  mkdir &quot;$TEMP_DIR/barrier&quot;
  mkdir &quot;$TEMP_DIR/test_dir&quot;
}

distributed_tear_down() {
  rm -rf &quot;$TEMP_DIR&quot;
}


distributed_set_up
ls $TEMP_DIR
backend=gloo
world_size=2
for i in seq 0 $(expr $world_size - 1);do
    MASTER_ADDR='29500' MASTER_PORT='127.0.0.1' RANK=$i BACKEND=$backend WORLD_SIZE=$world_size INIT_METHOD='file://'$TEMP_DIR'/shared_init_file' python main.py &amp;
done
wait
distributed_tear_down"><pre class="notranslate"><code class="notranslate">#!/bin/sh

distributed_set_up() {
  export TEMP_DIR="$(mktemp -d)"
  rm -rf "$TEMP_DIR/"*
  mkdir "$TEMP_DIR/barrier"
  mkdir "$TEMP_DIR/test_dir"
}

distributed_tear_down() {
  rm -rf "$TEMP_DIR"
}


distributed_set_up
ls $TEMP_DIR
backend=gloo
world_size=2
for i in seq 0 $(expr $world_size - 1);do
    MASTER_ADDR='29500' MASTER_PORT='127.0.0.1' RANK=$i BACKEND=$backend WORLD_SIZE=$world_size INIT_METHOD='file://'$TEMP_DIR'/shared_init_file' python main.py &amp;
done
wait
distributed_tear_down
</code></pre></div>
<p dir="auto">Perhaps you are not taking account modules not having any buffers, so I modified <a href="https://github.com/pytorch/pytorch/blob/master/torch/nn/parallel/distributed.py#L182-L192">https://github.com/pytorch/pytorch/blob/master/torch/nn/parallel/distributed.py#L182-L192</a> like this and it works,</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="        buffers = list(self.module._all_buffers())                                                                                                                                                  
        if len(buffers) &gt; 0:                                                                                                                                                                        
            flat_buffers = _flatten_tensors(buffers)                                                                                                                                                
            dist.broadcast(flat_buffers, 0)                                                                                                                                                         
            for buf, synced in zip(buffers, _unflatten_tensors(flat_buffers, buffers)):                                                                                                             
                buf.copy_(synced)                                                                                                                                                                   
                                                                                                                                                                                                    
            # intra-node buffer sync                                                                                                                                                                
            result = broadcast_coalesced(buffers, self.device_ids, self.broadcast_bucket_size)                                                                                                      
            for tensors, module in zip(result[1:], self._module_copies[1:]):                                                                                                                        
                for tensor, buf in zip(tensors, module._all_buffers()):                                                                                                                             
                    buf.set_(tensor)      "><pre class="notranslate"><code class="notranslate">        buffers = list(self.module._all_buffers())                                                                                                                                                  
        if len(buffers) &gt; 0:                                                                                                                                                                        
            flat_buffers = _flatten_tensors(buffers)                                                                                                                                                
            dist.broadcast(flat_buffers, 0)                                                                                                                                                         
            for buf, synced in zip(buffers, _unflatten_tensors(flat_buffers, buffers)):                                                                                                             
                buf.copy_(synced)                                                                                                                                                                   
                                                                                                                                                                                                    
            # intra-node buffer sync                                                                                                                                                                
            result = broadcast_coalesced(buffers, self.device_ids, self.broadcast_bucket_size)                                                                                                      
            for tensors, module in zip(result[1:], self._module_copies[1:]):                                                                                                                        
                for tensor, buf in zip(tensors, module._all_buffers()):                                                                                                                             
                    buf.set_(tensor)      
</code></pre></div>
<p dir="auto">My correction is not right?</p>
<p dir="auto">After this, maybe this is another problem, but i got the following message at process shutdown. The training seems to success, so I think this is not a critical problem.</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="terminate called after throwing an instance of 'gloo::EnforceNotMet'
  what():  [enforce fail at /home.local/kamo/work/pytorch/torch/lib/gloo/gloo/cuda.cu:249] error == cudaSuccess. 29 vs 0. Error at: /home.local/kamo/work/pytorch/torch/lib/gloo/gloo/cuda.cu:249: driver shutting down
terminate called after throwing an instance of 'gloo::EnforceNotMet'
  what():  [enforce fail at /home.local/kamo/work/pytorch/torch/lib/gloo/gloo/cuda.cu:249] error == cudaSuccess. 29 vs 0. Error at: /home.local/kamo/work/pytorch/torch/lib/gloo/gloo/cuda.cu:249: driver shutting down
./run.sh: line 22: 19545 Aborted                 (core dumped) MASTER_ADDR='29500' MASTER_PORT='127.0.0.1' RANK=$i BACKEND=$backend WORLD_SIZE=$world_size INIT_METHOD='file://'$TEMP_DIR'/shared_init_file' python main.py
./run.sh: line 22: 19546 Aborted                 (core dumped) MASTER_ADDR='29500' MASTER_PORT='127.0.0.1'  RANK=$i BACKEND=$backend WORLD_SIZE=$world_size INIT_METHOD='file://'$TEMP_DIR'/shared_init_file' python main.py"><pre class="notranslate"><code class="notranslate">terminate called after throwing an instance of 'gloo::EnforceNotMet'
  what():  [enforce fail at /home.local/kamo/work/pytorch/torch/lib/gloo/gloo/cuda.cu:249] error == cudaSuccess. 29 vs 0. Error at: /home.local/kamo/work/pytorch/torch/lib/gloo/gloo/cuda.cu:249: driver shutting down
terminate called after throwing an instance of 'gloo::EnforceNotMet'
  what():  [enforce fail at /home.local/kamo/work/pytorch/torch/lib/gloo/gloo/cuda.cu:249] error == cudaSuccess. 29 vs 0. Error at: /home.local/kamo/work/pytorch/torch/lib/gloo/gloo/cuda.cu:249: driver shutting down
./run.sh: line 22: 19545 Aborted                 (core dumped) MASTER_ADDR='29500' MASTER_PORT='127.0.0.1' RANK=$i BACKEND=$backend WORLD_SIZE=$world_size INIT_METHOD='file://'$TEMP_DIR'/shared_init_file' python main.py
./run.sh: line 22: 19546 Aborted                 (core dumped) MASTER_ADDR='29500' MASTER_PORT='127.0.0.1'  RANK=$i BACKEND=$backend WORLD_SIZE=$world_size INIT_METHOD='file://'$TEMP_DIR'/shared_init_file' python main.py
</code></pre></div>
<p dir="auto">My environment:</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="OS: CentOS7
gcc:  4.8.5 20150623 (Red Hat 4.8.5-11) (GCC) 
Python3.6
Pytorch: de757805fcdbcbb831d51827d72e73e55a49a106 (built by setup.py)
GPU: GeForce GTX 1080
CUDA8.0
CUDNN6.0
Without MPI"><pre class="notranslate"><code class="notranslate">OS: CentOS7
gcc:  4.8.5 20150623 (Red Hat 4.8.5-11) (GCC) 
Python3.6
Pytorch: de757805fcdbcbb831d51827d72e73e55a49a106 (built by setup.py)
GPU: GeForce GTX 1080
CUDA8.0
CUDNN6.0
Without MPI
</code></pre></div>
<h2 dir="auto"><g-emoji class="g-emoji" alias="bug" fallback-src="https://github.githubassets.com/images/icons/emoji/unicode/1f41b.png">🐛</g-emoji> Bug</h2>

<h2 dir="auto">To Reproduce</h2>
<p dir="auto">Steps to reproduce the behavior:</p>
<div class="highlight highlight-source-python notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="m = nn.MultiheadAttention(embed_dim=20, num_heads=5, dropout=0.1)

i = torch.randn(4, 5, 20)
q = k = v = i.transpose(0, 1)
key_padding_mask = seq_len_to_mask([5, 3, 1, 0], max_len=5)

ao, aow = m(q, k, v, key_padding_mask=key_padding_mask)
ao = ao.transpose(0, 1)

print(aow)"><pre class="notranslate"><span class="pl-s1">m</span> <span class="pl-c1">=</span> <span class="pl-s1">nn</span>.<span class="pl-v">MultiheadAttention</span>(<span class="pl-s1">embed_dim</span><span class="pl-c1">=</span><span class="pl-c1">20</span>, <span class="pl-s1">num_heads</span><span class="pl-c1">=</span><span class="pl-c1">5</span>, <span class="pl-s1">dropout</span><span class="pl-c1">=</span><span class="pl-c1">0.1</span>)

<span class="pl-s1">i</span> <span class="pl-c1">=</span> <span class="pl-s1">torch</span>.<span class="pl-en">randn</span>(<span class="pl-c1">4</span>, <span class="pl-c1">5</span>, <span class="pl-c1">20</span>)
<span class="pl-s1">q</span> <span class="pl-c1">=</span> <span class="pl-s1">k</span> <span class="pl-c1">=</span> <span class="pl-s1">v</span> <span class="pl-c1">=</span> <span class="pl-s1">i</span>.<span class="pl-en">transpose</span>(<span class="pl-c1">0</span>, <span class="pl-c1">1</span>)
<span class="pl-s1">key_padding_mask</span> <span class="pl-c1">=</span> <span class="pl-en">seq_len_to_mask</span>([<span class="pl-c1">5</span>, <span class="pl-c1">3</span>, <span class="pl-c1">1</span>, <span class="pl-c1">0</span>], <span class="pl-s1">max_len</span><span class="pl-c1">=</span><span class="pl-c1">5</span>)

<span class="pl-s1">ao</span>, <span class="pl-s1">aow</span> <span class="pl-c1">=</span> <span class="pl-en">m</span>(<span class="pl-s1">q</span>, <span class="pl-s1">k</span>, <span class="pl-s1">v</span>, <span class="pl-s1">key_padding_mask</span><span class="pl-c1">=</span><span class="pl-s1">key_padding_mask</span>)
<span class="pl-s1">ao</span> <span class="pl-c1">=</span> <span class="pl-s1">ao</span>.<span class="pl-en">transpose</span>(<span class="pl-c1">0</span>, <span class="pl-c1">1</span>)

<span class="pl-en">print</span>(<span class="pl-s1">aow</span>)</pre></div>
<p dir="auto">the print is:</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="     tensor([[[0.2822, 0.2024, 0.1926, 0.2565, 0.1463],
     [0.2020, 0.1906, 0.1721, 0.2159, 0.2425],
     [0.2074, 0.2127, 0.0882, 0.2422, 0.3333],
     [0.2298, 0.1938, 0.2093, 0.1432, 0.2335],
     [0.2531, 0.1772, 0.3045, 0.1246, 0.1881]],

    [[0.3693, 0.3857, 0.3561, 0.0000, 0.0000],
     [0.3655, 0.2712, 0.3918, 0.0000, 0.0000],
     [0.2230, 0.3795, 0.2549, 0.0000, 0.0000],
     [0.1698, 0.3805, 0.2318, 0.0000, 0.0000],
     [0.2986, 0.3093, 0.4405, 0.0000, 0.0000]],

    [[0.8889, 0.0000, 0.0000, 0.0000, 0.0000],
     [1.1111, 0.0000, 0.0000, 0.0000, 0.0000],
     [1.1111, 0.0000, 0.0000, 0.0000, 0.0000],
     [1.1111, 0.0000, 0.0000, 0.0000, 0.0000],
     [1.1111, 0.0000, 0.0000, 0.0000, 0.0000]],

    [[   nan,    nan,    nan,    nan,    nan],
     [   nan,    nan,    nan,    nan,    nan],
     [   nan,    nan,    nan,    nan,    nan],
     [   nan,    nan,    nan,    nan,    nan],
     [   nan,    nan,    nan,    nan,    nan]]], grad_fn=&lt;DivBackward0&gt;)"><pre class="notranslate"><code class="notranslate">     tensor([[[0.2822, 0.2024, 0.1926, 0.2565, 0.1463],
     [0.2020, 0.1906, 0.1721, 0.2159, 0.2425],
     [0.2074, 0.2127, 0.0882, 0.2422, 0.3333],
     [0.2298, 0.1938, 0.2093, 0.1432, 0.2335],
     [0.2531, 0.1772, 0.3045, 0.1246, 0.1881]],

    [[0.3693, 0.3857, 0.3561, 0.0000, 0.0000],
     [0.3655, 0.2712, 0.3918, 0.0000, 0.0000],
     [0.2230, 0.3795, 0.2549, 0.0000, 0.0000],
     [0.1698, 0.3805, 0.2318, 0.0000, 0.0000],
     [0.2986, 0.3093, 0.4405, 0.0000, 0.0000]],

    [[0.8889, 0.0000, 0.0000, 0.0000, 0.0000],
     [1.1111, 0.0000, 0.0000, 0.0000, 0.0000],
     [1.1111, 0.0000, 0.0000, 0.0000, 0.0000],
     [1.1111, 0.0000, 0.0000, 0.0000, 0.0000],
     [1.1111, 0.0000, 0.0000, 0.0000, 0.0000]],

    [[   nan,    nan,    nan,    nan,    nan],
     [   nan,    nan,    nan,    nan,    nan],
     [   nan,    nan,    nan,    nan,    nan],
     [   nan,    nan,    nan,    nan,    nan],
     [   nan,    nan,    nan,    nan,    nan]]], grad_fn=&lt;DivBackward0&gt;)
</code></pre></div>

<h2 dir="auto">Expected behavior</h2>

<h2 dir="auto">Environment</h2>
<p dir="auto">Please copy and paste the output from our<br>
<a href="https://raw.githubusercontent.com/pytorch/pytorch/master/torch/utils/collect_env.py" rel="nofollow">environment collection script</a><br>
(or fill out the checklist below manually).</p>
<p dir="auto">You can get the script and run it with:</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="wget https://raw.githubusercontent.com/pytorch/pytorch/master/torch/utils/collect_env.py
# For security purposes, please check the contents of collect_env.py before running it.
python collect_env.py"><pre class="notranslate"><code class="notranslate">wget https://raw.githubusercontent.com/pytorch/pytorch/master/torch/utils/collect_env.py
# For security purposes, please check the contents of collect_env.py before running it.
python collect_env.py
</code></pre></div>
<ul dir="auto">
<li>PyTorch Version (e.g., 1.0):  1.3.0</li>
<li>OS (e.g., Linux):  macos</li>
<li>How you installed PyTorch (<code class="notranslate">conda</code>, <code class="notranslate">pip</code>, source):</li>
<li>Build command you used (if compiling from source):</li>
<li>Python version:</li>
<li>CUDA/cuDNN version:</li>
<li>GPU models and configuration:</li>
<li>Any other relevant information:</li>
</ul>
<h2 dir="auto">Additional context</h2>

<p dir="auto">cc <a class="user-mention notranslate" data-hovercard-type="user" data-hovercard-url="/users/ezyang/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self" href="https://github.com/ezyang">@ezyang</a> <a class="user-mention notranslate" data-hovercard-type="user" data-hovercard-url="/users/gchanan/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self" href="https://github.com/gchanan">@gchanan</a> <a class="user-mention notranslate" data-hovercard-type="user" data-hovercard-url="/users/zou3519/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self" href="https://github.com/zou3519">@zou3519</a></p>