I think I got it!  This seems super inefficient though.  Can one of you please confirm this is what you meant?  Is this the right way to do this and its whats happening behind the scenes with dataparallel anyway?  I am specifically concerned about everything in Net().  I do know mathematically I wouldn’t have to ‘split’ every iteration if all I was actually doing in the non-minimal example was averaging.  And does this mean if I want to use dataparallel with these buffers I need to manually wrap every layer rather than just calling dataparallel on the network?  Sorry, im just generally getting the feeling I did not do this right.
This is the new output:
$ CUDA_VISIBLE_DEVICES=0,1 python mainMinimalExampleMultiGPU.py 
calling with:                                                                                                                                                                                                                                                                   
calling with:                                                                                                                                                                                                                                                                   
[[[[-0.41571250258624853]], [[0.04142077242016727]], [[0.06950999356652776]], [[0.03586417237630698]], [[0.04916706555685479]], [[0.0479719964996224]], [[0.04534353418505202]], [[0.036737063334092955]], [[0.04897484751185222]], [[0.04072305713577202]]]]                   
[[[[-0.4097441719176553]], [[0.05437682468935652]], [[0.056508038808905765]], [[0.04807834249760404]], [[0.04681839171079582]], [[0.04986861282603519]], [[0.0284716049455237]], [[0.04319707706917308]], [[0.053327966629887764]], [[0.02909731274037328]]]]                   
Conv Average Gradient 0:                                                                                                                                                                                                                                                        
[-0.041571250258624855, 0.004142077242016727, 0.006950999356652776, 0.003586417237630698, 0.004916706555685479, 0.0047971996499622405, 0.004534353418505202, 0.0036737063334092955, 0.004897484751185222, 0.004072305713577202]                                                 
Conv Average Gradient 1:                                                                                                                                                                                                                                                        
[-0.04097441719176553, 0.005437682468935653, 0.005650803880890577, 0.004807834249760404, 0.004681839171079582, 0.004986861282603519, 0.00284716049455237, 0.0043197077069173076, 0.005332796662988777, 0.002909731274037328]                                                    
Conv Average Gradient:                                                                                                                                                                                                                                                          
[-0.04127283372519519, 0.00478987985547619, 0.0063009016187716765, 0.004197125743695551, 0.004799272863382531, 0.00489203046628288, 0.003690756956528786, 0.003996707020163302, 0.005115140707086999, 0.003491018493807265]                                                     
updated Conv Average Gradient 0:                                                                                                                                                                                                                                                
[-0.04127283372519519, 0.00478987985547619, 0.0063009016187716765, 0.004197125743695551, 0.004799272863382531, 0.00489203046628288, 0.003690756956528786, 0.003996707020163302, 0.005115140707086999, 0.003491018493807265]                                                     
updated Conv Average Gradient 1:                                                                                                                                                                                                                                                
[-0.04127283372519519, 0.00478987985547619, 0.0063009016187716765, 0.004197125743695551, 0.004799272863382531, 0.00489203046628288, 0.003690756956528786, 0.003996707020163302, 0.005115140707086999, 0.003491018493807265]                                                     

$ CUDA_VISIBLE_DEVICES=0 python mainMinimalExampleMultiGPU.py                                                                                                                                       
calling with:                                                                                                                                                                                                                                                                   
[[[[-0.41571250258624853]], [[0.04142077242016727]], [[0.06950999356652776]], [[0.03586417237630698]], [[0.04916706555685479]], [[0.0479719964996224]], [[0.04534353418505202]], [[0.036737063334092955]], [[0.04897484751185222]], [[0.04072305713577202]]], [[[-0.4097441719176553]], [[0.05437682468935652]], [[0.056508038808905765]], [[0.04807834249760404]], [[0.04681839171079582]], [[0.04986861282603519]], [[0.0284716049455237]], [[0.04319707706917308]], [[0.053327966629887764]], [[0.02909731274037328]]]]                                      
Conv Average Gradient 0:                                                                                                                                                                                                                                                        
[-0.04127283372519519, 0.00478987985547619, 0.0063009016187716765, 0.004197125743695552, 0.004799272863382531, 0.00489203046628288, 0.003690756956528786, 0.003996707020163302, 0.005115140707087, 0.0034910184938072653]                                                       
Conv Average Gradient:                                                                                                                                                                                                                                                          
[-0.04127283372519519, 0.00478987985547619, 0.0063009016187716765, 0.004197125743695552, 0.004799272863382531, 0.00489203046628288, 0.003690756956528786, 0.003996707020163302, 0.005115140707087, 0.0034910184938072653]                                                       
updated Conv Average Gradient 0:                                                                                                                                                                                                                                                
[-0.04127283372519519, 0.00478987985547619, 0.0063009016187716765, 0.004197125743695552, 0.004799272863382531, 0.00489203046628288, 0.003690756956528786, 0.003996707020163302, 0.005115140707087, 0.0034910184938072653]  ```

and this is the new code for the minimal example:
import torch                                                                                                                                                                                                                                                                    
import torch.nn as nn                                                                                                                                                                                                                                                           
import torch.nn.functional as F                                                                                                                                                                                                                                                 
import torch.optim as optim                                                                                                                                                                                                                                                     
from torchvision import datasets, transforms                                                                                                                                                                                                                                    
import random                                                                                                                                                                                                                                                                   
                                                                                                                                                                                                                                                                                
use_cuda = torch.cuda.is_available()                                                                                                                                                                                                                                            
device = torch.device("cuda" if use_cuda else "cpu")                                                                                                                                                                                                                            
                                                                                                                                                                                                                                                                                
def saveAverageD(grad_out, Values):                                                                                                                                                                                                                                             
    print('calling with:')                                                                                                                                                                                                                                                      
    print(grad_out.tolist())                                                                                                                                                                                                                                                    
    with torch.no_grad():                                                                                                                                                                                                                                                       
        Values.average = Values.average * 0.9 + grad_out.mean((0,2,3)) * 0.1                                                                                                                                                                                                    
                                                                                                                                                                                                                                                                                
class valueTracker(nn.Module):                                                                                                                                                                                                                                                  
    def __init__(self, out_channels):                                                                                                                                                                                                                                           
        super(valueTracker, self).__init__()                                                                                                                                                                                                                                    
        self.register_buffer('average', torch.zeros(out_channels, device=device, dtype=torch.double))                                                                                                                                                                           
                                                                                                                                                                                                                                                                                
class averageSaveConv(nn.Module):                                                                                                                                                                                                                                               
    def __init__(self, startLayer, out_channels):                                                                                                                                                                                                                               
        super(averageSaveConv, self).__init__()                                                                                                                                                                                                                                 
        self.layer = startLayer.double()                                                                                                                                                                                                                                        
        self.out_channels = out_channels                                                                                                                                                                                                                                        
        self.values=valueTracker(self.out_channels)                                                                                                                                                                                                                             
                                                                                                                                                                                                                                                                                
    def forward(self,x):                                                                                                                                                                                                                                                        
        out = self.layer(x)                                                                                                                                                                                                                                                     
        out.register_hook(lambda grad: saveAverageD(grad, self.values))                                                                                                                                                                                                         
        return out                                                                                                                                                                                                                                                              
                                                                                                                                                                                                                                                                                
if(torch.cuda.device_count() == 1):                                                                                                                                                                                                                                             
    device_ids = [0]                                                                                                                                                                                                                                                            
else:                                                                                                                                                                                                                                                                           
    device_ids = [0,1]                                                                                                                                                                                                                                                          
class Net(nn.Module):                                                                                                                                                                                                                                                           
    def __init__(self):
        super(Net, self).__init__()
        self.conv = averageSaveConv(nn.Conv2d(1, 10, 28,1),10).to(device)
        self.replicas = nn.parallel.replicate(self.conv, device_ids)
    def forward(self, x):
        output_device = 0
        inputs = nn.parallel.scatter(x, device_ids)
        replicas = self.replicas[:len(inputs)]
        outputs = nn.parallel.parallel_apply(replicas, inputs)
        x = nn.parallel.gather(outputs, output_device)
        x = torch.flatten(x, 1)
        output = F.log_softmax(x, dim=1)
        return output
    def gather(self):
        self.conv.values.average = nn.parallel.gather([self.replicas[x].values.average for x in range(len(device_ids))], self.conv.values.average.device).reshape(len(device_ids),-1).mean(0)
    def split(self):
        for replica in self.replicas:
            replica.values.average.copy_(self.conv.values.average)

if __name__ == '__main__':
    random_seed = 4
    random.seed(random_seed)
    torch.manual_seed(random_seed)
    torch.cuda.manual_seed(random_seed)    
    model = Net()
    optimizer = optim.SGD(model.parameters(), lr=0.1)
    optimizer.zero_grad()
    data, target = torch.rand((2,1,28,28),dtype=torch.float64).to(device), torch.zeros(2).to(device).long()
    output = model(data)
    loss = F.nll_loss(output, target)
    loss.backward()
    model.gather()
    optimizer.step()
    for id in device_ids:
        print('Conv Average Gradient %d:' % id)
        print(model.replicas[id].values.average.tolist())
    print('Conv Average Gradient:')
    print(model.conv.values.average.tolist())
    model.split()
    for id in device_ids:
        print('updated Conv Average Gradient %d:' % id)
        print(model.replicas[id].values.average.tolist())