Another strange behaviour. When I “create” VGG (not simply import it from torchvision), I have totally different kind of error:
---------------------------------------------------------------------------
PureTorchTensorFoundError                 Traceback (most recent call last)
~/miniconda3/envs/pysyft/lib/python3.7/site-packages/syft/frameworks/torch/tensors/interpreters/native.py in handle_func_command(cls, command)
    259             new_args, new_kwargs, new_type, args_type = syft.frameworks.torch.hook_args.hook_function_args(
--> 260                 cmd, args, kwargs, return_args_type=True
    261             )

~/miniconda3/envs/pysyft/lib/python3.7/site-packages/syft/frameworks/torch/hook/hook_args.py in hook_function_args(attr, args, kwargs, return_args_type)
    156         # Try running it
--> 157         new_args = hook_args(args)
    158 

~/miniconda3/envs/pysyft/lib/python3.7/site-packages/syft/frameworks/torch/hook/hook_args.py in <lambda>(x)
    350 
--> 351     return lambda x: f(lambdas, x)
    352 

~/miniconda3/envs/pysyft/lib/python3.7/site-packages/syft/frameworks/torch/hook/hook_args.py in two_fold(lambdas, args, **kwargs)
    515 def two_fold(lambdas, args, **kwargs):
--> 516     return lambdas[0](args[0], **kwargs), lambdas[1](args[1], **kwargs)
    517 

~/miniconda3/envs/pysyft/lib/python3.7/site-packages/syft/frameworks/torch/hook/hook_args.py in <lambda>(i)
    328         # Last if not, rule is probably == 1 so use type to return the right transformation.
--> 329         else lambda i: forward_func[type(i)](i)
    330         for a, r in zip(args, rules)  # And do this for all the args / rules provided

~/miniconda3/envs/pysyft/lib/python3.7/site-packages/syft/frameworks/torch/hook/hook_args.py in <lambda>(i)
     55     if hasattr(i, "child")
---> 56     else (_ for _ in ()).throw(PureTorchTensorFoundError),
     57     torch.nn.Parameter: lambda i: i.child

~/miniconda3/envs/pysyft/lib/python3.7/site-packages/syft/frameworks/torch/hook/hook_args.py in <genexpr>(.0)
     55     if hasattr(i, "child")
---> 56     else (_ for _ in ()).throw(PureTorchTensorFoundError),
     57     torch.nn.Parameter: lambda i: i.child

PureTorchTensorFoundError: 

During handling of the above exception, another exception occurred:

RuntimeError                              Traceback (most recent call last)
<ipython-input-22-ff894affe93a> in <module>
      1 for epoch in range(1, args.epochs + 1):
----> 2     train(args, model_perso_vgg, device, federated_train_loader, optimizer, epoch)

<ipython-input-18-cab4c7130012> in train(args, model, device, train_loader, optimizer, epoch)
      6         optimizer.zero_grad()
      7         output = model(data)
----> 8         loss = F.nll_loss(output, target)
      9         loss.backward()
     10         optimizer.step()

~/miniconda3/envs/pysyft/lib/python3.7/site-packages/syft/frameworks/torch/hook/hook.py in overloaded_func(*args, **kwargs)
    715             cmd_name = f"{attr.__module__}.{attr.__name__}"
    716             command = (cmd_name, None, args, kwargs)
--> 717             response = TorchTensor.handle_func_command(command)
    718             return response
    719 

~/miniconda3/envs/pysyft/lib/python3.7/site-packages/syft/frameworks/torch/tensors/interpreters/native.py in handle_func_command(cls, command)
    268             new_command = (cmd, None, new_args, new_kwargs)
    269             # Send it to the appropriate class and get the response
--> 270             response = new_type.handle_func_command(new_command)
    271             # Put back the wrappers where needed
    272             response = syft.frameworks.torch.hook_args.hook_response(

~/miniconda3/envs/pysyft/lib/python3.7/site-packages/syft/frameworks/torch/pointers/object_pointer.py in handle_func_command(cls, command)
     86 
     87         # Send the command
---> 88         response = owner.send_command(location, command)
     89 
     90         return response

~/miniconda3/envs/pysyft/lib/python3.7/site-packages/syft/workers/base.py in send_command(self, recipient, message, return_ids)
    425 
    426         try:
--> 427             ret_val = self.send_msg(codes.MSGTYPE.CMD, message, location=recipient)
    428         except ResponseSignatureError as e:
    429             ret_val = None

~/miniconda3/envs/pysyft/lib/python3.7/site-packages/syft/workers/base.py in send_msg(self, msg_type, message, location)
    221 
    222         # Step 2: send the message and wait for a response
--> 223         bin_response = self._send_msg(bin_message, location)
    224 
    225         # Step 3: deserialize the response

~/miniconda3/envs/pysyft/lib/python3.7/site-packages/syft/workers/virtual.py in _send_msg(self, message, location)
      8 class VirtualWorker(BaseWorker, FederatedClient):
      9     def _send_msg(self, message: bin, location: BaseWorker) -> bin:
---> 10         return location._recv_msg(message)
     11 
     12     def _recv_msg(self, message: bin) -> bin:

~/miniconda3/envs/pysyft/lib/python3.7/site-packages/syft/workers/virtual.py in _recv_msg(self, message)
     11 
     12     def _recv_msg(self, message: bin) -> bin:
---> 13         return self.recv_msg(message)
     14 
     15     @staticmethod

~/miniconda3/envs/pysyft/lib/python3.7/site-packages/syft/workers/base.py in recv_msg(self, bin_message)
    252             print(f"worker {self} received {sy.codes.code2MSGTYPE[msg_type]} {contents}")
    253         # Step 1: route message to appropriate function
--> 254         response = self._message_router[msg_type](contents)
    255 
    256         # Step 2: Serialize the message to simple python objects

~/miniconda3/envs/pysyft/lib/python3.7/site-packages/syft/workers/base.py in execute_command(self, message)
    383                 command = getattr(command, path)
    384 
--> 385             response = command(*args, **kwargs)
    386 
    387         # some functions don't return anything (such as .backward())

~/miniconda3/envs/pysyft/lib/python3.7/site-packages/syft/frameworks/torch/hook/hook.py in overloaded_func(*args, **kwargs)
    715             cmd_name = f"{attr.__module__}.{attr.__name__}"
    716             command = (cmd_name, None, args, kwargs)
--> 717             response = TorchTensor.handle_func_command(command)
    718             return response
    719 

~/miniconda3/envs/pysyft/lib/python3.7/site-packages/syft/frameworks/torch/tensors/interpreters/native.py in handle_func_command(cls, command)
    285             # in the execute_command function
    286             if isinstance(args, tuple):
--> 287                 response = eval(cmd)(*args, **kwargs)
    288             else:
    289                 response = eval(cmd)(args, **kwargs)

~/miniconda3/envs/pysyft/lib/python3.7/site-packages/torch/nn/functional.py in nll_loss(input, target, weight, size_average, ignore_index, reduce, reduction)
   1869                          .format(input.size(0), target.size(0)))
   1870     if dim == 2:
-> 1871         ret = torch._C._nn.nll_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index)
   1872     elif dim == 4:
   1873         ret = torch._C._nn.nll_loss2d(input, target, weight, _Reduction.get_enum(reduction), ignore_index)

RuntimeError: Expected object of scalar type Long but got scalar type Float for argument #2 'target'


The full code that generates the problem :
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import syft as sy 
from torch.utils.data import Dataset
from torchvision import datasets, transforms
from resizeimage import resizeimage

class Arguments():
    def __init__(self):
        self.batch_size = 64
        self.test_batch_size = 1000
        self.epochs = 10
        self.lr = 0.01
        self.momentum = 0.5
        self.no_cuda = False
        self.seed = 1
        self.log_interval = 10
        self.save_model = False

args = Arguments()

use_cuda = not args.no_cuda and torch.cuda.is_available()

torch.manual_seed(args.seed)

device = torch.device("cuda" if use_cuda else "cpu")

kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}

hook = sy.TorchHook(torch)  # <-- NEW: hook PyTorch ie add extra functionalities to support Federated Learning
bob = sy.VirtualWorker(hook, id="bob")  # <-- NEW: define remote worker bob
alice = sy.VirtualWorker(hook, id="alice")  # <-- NEW: and alice

from scipy.io import loadmat
import matplotlib.pyplot as plt
import numpy as np
from matplotlib.pyplot import imshow
import pandas as pd
from PIL import Image
from sklearn.model_selection import train_test_split
from scipy.misc import imread

%matplotlib inline

frame = pd.read_csv('advanced/filename.csv')

ID_GENDER_MAP = {0: 'male', 1: 'female'}
GENDER_ID_MAP = dict((g, i) for i, g in ID_GENDER_MAP.items())
ID_RACE_MAP = {0: 'white', 1: 'black', 2: 'asian', 3: 'indian', 4: 'others'}
RACE_ID_MAP = dict((r, i) for i, r in ID_RACE_MAP.items())

frame['gender'] = frame['gender'].map(lambda gender: GENDER_ID_MAP[gender])
frame['race'] = frame['race'].map(lambda race: RACE_ID_MAP[race])



#print(frame)
count_row = frame.shape[0]

X = torch.tensor([imread(x, flatten=False, mode='RGB') for x in frame['file']],dtype=torch.float).permute(0, 3, 1, 2)
y = torch.tensor(frame.as_matrix(columns=frame.columns[1:2]), dtype=torch.float)

y = y[:,0]

base=sy.BaseDataset(X[0:100],y[0:100])
base_federated=base.federate((bob, alice))
federated_train_loader = sy.FederatedDataLoader( # <-- this is now a FederatedDataLoader 
                         base_federated,batch_size=args.batch_size)

def train(args, model, device, train_loader, optimizer, epoch):
    model.train()
    for batch_idx, (data, target) in enumerate(federated_train_loader): # <-- now it is a distributed dataset
        model.send(data.location) # <-- NEW: send the model to the right location
        data, target = data.to(device=device), target.to(device=device)
        optimizer.zero_grad()
        output = model(data)
        loss = F.nll_loss(output, target)
        loss.backward()
        optimizer.step()
        model.get() # <-- NEW: get the model back
        if batch_idx % args.log_interval == 0:
            loss = loss.get() # <-- NEW: get the loss back
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * args.batch_size, len(train_loader) * args.batch_size, #batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader), loss.item()))

class VGG(nn.Module):

    def __init__(self, features, num_classes=1000, init_weights=True):
        super(VGG, self).__init__()
        self.features = features
        self.avgpool = nn.AdaptiveAvgPool2d((7, 7))
        self.classifier = nn.Sequential(
            nn.Linear(512 * 7 * 7, 4096),
            nn.ReLU(True),
            nn.Dropout(),
            nn.Linear(4096, 4096),
            nn.ReLU(True),
            nn.Dropout(),
            nn.Linear(4096, num_classes),
        )
        if init_weights:
            self._initialize_weights()

    def forward(self, x):
        x = self.features(x)
        x = self.avgpool(x)
#         print(x.shape)
        x = torch.flatten(x, 1)
        x = self.classifier(x)
        return x

    def _initialize_weights(self):
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
                if m.bias is not None:
                    nn.init.constant_(m.bias, 0)
            elif isinstance(m, nn.BatchNorm2d):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)
            elif isinstance(m, nn.Linear):
                nn.init.normal_(m.weight, 0, 0.01)
                nn.init.constant_(m.bias, 0)


def make_layers(cfg, batch_norm=False):
    layers = []
    in_channels = 3
    for v in cfg:
        if v == 'M':
            layers += [nn.MaxPool2d(kernel_size=2, stride=2)]
        else:
            conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1)
            if batch_norm:
                layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)]
            else:
                layers += [conv2d, nn.ReLU(inplace=True)]
            in_channels = v
    return nn.Sequential(*layers)


cfgs = {
    'A': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M']
}


def _vgg(arch, cfg, batch_norm, pretrained, progress, **kwargs):
    if pretrained:
        kwargs['init_weights'] = False
    model = VGG(make_layers(cfgs[cfg], batch_norm=batch_norm), **kwargs)
    if pretrained:
        state_dict = load_state_dict_from_url(model_urls[arch],
                                              progress=progress)
        model.load_state_dict(state_dict)
    return model


def vgg11(pretrained=False, progress=True, **kwargs):
    r"""VGG 11-layer model (configuration "A") from
    `"Very Deep Convolutional Networks For Large-Scale Image Recognition" <https://arxiv.org/pdf/1409.1556.pdf>'_
    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
        progress (bool): If True, displays a progress bar of the download to stderr
    """
    return _vgg('vgg11', 'A', False, pretrained, progress, **kwargs)

model_perso_vgg = vgg11(pretrained=False)  #.cuda()
model_perso_vgg = model_perso_vgg.to(device=device)
optimizer = optim.SGD(model.parameters(), lr=args.lr)

for epoch in range(1, args.epochs + 1):
    train(args, model_perso_vgg, device, federated_train_loader, optimizer, epoch)

It seems so strange. Maybe you will have any ideas why it is so?