I think I may have it!  I am getting a different error trying to concatenate a tensor of size [431, 2] with my categorical and numeric data.  The tensor consists of predictions from two image models that I want to concatenate with tabular data. The error is:
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-243-186b3ea7dfcf> in <module>
     39         print('Duration:', dt)
     40 
---> 41 tab_run(train_predictions = train_tensor, test_predictions = test_tensor)

<ipython-input-243-186b3ea7dfcf> in tab_run(train_predictions, test_predictions)
     14 
     15             # Forward pass
---> 16             outputs = tab_model(categorical_data, numerical_data, train_predictions)
     17             loss = criterion(outputs, label)
     18             print(f"tab epoch: {epoch}, tab loss: {loss}")

~\AppData\Local\Continuum\anaconda3\envs\torch_env\lib\site-packages\torch\nn\modules\module.py in __call__(self, *input, **kwargs)
    530             result = self._slow_forward(*input, **kwargs)
    531         else:
--> 532             result = self.forward(*input, **kwargs)
    533         for hook in self._forward_hooks.values():
    534             hook_result = hook(self, input, result)

<ipython-input-241-49f06acf2d93> in forward(self, x_categorical, x_numerical, predictions)
     45 #         x = x.float()
     46         print('1 concat', x.size())
---> 47         x4 = torch.cat((x, predictions), dim = 1)
     48         print('X4', x4.size())
     49 

RuntimeError: invalid argument 0: Sizes of tensors must match except in dimension 1. Got 431 and 1 in dimension 0 at C:/w/1/s/tmp_conda_3.7_100118/conda/conda-bld/pytorch_1579082551706/work/aten/src\THC/generic/THCTensorMath.cu:71

These tensors are outside the data loader.  I’m not sure if that matters.
def tab_run(train_predictions, test_predictions):
    train_predictions = train_predictions.to(device)
    test_predictions = test_predictions.to(device)
    for epoch in range(epochs):
        for label, policy, categorical_data, numerical_data in train_loader_tab:
            # move data to GPU
            label = label.long()
            label = label.to(device)
            categorical_data = categorical_data.long()
            categorical_data = categorical_data.to(device)
            numerical_data = numerical_data.to(device)


            # zero the parameter gradients
            optimizer.zero_grad()

            # Forward pass
            outputs = tab_model(categorical_data, numerical_data, train_tensor)
            loss = criterion(outputs, label)
            print(f"tab epoch: {epoch}, tab loss: {loss}")
            # Backward and optimize
            loss.backward()
            optimizer.step()

        for label, policy, categorical_data, numerical_data in test_loader_tab:
            # move data to GPU

            label = label.to(device)
            policy = policy.to(device)
            categorical_data = categorical_data.to(device)
            numerical_data = numerical_data.to(device)
            #predictions come in here


            outputs = tab_model(categorical_data, numerical_data, test_predictions)
            loss = criterion(outputs, label)
            test_loss.append(loss.item())

        #         print(f"test epoch: {epoch}, test loss: {loss}")

        dt = datetime.now() - t0
        print('Duration:', dt)
        
tab_run(train_predictions = train_tensor, test_predictions = test_tensor)