wonchulSon:

My purpose is after crop half the cifar10 image like 16x32, make two input up and down data sets, to put in each model input size is 16x32.


Currently you are cropping each image and append each crop to a new tensor.
To do so I would recommend to avoid concatenating the tensor in each iteration and just append the crops to a list and call torch.stack on it, as it will be much faster:
#train
train_up = []
train_down = []

for i in range(train_dataset.data.shape[0]):
    img = ToPIL(train_dataset.data[i])
    up,down = crop_half(img)
    train_up.append(ToTensor(up))
    train_down.append(ToTensor(down))

train_up = torch.stack(train_up)
train_down = torch.stack(train_down)

Once this is done, I would wrap these tensors in a custom Dataset and yield each the up and down crops for each sample:
class MyDataset(Dataset):
    def __init__(self, data_up, data_down, targets):
        self.data_up = data_up
        self.data_down = data_down
        self.targets = targets
        
    def __getitem__(self, index):
        up = self.data_up[index]
        down = self.data_down[index]
        y = self.targets[index]
        
        return up, down, y
    
    def __len__(self):
        return len(self.data_up)

batch_size = 128
train_dataset = MyDataset(train_up, train_down, train_dataset.targets)
train_loader = DataLoader(
    train_dataset,
    batch_size=batch_size,
    shuffle=True
)




 wonchulSon:

After put in these data to VGG16’s first layer, concat two feature map like before crop, and flow it as normal VGG16 model.


Therefore I would just create a head and features layer in your VGG model, pass both crops separately through head, and concatenate the activations afterwards:

class Vgg16(nn.Module):
    def __init__(self, num_classes=10):
        super(Vgg16, self).__init__()
        
        self.head = nn.Sequential(
            nn.Conv2d(3, 64, kernel_size=3, padding=1),
            nn.BatchNorm2d(64),
            nn.ReLU(True),
            nn.Conv2d(64, 64, kernel_size=3, padding=1),
            nn.BatchNorm2d(64),
            nn.ReLU(True),
            nn.MaxPool2d(kernel_size=2, stride=2),
        )
        
        self.features = nn.Sequential(
            nn.Conv2d(64, 128, kernel_size=3, padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(True),
            nn.Conv2d(128, 128, kernel_size=3, padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(True),
            nn.MaxPool2d(kernel_size=2, stride=2),

            nn.Conv2d(128, 256, kernel_size=3, padding=1),
            nn.BatchNorm2d(256),
            nn.ReLU(True),
            nn.Conv2d(256, 256, kernel_size=3, padding=1),
            nn.BatchNorm2d(256),
            nn.ReLU(True),
            nn.MaxPool2d(kernel_size=2, stride=2),

            nn.Conv2d(256, 512, kernel_size=3, padding=1),
            nn.BatchNorm2d(512),
            nn.ReLU(True),
            nn.Conv2d(512, 512, kernel_size=3, padding=1),
            nn.BatchNorm2d(512),
            nn.ReLU(True),
            nn.MaxPool2d(kernel_size=2, stride=2),
        )
        
        self.classifier = nn.Linear(2048, num_classes)

    def forward(self, x_up, x_down):
        x_up = self.head(x_up)
        x_down = self.head(x_down)
        x = torch.cat((x_up, x_down), dim=2)
        
        x = self.features(x)
        x = x.view(x.size(0), -1)
        x = self.classifier(x)
        return x

Note that Variables are deprecated since PyTorch 0.4.0 and also I would recommend using .to(device) instead of .cuda(), so I’ve rewritten your code to the new style:
device = 'cuda' if torch.cuda.is_available() else 'cpu'
model = Vgg16()
model.to(device)
optimizer = optim.SGD(model.parameters(), lr=1e-3, momentum=0.9, weight_decay=1e-4)
criterion = nn.CrossEntropyLoss()

def train(epoch):
    model.train()
    train_loss = 0
    total = 0
    correct = 0
    
    for batch_idx, (data_up, data_down, target) in enumerate(train_loader):
        data_up, data_down, target = data_up.to(device), data_down.to(device), target.to(device)

        optimizer.zero_grad()
        output = model(data_up, data_down)
        loss = criterion(output, target)
        loss.backward()
        optimizer.step()

        train_loss += loss.data
        _, predicted = torch.max(output.data, 1)
        total += target.size(0)
        correct += predicted.eq(target.data).cpu().sum()
        if batch_idx % 10 == 0:
            print('Epoch: {} | Batch: {} |  Loss: ({:.4f}) | Acc: ({:.2f}%) ({}/{})'
                  .format(epoch, batch_idx, train_loss/(batch_idx+1), 100.*correct/total, correct, total))

Your learning rate was a bit high with 0.1 so I lowered it also to 1e-3 so that your model started to learn.