I am using pretrained models from torchvision.models.  So, I believe it should do the right thing


I have been using PyTorch python API for the past few months.  Two reasons why I am exploring C++ API:
(a) I was under the impression it will be faster.  But, based on your comments it shouldn’t be that different compared to python API
(b) For deployment in production


In-fact, from my experiments I see C++ API is slower than python API for some models I tried:
C++ API, torch JIT
./test_PyTorch_JIT_Cmp
Network: alexnet, Batch-size: 1, Images/Sec: 633.132
Network: vgg-16, Batch-size: 1, Images/Sec: 208.806
Network: resnet-50, Batch-size: 1, Images/Sec: 128.854
Python
$ python test_PyTorch_Cmp.py
Network: alexnet, Batch-size: 1, Images/Sec: 717.5638130431807
Network: vgg-16, Batch-size: 1, Images/Sec: 236.11391642601006
Network: resnet-50, Batch-size: 1, Images/Sec: 127.30998980136953
For reference (partial code below)
Python
dry_run = 5 # use 5 iterations to warm up
    num_batches = 10

    for i in range(dry_run+num_batches):
        if i == dry_run:
            tic = time.time()
        batch_TensorTemp = torch.autograd.Variable(batch_Tensor)
        if dev == "gpu":
            # move tensor to GPU
            batch_TensorTemp = batch_TensorTemp.cuda()
        output_batch = model(batch_TensorTemp)
        if dev == "gpu":
            # move ouput to cpu
            output_batch = output_batch.data.cpu()
    end = time.time()

    print("Network: {}, Batch-size: {}, Images/Sec: {}\n".format(network, batch_size, (num_batches*batch_size/(end - tic))))

C++
int dry_run = 5;
        int num_batches = 10;

        for(int iter = 0; iter < dry_run + num_batches; ++iter){
          if(iter == dry_run){
            cudaEventRecord(start);
          }
          std::vector<torch::jit::IValue> inputs;
          inputs.push_back((image_as_tensor).cuda());
          torch::Tensor output = module->forward(inputs).toTensor();
          output = output.to(torch::kCPU);    
        }
        cudaEventRecord(stop);
    
        cudaEventSynchronize(stop);
        cudaEventElapsedTime(&milliseconds, start, stop);
        seconds = milliseconds/1000;
        float throughPut = num_batches*batchSize/seconds;
        std::cout << "Network: " << networkName << ", Batch-size: " << batchSize << ", Images/Sec: " << throughPut << "\n";