This is the code snippet I used to test:
device_type = torch::kCPU;
torch::Device device(device_type);

Yolo net(device);
/*
auto train_set = VOCDataset(device).map(Collate{});//.map(torch::data::transforms::Stack<>());
auto train_size = train_set.size().value();
auto train_loader = torch::data::make_data_loader<torch::data::samplers::SequentialSampler>
		(std::move(train_set), 
			torch::data::DataLoaderOptions().workers(4).batch_size(train_opts.batch_size));

torch::optim::Adam optimizer(net->parameters(), 
	torch::optim::AdamOptions(train_opts.learn_rate)
	.weight_decay(train_opts.weight_decay));
*/
torch::load(net, train_opts.model_path);

cv::Mat origin_image = cv::imread("./imgs/dog.jpg"), resized_image;
cv::cvtColor(origin_image, resized_image, cv::COLOR_BGR2RGB);
resized_image = resizeKeepAspectRatio(origin_image, cv::Size(416, 416), { 128,128,128 });

cv::Mat img_float;
resized_image.convertTo(img_float, CV_32F, 1.0 / 255);

auto img_tensor = torch::from_blob(img_float.data, { 1, 416, 416, 3 }).to(device);
img_tensor = img_tensor.permute({ 0,3,1,2 }).contiguous();
auto output = net->forward(img_tensor);

std::cout << non_max_suppression(output, 20, 0.1, 0.9);

Here is what I got on server:
   0.0000   66.9942  147.0321  173.0608  340.8369    0.1800    0.9806    0.0000
[ CPUFloatType{1,8} ]

while here is what I got on my laptop:
   0.0000  222.2365  106.0619  242.9099  132.0313    0.1007    0.9915    0.0000
   0.0000  136.0336  192.0411  292.1751  281.3036    0.1112    0.9894    0.0000
   0.0000   66.4557  148.0490  172.6351  341.0895    0.2453    0.9815    0.0000
[ CPUFloatType{3,8} ]