Sorry, I discovered that before tracing the model in python I didn’t move it to cuda. If I move it, trace it, and reload it in c++, it correctly runs on the GPU.
Still I can’t copy a cv::Mat into a torch::Tensor and move it to the GPU
at::Tensor output = torch::from_blob(img.data, {1, 3, img.rows, img.cols}, torch::kFloat32).clone();
output.to(torch::Device(torch::kCUDA, 0));
output.to(torch::kCUDA); //trying both ways just in case
assert(output.device().type() == torch::kCUDA); // Assertation failed

After, I tried to copy the data manually like so:
auto options = torch::TensorOptions().dtype(torch::kFloat32).device(torch::kCUDA, 0);
at::Tensor output = torch::rand({1, 3, img.rows, img.cols}, options); // maybe cols rows
auto foo_a = output.accessor<float, 4>(); // define accessor    
for(int i = 0; i < img.rows; i++){
    for(int j = 0; j < img.cols; j++){
        foo_a[0][0][i][j] = img.at<cv::Vec3f>(i, j)[0]; //Segmentation Fault here when accessing foo_a
        foo_a[0][1][i][j] = img.at<cv::Vec3f>(i, j)[1];
        foo_a[0][2][i][j] = img.at<cv::Vec3f>(i, j)[2];
    }
}
assert(output.device().type() == torch::kCUDA);

but I get a segmentation fault (line with the comment).
Any ideas on what could be causing these errors?