I understand now; you were right about my source of confusion!
In summary, calling torch.load and .load_state_dict() as a one-liner will put the CPU-trained model on the CPU, even if you specify map_location to a GPU device in torch.load.  However, if one splits these operations into two lines, then torch.load will put the weights in the dictionary onto the desired device.  Then, the second line’s load_state_dict will put the model back onto the CPU, where it was trained from.
The following code and output demonstrate this
Illustration code
import torch
import torch.nn as nn
import torch.nn.functional as F

print(f"PyTorch version: {torch.__version__}\n-------")

gpu = torch.device('cuda:0')
PATH = 'todays_cpu_model.pth'

class Net(nn.Module):
    def __init__(self, indim):
        super(Net, self).__init__()
        self.fc1 = nn.Linear(indim, 16)
        self.fc2 = nn.Linear(16, 1)
        
    def forward(self, t):
        t = F.relu(self.fc1(t))
        t = self.fc2(t)
        return t

print("Calling `torch.load` and `model.load_state_dict()` in one line")
model_1 = Net(4)
model_1.load_state_dict(torch.load(PATH, map_location=gpu))
w_1 = next(model_1.parameters()).detach()
print(f"Device of first layer params: {w_1.device}")


print("------\nCalling `torch.load` and `model.load_state_dict()` in TWO DIFFERENT lines")
model_2 = Net(4)
loaded = torch.load(PATH, map_location=gpu)

# access weight in ordered dictionary
w_2a = loaded["fc1.weight"].detach()
print(f"Device of first layer params after `torch.load`: {w_2a.device}")

# now load the state dict and access weight in model object
model_2.load_state_dict(loaded)
w_2b = model_2.fc1.weight.detach()
print(f"Device of first layer params after `.load_state_dict()`: {w_2b.device}")

# Sanity check
# Confirm that the two models are, in fact, equivalent
tests = list()
for kk in model_1.state_dict():
    tests.append(
        torch.equal(model_1.state_dict()[kk].cpu(),model_2.state_dict()[kk].cpu())
    )
    
print(f"----\nTest that the models are identical: {all(tests)}")

The output:
-------
Calling `torch.load` and `model.load_state_dict()` in one line
Device of first layer params: cpu
------
Calling `torch.load` and `model.load_state_dict()` in TWO DIFFERENT lines
Device of first layer params after `torch.load`: cuda:0
Device of first layer params after `.load_state_dict()`: cpu
----
Test that the models are identical: True