The output from the code below is:
pytorch answer = [1.14435797]
pytorch takes 12.410856008529663 s
my answer = [1.14435797]
my imlementation takes 1.566734790802002 s

So why is the pytorch forward pass 8x slower than numpy here?
import IPython as ipy
import numpy as np
import torch
import time

class Net(torch.nn.Module):
    def __init__(self, input_size, layer_width):
        super(Net, self).__init__()
        self.input_size = input_size
        output_size = 1
        self.fc1 = torch.nn.Linear(input_size, layer_width)
        self.fc2 = torch.nn.Linear(layer_width, layer_width)
        self.fc3 = torch.nn.Linear(layer_width, output_size)

    def forward(self, x):
        x = torch.nn.functional.relu(self.fc1(x))
        x = torch.nn.functional.relu(self.fc2(x))
        x = torch.exp(self.fc3(x))
        return x

    def get_input_size(self):
        return self.input_size

def main():
    torch.set_default_dtype(torch.float64)
    torch.set_grad_enabled(False)
    input_size = 20
    layer_width = 20
    net = Net(input_size, layer_width)

    N_evals = 100000

    pytorch_x = torch.ones(input_size)
    pytorch_answer = net(pytorch_x).detach().numpy()
    print("pytorch answer = " + str(pytorch_answer))
    t0 = time.time()
    pytorch_x = torch.ones(input_size)
    for i in range(N_evals):
        pytorch_answer = net(pytorch_x)
    t1 = time.time()
    print("pytorch takes " + str(t1 - t0) + " s")

    x = pytorch_x.detach().numpy()
    w1 = net.fc1.weight.detach().numpy()
    w2 = net.fc2.weight.detach().numpy()
    w3 = net.fc3.weight.detach().numpy()
    b1 = net.fc1.bias.detach().numpy()
    b2 = net.fc2.bias.detach().numpy()
    b3 = net.fc3.bias.detach().numpy()

    y = np.matmul(w1, x) + b1
    y = y * (y > 0)
    y = np.matmul(w2, y) + b2
    y = y * (y > 0)
    my_answer = np.exp(np.matmul(w3, y) + b3)
    print("my answer = " + str(my_answer))
    
    t0 = time.time()
    x = np.ones(input_size)
    for i in range(N_evals):
        y = np.matmul(w1, x) + b1
        y = y * (y > 0)
        y = np.matmul(w2, y) + b2
        y = y * (y > 0)
        my_answer = np.exp(np.matmul(w3, y) + b3)
    t1 = time.time()
    print("my imlementation takes " + str(t1 - t0) + " s")

if __name__ == "__main__":
    main()