Your code is not fully defined, as e.g. filters are missing.
However, it seems to work fine and the output shape corresponds to the passed number of filters as seen here:
class Net(nn.Module):
    def __init__(self, weight):
        super(Net, self).__init__()
        # assumes there are 4 grayscale filters
        self.conv = nn.Conv2d(1, 4, kernel_size=(4, 4), bias=False)
        self.conv.weight = torch.nn.Parameter(weight)

    def forward(self, x):
        conv_x = self.conv(x)
        activated_x = F.relu(conv_x)
        return conv_x, activated_x


weight = torch.randn(4, 1, 4, 4)
model = Net(weight)
x = torch.randn(1, 1, 24, 24)
out = model(x)
print(out[0].shape)
> torch.Size([1, 4, 21, 21])

weight = torch.randn(5, 1, 4, 4)
model = Net(weight)
out = model(x)
print(out[0].shape)
> torch.Size([1, 5, 21, 21])