I am getting the same error:
def __init__(self, input_dim=100, conv_input=3, conv_num=2, conv_dim=16, kernel_size=11, hidden_num=3, hidden_dim=200, output_dim=7):
      # Building the network from here
        super(ConvNet, self).__init__()

      # Class attributes
        self.input_dim = input_dim
        self.conv_input = conv_input
        self.conv_num = conv_num
        self.conv_dim = conv_dim
        self.kernel_size = kernel_size
        self.hidden_num = hidden_num
        self.hidden_dim = hidden_dim
        self.pool = nn.MaxPool2d(2)
        self.output_conv = input_dim

      # Output dimension after convolution
        for i in range(conv_num):
            self.output_conv = (self.output_conv-(self.kernel_size-1)-1)/1+1
            self.output_conv = math.floor(self.output_conv/2)
        self.output_conv = self.output_conv*self.output_conv*self.conv_dim
        print("Output:", self.output_conv)

      # Convolutional layers
        convolutional = [nn.Conv2d(conv_input, conv_dim, kernel_size) if i==0 else nn.Conv2d(conv_dim, conv_dim, kernel_size) for i in range(conv_num)]
        self.convolution = nn.ModuleList(convolutional)
        kernels = [kernel_size]*conv_num
        paddings = [0]*conv_num

      # Hidden layers
        hidden = [nn.Linear(self.output_conv, hidden_dim) if i==0 else nn.Linear(hidden_dim, hidden_dim) for i in range(hidden_num)]
        self.linears = nn.ModuleList(hidden)

      # Output layer
        self.ol = nn.Linear(hidden_dim, output_dim)

    def forward(self, X):
        for i, cv in enumerate(self.convolution):
            X = self.convolution[i](X)
            print(X.shape)
            X = F.relu(X)
            X = self.pool(X)
            print(X.shape)
        X = X.view(-1, self.num_flat_features(X))
        print(X.shape)
        for j, f in enumerate(self.linears):
            print(j)
            X = self.linears[i](X)
            X = F.relu(X)
        return F.softmax(self.ol(X), dim=-1)
    
    def num_flat_features(self, x):
        size = x.size()[1:]  
        num_features = 1
        for s in size:
            num_features *= s
        return num_features

Manual implementation actually works so I do not know what could I be doing wrong. My input images are 100x100. The error reads:

RuntimeError: mat1 and mat2 shapes cannot be multiplied (128x4624 and 200x200)

Why is the shape suddenly changed from the batch size to 200?