I am creating binary classifier to classify cat and dog images for this below is the network architecture I have for my model.
class CNN(nn.Module):
    def __init__(self):
        super(CNN, self).__init__()
        self.layer1 = nn.Sequential(
            nn.Conv2d(3, 32, kernel_size=5, stride =1, padding=2),
            nn.ReLU(),
            nn.BatchNorm2d(32),
            nn.MaxPool2d(kernel_size = 2, stride=2))
        self.layer2 = nn.Sequential(
            nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.BatchNorm2d(64),
            nn.MaxPool2d(kernel_size = 2, stride=2))
        self.fc1 = nn.Linear(64 * 37 * 37, 1024)
        self.relu = nn.ReLU()
        self.fc2 = nn.Linear(1024, 1)
        self.sigmoid = nn.Sigmoid()

    def forward(self, x):
        out = self.layer1(x)
        out = self.layer2(out)
        out = out.view(out.size(0), -1)
        out = self.relu(self.fc1(out))
        out = self.fc2(out)
        out = self.sigmoid(out)
        return out

I have created custom dataset to read images from folder below is the code for the same.
class CatDogDataset(Dataset):
    def __init__(self,root_dir, transform=transform):
        self.transform = transform
        self.root_dir = root_dir

        self.image_list = glob.glob(self.root_dir+'/*.tif')
        #print(len(self.image_list))


    def __len__(self):
        return len(self.image_list)


    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.to_list()

        img_name = self.image_list[idx]
        image = Image.open(img_name)
        img_label = img_name.split('/')[-1][:3]

        # setting 0 for cat and 1 for dog
        if (img_label == 'cat'):
            label = 0
        else:
            label = 1

        label = np.array(label)
        label = label.reshape(-1)

        if self.transform:
            image = self.transform(image)

        return (image, label)


Now the problem is when I try to run the model for certain epochs (say 3 for this example), It runs successfully till 3 epochs but after that it gives below size error. I am not getting why this error coming after training for certain epochs since if there is size issue then it should occur before even training.
Epoch: 0/3..  Training Loss: 0.606..  Validation Loss: 0.578..  Training Accuracy:  50.012..  Validation Accuracy: 50.215
Epoch: 1/3..  Training Loss: 0.538..  Validation Loss: 0.542..  Training Accuracy:  50.006..  Validation Accuracy: 49.863
Epoch: 2/3..  Training Loss: 0.495..  Validation Loss: 0.516..  Training Accuracy:  50.010..  Validation Accuracy: 50.039
Traceback (most recent call last):
  File "Classifier.py", line 153, in <module>
    outputs = model(images)
  File "/usr/local/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 547, in __call__
    result = self.forward(*input, **kwargs)
  File "Classifier.py", line 61, in forward
    out = self.relu(self.fc1(out))
  File "/usr/local/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 547, in __call__
    result = self.forward(*input, **kwargs)
  File "/usr/local/anaconda3/lib/python3.6/site-packages/torch/nn/modules/linear.py", line 87, in forward
    return F.linear(input, self.weight, self.bias)
  File "/usr/local/anaconda3/lib/python3.6/site-packages/torch/nn/functional.py", line 1369, in linear
    ret = torch.addmm(bias, input, weight.t())
RuntimeError: size mismatch, m1: [64 x 360000], m2: [87616 x 1024] at /opt/conda/conda-bld/pytorch_1565272279342/work/aten/src/THC/generic/THCTensorMathBlas.cu:273

Note:- Since the images are of high resolution due to which it takes lot of time to train, I have resized image to 148 * 148
transform = transforms.Compose([transforms.RandomHorizontalFlip(),
                                transforms.Resize([148,148]),
                                transforms.ToTensor(),
                                transforms.Normalize(mean=[0.5],std=[0.5])
                            ])