Hello.
I’m training CNN models now.
I got the result, but it seems overfitting.(good Accuracy on training set, bad Accuracy on validation set)
Here’s my code. I trained GoogLeNet, but when I trained ResNet34, the result was also overfitting.
Some Ignite code are used to train more easily, but I think this is not main problem.
'''
training-GoogLeNet.py

This file is for training GoogLeNet
'''





# Import modules
import torch
import torch.nn as nn
import torch.optim as optim

from torch.nn import DataParallel
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter

from torchvision import models, datasets, transforms

from ignite.engine import Events, create_supervised_trainer, create_supervised_evaluator
from ignite.metrics import Accuracy, TopKCategoricalAccuracy, Loss
from ignite.handlers import ModelCheckpoint, EarlyStopping
from ignite.contrib.handlers.param_scheduler import LRScheduler

import matplotlib.pyplot as plt

from PIL import Image, ImageFile






# Error preventing
torch.multiprocessing.set_sharing_strategy('file_system')
ImageFile.LOAD_TRUNCATED_IMAGES = True





# CUDA Setting
cuda = True if torch.cuda.is_available() else False

if cuda == True:
    device = torch.device('cuda')
    device_ids = [0, 1, 2, 3]
else:
    device = torch.device('cpu')

    
    
    
        
# TensorBoard Setting
tensorboard_log_directory = 'runs/GoogLeNet training'

writer = SummaryWriter(tensorboard_log_directory)
    
    
    
    

# Model Checkpointer Setting
checkpointer = ModelCheckpoint('models/GoogLeNet', 'GoogLeNet', save_interval = 3, n_saved = 5)
    

    
    
# Training Setting
max_epoch = 70





# Dataset: ImageNet
ImageNet_directory = ~~ (ImageNet directory)

ImageNet_transform = transforms.Compose([
    transforms.Resize(320),
    transforms.CenterCrop(299),
    transforms.ToTensor(),
# ***Image Normalization to avoid overfitting***
    transforms.Normalize(mean = [0.485, 0.456, 0.406], std = [0.229, 0.224, 0.225])
])

if cuda == True:
    batch_size = 100 * len(device_ids)
else:
    batch_size = 10

ImageNet_train_data = datasets.ImageNet(root = ImageNet_directory, download = False, split = 'train', transform = ImageNet_transform)
ImageNet_train_dataloader = DataLoader(ImageNet_train_data, batch_size = batch_size, shuffle = True, num_workers = 16)
    
ImageNet_val_data = datasets.ImageNet(root = ImageNet_directory, download = False, split = 'val', transform = ImageNet_transform)
ImageNet_val_dataloader = DataLoader(ImageNet_val_data, batch_size = batch_size, shuffle = True, num_workers = 16)




# Model: GoogLeNet, not pre-trained(we're gonna train)
GoogLeNet = models.inception_v3(pretrained = False, progress = True, aux_logits = False)

if cuda == True: #Ignite Engine doesn't use data-parallelism(only 1 GPU), so manually distributes the model
    GoogLeNet = DataParallel(GoogLeNet, device_ids = device_ids)

    
    
    
    
# loss function
loss = nn.CrossEntropyLoss()





# optimizer: "Our training used asynchronous stochastic gradient descent with 0.9 momentum"
# ***Also, I used weight_decay(regularization) to avoid overfitting***
optimizer = optim.SGD(GoogLeNet.parameters(), lr = 0.1, momentum = 0.9, weight_decay = 0.0001)





# learning rate scheduler: "fixed learning rate schedule (decreasing the learning rate by 4% every 8 epochs)"
scheduler = LRScheduler(optim.lr_scheduler.StepLR(optimizer, step_size = 8, gamma = 0.04))





# Define training, evaluating Engine
trainer = create_supervised_trainer(GoogLeNet, optimizer, loss, device = device)
evaluator = create_supervised_evaluator(GoogLeNet, metrics = {'Top-1': Accuracy(),
                                                              'Top-5': TopKCategoricalAccuracy(k = 5),
                                                              'Loss': Loss(loss)}, device = device)





# Register Events

## train loss logger
def log_train_loss(trainer):
    writer.add_scalar('Train/Loss', trainer.state.output, global_step = trainer.state.iteration)
    
trainer.add_event_handler(Events.ITERATION_COMPLETED, log_train_loss)    
    


## general metrics logger
validate_every = 3

def log_metrics(trainer):
    if trainer.state.epoch % validate_every == 0:
        # validate training set
        evaluator.run(ImageNet_train_dataloader)
        metrics = evaluator.state.metrics
        writer.add_scalar('Train/Top-1', metrics['Top-1'], global_step = trainer.state.epoch)
        writer.add_scalar('Train/Top-5', metrics['Top-5'], global_step = trainer.state.epoch)
        
        # validate validation set
        evaluator.run(ImageNet_val_dataloader)
        metrics = evaluator.state.metrics
        writer.add_scalar('Validation/Top-1', metrics['Top-1'], global_step = trainer.state.epoch)
        writer.add_scalar('Validation/Top-5', metrics['Top-5'], global_step = trainer.state.epoch)
        writer.add_scalar('Validation/Loss', metrics['Loss'], global_step = trainer.state.epoch)
        
trainer.add_event_handler(Events.EPOCH_COMPLETED, log_metrics)        
        
    
    
## scheduler
trainer.add_event_handler(Events.EPOCH_STARTED, scheduler)
        
        

## model checkpointer        
trainer.add_event_handler(Events.EPOCH_COMPLETED, checkpointer, {'EPOCH': GoogLeNet})        
        
        
## early stopper
def score_function(engine):
    validation_loss = engine.state.metrics['Loss']
    return -validation_loss

earlystopper = EarlyStopping(patience = 5, score_function = score_function, trainer = trainer)

evaluator.add_event_handler(Events.COMPLETED, earlystopper)









# run Engine
if __name__ == '__main__':
    trainer.run(ImageNet_train_dataloader, max_epoch)
    


I tried to avoid overfitting by normalization and regularization, but the result was…

Capture.PNG1161×769 67.9 KB

As you can see, Top-1 and Top-5 result on training set are reasonable(70%, 90% Accuracy each), but on validation set, Top-1 and Top-5 Accuracy are 10%, 14% each, also Loss is increasing.
What’s the problem with my code? Please someone helps me.
Any opinions are welcomed. Thanks.