Dear ptrblck,
thank you a lot - you made my day!
The training works now. Here comes the updated source code:
File Alexnet.ipynb
from __future__ import print_function
import zipfile
import os

import torchvision.transforms as transforms
from torchvision import datasets, transforms

import PIL
import numpy as np

import torch
import torch.nn as nn
import torch.nn.functional as F
from torchvision import models
import torch.optim as optim
import datetime

import shutil
import time

import torch.nn.parallel
import torch.backends.cudnn as cudnn
import torch.optim
import torch.utils.data
import torchvision.datasets as datasets


IMG_SIZE = 64
NUM_CLASSES = 43


# Prepare dataset
def prepare_dataset(folder):
    train_zip = folder + '/train_images.zip'
    test_zip = folder + '/test_images.zip'
    if not os.path.exists(train_zip) or not os.path.exists(test_zip):
        raise(RuntimeError("Could not find " + train_zip + " and " + test_zip))

    # extract train_data.zip to train_data
    train_folder = folder + '/train_images'

    if not os.path.isdir(train_folder):
        print(train_folder + ' not found, extracting ' + train_zip)
        zip_ref = zipfile.ZipFile(train_zip, 'r')
        zip_ref.extractall(folder)
        zip_ref.close()

    # extract test_data.zip to test_data
    test_folder = folder + '/test_images'
    if not os.path.isdir(test_folder):
        print(test_folder + ' not found, extracting ' + test_zip)
        zip_ref = zipfile.ZipFile(test_zip, 'r')
        zip_ref.extractall(folder)
        zip_ref.close()

    # make validation_data by using images 00000*, 00001* and 00002* in each class
    val_folder = folder + '/val_images'
    if not os.path.isdir(val_folder):
        print(val_folder + ' not found, making a validation set')
        os.mkdir(val_folder)
        for dirs in os.listdir(train_folder):
            if dirs.startswith('000'):
                os.mkdir(val_folder + '/' + dirs)
                for f in os.listdir(train_folder + '/' + dirs):
                    if f.startswith('00000') or f.startswith('00001') or f.startswith('00002'):
                        # move file to validation folder
                        os.rename(train_folder + '/' + dirs + '/' + f, val_folder + '/' + dirs + '/' + f)


def train(train_loader, model, criterion, optimizer, epoch):
    batch_time = AverageMeter()
    data_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    top5 = AverageMeter()

    # switch to train mode
    model.train()

    end = time.time()
    for i, (input, target) in enumerate(train_loader):
        # measure data loading time
        data_time.update(time.time() - end)

        if torch.cuda.is_available():
            target = target.cuda()
        else:
            target = target.cpu()

        input_var = torch.autograd.Variable(input)
        target_var = torch.autograd.Variable(target)

        # compute output
        output = model(input_var)
        loss = criterion(output, target_var)

        # measure accuracy and record loss
        prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
        losses.update(loss.item(), input.size(0))
        top1.update(prec1.item(), input.size(0))
        top5.update(prec5.item(), input.size(0))

        # compute gradient and do SGD step
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        if i % 10 == 0:

            out_train = ('Epoch: [{0}][{1}/{2}]\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                  'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
                  'Prec@5 {top5.val:.3f} ({top5.avg:.3f})').format(
                   epoch, i, len(train_loader), batch_time=batch_time,
                   data_time=data_time, loss=losses, top1=top1, top5=top5)

            with open("######Progress-training.txt", "a") as myfile:
                myfile.write(str(datetime.datetime.now()) + " -- " + out_train + "\n")

            print(out_train)




def validate(val_loader, model, criterion):
    batch_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    top5 = AverageMeter()

    # switch to evaluate mode
    model.eval()

    end = time.time()
    for i, (input, target) in enumerate(val_loader):

        if torch.cuda.is_available():
            target = target.cuda()
        else:
            target = target.cpu()

        input_var = torch.autograd.Variable(input, volatile=True)
        target_var = torch.autograd.Variable(target, volatile=True)

        # compute output
        output = model(input_var)
        loss = criterion(output, target_var)

        # measure accuracy and record loss
        prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
        losses.update(loss.item(), input.size(0))
        top1.update(prec1.item(), input.size(0))
        top5.update(prec5.item(), input.size(0))

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        if i % 10 == 0:
            out_val = ('Test: [{0}/{1}]\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                  'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
                  'Prec@5 {top5.val:.3f} ({top5.avg:.3f})').format(
                   i, len(val_loader), batch_time=batch_time, loss=losses,
                   top1=top1, top5=top5)

            with open("######Progress-validation.txt", "a") as myfile:
                myfile.write(str(datetime.datetime.now()) + " -- " + out_val + "\n")

            print(out_val)

    out_val_final = (' * Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}'
          .format(top1=top1, top5=top5))

    with open("######Progress-validation-final.txt", "a") as myfile:
        myfile.write(str(datetime.datetime.now()) + " -- " + out_val_final + "\n")

    print(out_val)

    return top1.avg




def save_checkpoint(state, is_best, filename='checkpoint.pth'):
    #torch.save(state, filename)
    torch.save(model.state_dict(), filename)
    if is_best:
        shutil.copyfile(filename, 'model_best.pth')




def adjust_learning_rate(optimizer, epoch):
    """Sets the learning rate to the initial LR decayed by 10 every 30 epochs"""
    lr = 0.1 * (0.1 ** (epoch // 30))
    for param_group in optimizer.param_groups:
        param_group['lr'] = lr




def accuracy(output, target, topk=(1,)):
    """Computes the precision@k for the specified values of k"""
    maxk = max(topk)
    batch_size = target.size(0)

    _, pred = output.topk(maxk, 1, True, True)
    pred = pred.t()
    correct = pred.eq(target.view(1, -1).expand_as(pred))

    res = []
    for k in topk:
        correct_k = correct[:k].view(-1).float().sum(0)
        res.append(correct_k.mul_(100.0 / batch_size))
    return res




class AverageMeter(object):
    """Computes and stores the average and current value"""
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count


device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

prepare_dataset('data')

normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                 std=[0.229, 0.224, 0.225])

# Load training dataset
traindir = 'data/train_images'

train_loader = torch.utils.data.DataLoader(
    datasets.ImageFolder(traindir, transforms.Compose(
        [
            transforms.Resize((IMG_SIZE, IMG_SIZE)),
            transforms.RandomHorizontalFlip(),
            transforms.ToTensor(),
            normalize,
        ]
    )), batch_size=100, shuffle=True, num_workers=1)
    # Model will be trained on 354 batches, each containing 100 images
    # (as specified in the batch_size of your train_loader). 
    # Since the training data is splitted into training and validation, 
    # this number (354) is a bit smaller than 39209//100.


# Load validation dataset    
valdir = 'data/val_images'

val_loader = torch.utils.data.DataLoader(
    datasets.ImageFolder(valdir, transforms.Compose(
        [
            transforms.Resize((IMG_SIZE, IMG_SIZE)),
            transforms.RandomHorizontalFlip(),
            transforms.ToTensor(),
            normalize,
        ]
    )), batch_size=100, shuffle=True, num_workers=1)






from model import AlexNet

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = AlexNet().to(device)

#####################
use_sgd_optimizer = True
#####################

if use_sgd_optimizer == True:
    if torch.cuda.is_available():
        criterion = nn.CrossEntropyLoss().cuda()
    else:
        criterion = nn.CrossEntropyLoss()

    optimizer = torch.optim.SGD(model.parameters(), 
                                0.1, # learning rate
                                momentum=0.9,
                                weight_decay=1e-4)
else:
    optimizer = optim.Adam(model.parameters(), lr=0.001)



#####################
epochs = 51 # trains for 50 epochs
best_prec1 = 0
#####################
for epoch in range(1, epochs):

    progress_epochs = "\n============================\nStart training epoch: " + str(epoch) + "/" + str(epochs-1) + "\n@ " + str(datetime.datetime.now()) + "\n============================"

    with open("######Progress-epochs.txt", "a") as myfile:
        myfile.write(progress_epochs + "\n")

    print(progress_epochs)    


    adjust_learning_rate(optimizer, epoch)

    # train for one epoch
    train(train_loader, model, criterion, optimizer, epoch)

    # evaluate on validation set
    prec1 = validate(val_loader, model, criterion)

    # remember best prec@1 and save checkpoint
    is_best = prec1 > best_prec1
    best_prec1 = max(prec1, best_prec1)
    save_checkpoint({
        'epoch': epoch + 1,
        'arch': "alexnet",
        'state_dict': model.state_dict(),
        'best_prec1': best_prec1,
    }, is_best)


    progress_epochs = "\n=============================\nFinished training epoch: " + str(epoch) + "/" + str(epochs-1) + "\n@ " + str(datetime.datetime.now()) + "\n============================="

    with open("######Progress-epochs.txt", "a") as myfile:
        myfile.write(progress_epochs + "\n")

    print(progress_epochs)    

File model.py
import torch
import torch.nn as nn
import torch.nn.functional as F
from torchvision import models

from constants import IMG_SIZE, NUM_CLASSES

import torch
import torch.nn as nn
import torch.nn.functional as F
from torchvision import models

from constants import IMG_SIZE, NUM_CLASSES

class AlexNet(nn.Module):

    def __init__(self, num_classes=NUM_CLASSES):
        super(AlexNet, self).__init__()
        self.features = nn.Sequential(
            nn.Conv2d(3, 64, kernel_size=11, stride=4, padding=2),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),
            nn.Conv2d(64, 192, kernel_size=5, padding=2),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),
            nn.Conv2d(192, 384, kernel_size=3, padding=1),
            nn.ReLU(inplace=True),
            nn.Conv2d(384, 256, kernel_size=3, padding=1),
            nn.ReLU(inplace=True),
            nn.Conv2d(256, 256, kernel_size=3, padding=1),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),
        )
        self.classifier = nn.Sequential(
            nn.Dropout(),
            #nn.Linear(256 * 6 * 6, 4096),
            nn.Linear(256, 4096),
            nn.ReLU(inplace=True),
            nn.Dropout(),
            nn.Linear(4096, 4096),
            nn.ReLU(inplace=True),
            nn.Linear(4096, NUM_CLASSES),
        )

    def forward(self, x):
        x = self.features(x)
        # x = x.view(x.size(0), 256 * 6 * 6)
        x = x.view(x.size(0), -1)
        x = self.classifier(x)
        return x

File constants.py
NUM_CLASSES = 43
IMG_SIZE = 64

It now comes to the following output:




Epoch: [1][0/354]
Time 2.683 (2.683)
Data 0.123 (0.123)
Loss 3.7641 (3.7641)
Prec@1 0.000 (0.000)
Prec@5 7.000 (7.000)




Epoch: [1][10/354]
Time 2.030 (2.100)
Data 0.003 (0.014)
Loss 3.5542 (3.6974)
Prec@1 5.000 (5.455)
Prec@5 27.000 (24.182)


Epoch: [1][20/354]
Time 2.014 (2.049)
Data 0.003 (0.009)
Loss 3.4839 (3.6009)
Prec@1 1.000 (5.857)
Prec@5 17.000 (24.333)


Epoch: [1][30/354]
Time 2.437 (2.051)
Data 0.003 (0.007)
Loss 3.3573 (3.5383)
Prec@1 5.000 (6.097)
Prec@5 33.000 (25.742)


Epoch: [1][40/354]
Time 1.994 (2.074)
Data 0.003 (0.006)
Loss 3.4271 (3.5123)
Prec@1 9.000 (5.976)
Prec@5 33.000 (26.415)


Epoch: [1][50/354]
Time 2.034 (2.065)
Data 0.003 (0.005)
Loss 3.4206 (3.5004)
Prec@1 6.000 (5.843)
Prec@5 23.000 (26.059)


Epoch: [1][60/354]
Time 1.970 (2.051)
Data 0.003 (0.005)
Loss 3.3889 (3.4947)
Prec@1 4.000 (5.984)
Prec@5 32.000 (26.705)


Epoch: [1][70/354]
Time 1.965 (2.058)
Data 0.003 (0.005)
Loss 3.5263 (3.4855)
Prec@1 3.000 (5.901)
Prec@5 20.000 (26.930)


Epoch: [1][80/354]
Time 2.017 (2.057)
Data 0.002 (0.004)
Loss 3.5213 (3.4840)
Prec@1 5.000 (5.827)
Prec@5 22.000 (27.012)


Epoch: [1][90/354]
Time 2.030 (2.049)
Data 0.002 (0.004)
Loss 3.4811 (3.4783)
Prec@1 7.000 (5.802)
Prec@5 24.000 (27.374)


Epoch: [1][100/354]
Time 2.056 (2.044)
Data 0.003 (0.004)
Loss 3.3517 (3.4692)
Prec@1 7.000 (6.000)
Prec@5 36.000 (28.178)


Epoch: [1][110/354]
Time 1.996 (2.037)
Data 0.003 (0.004)
Loss 3.3756 (3.4605)
Prec@1 6.000 (6.054)
Prec@5 39.000 (28.730)


Epoch: [1][120/354]
Time 1.961 (2.034)
Data 0.003 (0.004)
Loss 3.2252 (3.4559)
Prec@1 7.000 (6.256)
Prec@5 33.000 (29.074)


Epoch: [1][130/354]
Time 1.949 (2.032)
Data 0.003 (0.004)
Loss 3.3313 (3.4484)
Prec@1 5.000 (6.359)
Prec@5 29.000 (29.351)


Epoch: [1][140/354]
Time 2.181 (2.031)
Data 0.002 (0.004)
Loss 3.3257 (3.4427)
Prec@1 4.000 (6.348)
Prec@5 32.000 (29.567)


Epoch: [1][150/354]
Time 2.000 (2.028)
Data 0.002 (0.004)
Loss 3.3333 (3.4370)
Prec@1 3.000 (6.338)
Prec@5 32.000 (29.954)


Epoch: [1][160/354]
Time 2.047 (2.027)
Data 0.001 (0.003)
Loss 3.1608 (3.4282)
Prec@1 14.000 (6.578)
Prec@5 43.000 (30.472)


Epoch: [1][170/354]
Time 2.045 (2.026)
Data 0.003 (0.003)
Loss 3.2424 (3.4190)
Prec@1 8.000 (6.854)
Prec@5 36.000 (30.930)


Epoch: [1][180/354]
Time 1.969 (2.025)
Data 0.003 (0.003)
Loss 3.3580 (3.4087)
Prec@1 7.000 (7.177)
Prec@5 41.000 (31.459)


Epoch: [1][190/354]
Time 1.977 (2.024)
Data 0.003 (0.003)
Loss 3.4424 (3.3933)
Prec@1 10.000 (7.445)
Prec@5 31.000 (32.099)


Epoch: [1][200/354]
Time 2.063 (2.023)
Data 0.002 (0.003)
Loss 2.9468 (3.3781)
Prec@1 17.000 (7.736)
Prec@5 54.000 (32.796)


Epoch: [1][210/354]
Time 2.003 (2.021)
Data 0.003 (0.003)
Loss 2.8463 (3.3585)
Prec@1 14.000 (8.118)
Prec@5 51.000 (33.583)


Epoch: [1][220/354]
Time 1.963 (2.020)
Data 0.004 (0.003)
Loss 3.2116 (3.3477)
Prec@1 9.000 (8.489)
Prec@5 39.000 (33.995)


Epoch: [1][230/354]
Time 2.021 (2.020)
Data 0.002 (0.003)
Loss 3.1663 (3.3409)
Prec@1 12.000 (8.632)
Prec@5 38.000 (34.468)


Epoch: [1][240/354]
Time 1.954 (2.018)
Data 0.003 (0.003)
Loss 3.0815 (3.3294)
Prec@1 14.000 (8.983)
Prec@5 41.000 (34.929)


Epoch: [1][250/354]
Time 1.992 (2.018)
Data 0.003 (0.003)
Loss 3.2567 (3.3148)
Prec@1 9.000 (9.319)
Prec@5 41.000 (35.514)


Epoch: [1][260/354]
Time 1.988 (2.017)
Data 0.003 (0.003)
Loss 3.2490 (3.3170)
Prec@1 10.000 (9.303)
Prec@5 38.000 (35.475)


Epoch: [1][270/354]
Time 2.091 (2.016)
Data 0.004 (0.003)
Loss 3.1162 (3.3074)
Prec@1 22.000 (9.579)
Prec@5 45.000 (35.908)


Epoch: [1][280/354]
Time 1.925 (2.016)
Data 0.003 (0.003)
Loss 2.8177 (3.2990)
Prec@1 23.000 (9.861)
Prec@5 57.000 (36.317)


Epoch: [1][290/354]
Time 2.057 (2.015)
Data 0.003 (0.003)
Loss 2.9302 (3.2892)
Prec@1 19.000 (10.103)
Prec@5 46.000 (36.814)


Epoch: [1][300/354]
Time 2.026 (2.021)
Data 0.002 (0.003)
Loss 2.6849 (3.2711)
Prec@1 22.000 (10.495)
Prec@5 53.000 (37.452)


Epoch: [1][310/354]
Time 2.024 (2.032)
Data 0.003 (0.003)
Loss 2.8718 (3.2517)
Prec@1 21.000 (10.820)
Prec@5 49.000 (38.096)


Epoch: [1][320/354]
Time 1.999 (2.033)
Data 0.002 (0.003)
Loss 3.4280 (3.2563)
Prec@1 13.000 (10.913)
Prec@5 31.000 (38.150)



I do have some last questions:

Does the output above mean, the Alexnet will only be trained on 354 images, whereby the whole trainings dataset is about 39209 images?
I guess that ‘Prec’ is the training accuracy?
If so, why is this value greater than zero? Shouldn’t it be less than zero converging to 1.0 (whereby 1.0 means 100 %)?