Hi, everyone,
I am creating a sign language classifier and i got following problem with its evaluation/usage:
I have two different functions which compute an accuracy of my trained model. One of them (with_dl) computes 94% while the other (without_dl) outputs 6.4%…
Actually my aim is to use the model on my webcam and after some research i ended up with this problem here.
This is my Code: (having cuda available)
import torch
from net import FingeralphabetNet
from torch.utils.data import DataLoader
import torchvision as tv
import pandas as pd
from PIL import Image
import os
fn = "Nets/11.try/15.pth"
nn = FingeralphabetNet()
if torch.cuda.is_available():
    device = torch.device('cuda')
    nn.load_state_dict(torch.load(fn))
    nn = nn.to(device)
else:
    device = torch.device('cpu')
    nn.load_state_dict(torch.load(fn, map_location=device))

abc = [chr(i) for i in range(ord("A"), ord("Z")+1)] + ["SCH", "CH", "NOTHING"]
transforms = tv.transforms.Compose([
    tv.transforms.Resize((120, 160)),
    tv.transforms.ToTensor(),
    tv.transforms.Normalize([.5]*3, [.5]*3),
])

def without_dl(nn, img_dir):
    nn.train(False)
    with torch.no_grad():
        acc = 0
        count = 0
        for ind in range(29):
            for file in os.listdir("{}/{}".format(img_dir, ind)):
                img = Image.open("{}/{}/{}".format(img_dir, ind, file))
                img = transforms(img)
                outputs = nn(img.unsqueeze(0))
                _, prediction = torch.max(outputs, 1)
                if prediction[0] == ind:
                    acc += 1
                count += 1
                if count % 500 == 499:
                    print(count + 1)
    return acc/count

def with_dl(nn, img_dir):
    dataset = tv.datasets.ImageFolder(img_dir, transform=transforms)
    dataloader = DataLoader(dataset, 
                    batch_size=32, shuffle=False, num_workers=8, 
                    pin_memory=True, drop_last=True)
    acc = 0
    count = 0
    nn.train(False)
    with torch.no_grad():
        for X, y in dataloader:
            outputs = nn(X)
            _, prediction = torch.max(outputs, 1)
            y = y.to(device=torch.device('cuda'))
            for i in range(y.shape[0]):
                if y[i] == prediction[i]:
                    acc += 1
                count += 1
                if count % 500 == 499:
                    print(count + 1)
        return acc/count

img_dir = "../dataset/test2"
print("Without:")
without_res = without_dl(nn, img_dir)
print("With:")
with_res = with_dl(nn, img_dir)

print(without_res)
# OUTPUT: 0.06563218390804598
print(with_res)
# OUTPUT: 0.9443035055350554

I am looking forward to your help