Hi everyone,
I’m just starting out with NNs and for my first NN written from scratch, I was gonna try to replicate the net in this tutorial NLP From Scratch: Classifying Names with a Character-Level RNN — PyTorch Tutorials 1.7.1 documentation 4, but with a dataset, a dataloader and an actual rnn unit.
The following is my current code:
import os
import torch
import torch.nn as nn
import torch.nn.functional as F
import string
from torch.utils.data import DataLoader, Dataset
import numpy as np

class RNN(nn.Module):

    def __init__(self):
        super(RNN, self).__init__()
        self.rnn = nn.RNN(87, 128, batch_first=True)
        self.linear = nn.Linear(128, 1)

    def forward(self, x, h):
        y, h = self.rnn(x, h)
        y = self.linear(y)
        y = F.softmax(y)
        return y, h


class NameDataset(Dataset):
    def __init__(self, file_path):
        self.names = []
        self.labels = []
        self.all_names = []
        self.all_categories = []
        self.all_letters = []
        self.__readfiles(file_path)

        self.labels_to_index = {}

        for i, label in enumerate(self.all_categories):
            self.labels_to_index[label] = i
        self.labels = [self.labels_to_index[l] for l in self.labels]

        self.letter_to_index = {}
        self.all_letters.sort()

        for i, letter in enumerate(self.all_letters):
            self.letter_to_index[letter] = i

        for name in self.all_names:
            self.names.append(self.__encode_name(name))


    def __len__(self):
        return len(self.names)

    def __getitem__(self, index):
        name = self.names[index]
        label = self.labels[index]
        return name, label

    def __readfiles(self, file_path):
        for filename in os.listdir(file_path):
            label = os.path.splitext(os.path.basename(filename))[0]
            self.all_categories.append(label)
            with open(os.path.join(file_path, filename), "r", encoding='utf-8') as f:
                for name in f.read().strip().split('\n'):
                    self.all_names.append(name)
                    self.labels.append(label)
                    for letter in name:
                        if letter not in self.all_letters:
                            self.all_letters.append(letter)

    def __encode_name(self, name):
        oh_name = np.zeros((len(name), len(self.all_letters)))
        for i, char in enumerate(name):
            oh_char = np.zeros(len(self.all_letters))
            oh_char[self.letter_to_index[char]] = 1
            oh_name[1] = oh_char
        return oh_name

dataset = NameDataset(r"data/names")
train_loader = DataLoader(dataset, batch_size=1, shuffle=True)

rnn = RNN()

criterion = nn.CrossEntropyLoss
optimizer = torch.optim.Adam(rnn.parameters(), lr=0.0001)
epochs = 10000


for epoch in range(1, epochs+1):
    h0 = torch.zeros(1, 1, 128)
    for x, y in train_loader:
        h = h0
        optimizer.zero_grad()
        y_hat, h = rnn(x, h)

        loss = criterion(y_hat, y)
        loss.backward()
        optimizer.step()

    print('%d %d%% %.4f' % (epoch, epoch / epochs * 100, loss))


I realize there is quite a bit to refactor besides my problem, but for now I’m wondering why it’s not working at all.
I’m getting the following exception:
C:\ProgramData\Anaconda3\envs\pythonProject\python.exe H:/PycharmProjects/pythonProject/RNNFromScratch.py
Traceback (most recent call last):
  File "H:/PycharmProjects/pythonProject/RNNFromScratch.py", line 91, in <module>
    y_hat, h = rnn(x, h)
  File "C:\ProgramData\Anaconda3\envs\pythonProject\lib\site-packages\torch\nn\modules\module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "H:/PycharmProjects/pythonProject/RNNFromScratch.py", line 17, in forward
    y, h = self.rnn(x, h)
  File "C:\ProgramData\Anaconda3\envs\pythonProject\lib\site-packages\torch\nn\modules\module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "C:\ProgramData\Anaconda3\envs\pythonProject\lib\site-packages\torch\nn\modules\rnn.py", line 234, in forward
    result = _impl(input, hx, self._flat_weights, self.bias, self.num_layers,
RuntimeError: expected scalar type Double but found Float


Any help would be much appreciated!