What are some possible causes for the long flat part at the start of training ?
image2338×648 55.7 KB
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        # input_shape = (64, 100, 40, 1)
        ## build the convolutional block
        self.conv1 = nn.Conv2d(1, 32, (1, 2), stride=(1, 2))
        nn.init.xavier_uniform_(self.conv1.weight)
        self.relu1 = nn.LeakyReLU(0.1)
        self.conv2 = nn.Conv2d(32, 32, (4, 1))
        nn.init.xavier_uniform_(self.conv2.weight)
        self.relu2 = nn.LeakyReLU(0.1)
        self.conv3 = nn.Conv2d(32, 32, (4, 1))
        nn.init.xavier_uniform_(self.conv3.weight)
        self.relu3 = nn.LeakyReLU(0.1)

        self.conv4 = nn.Conv2d(32, 32, (1, 2), stride=(1, 2))
        nn.init.xavier_uniform_(self.conv4.weight)
        self.relu4 = nn.LeakyReLU(0.1)
        self.conv5 = nn.Conv2d(32, 32, (4, 1))
        nn.init.xavier_uniform_(self.conv5.weight)
        self.relu5 = nn.LeakyReLU(0.1)
        self.conv6 = nn.Conv2d(32, 32, (4, 1))
        nn.init.xavier_uniform_(self.conv6.weight)
        self.relu6 = nn.LeakyReLU(0.1)

        self.conv7 = nn.Conv2d(32, 32, (1, 10), stride=(1, 2))
        nn.init.xavier_uniform_(self.conv7.weight)
        self.relu7 = nn.LeakyReLU(0.1)
        self.conv8 = nn.Conv2d(32, 32, (4, 1))
        nn.init.xavier_uniform_(self.conv8.weight)
        self.relu8 = nn.LeakyReLU(0.1)
        self.conv9 = nn.Conv2d(32, 32, (4, 1))
        nn.init.xavier_uniform_(self.conv9.weight)
        self.relu9 = nn.LeakyReLU(0.1)

        # Build the inception module
        # convsecond_1 = Conv2D(64, (1, 1), padding="same")(conv_first1)
        self.conv10 = nn.Conv2d(32, 64, (1, 1))
        nn.init.xavier_uniform_(self.conv10.weight)
        self.relu10 = nn.LeakyReLU(0.1)
        self.conv11 = nn.Conv2d(64, 64, (3, 1), padding=(1, 0))
        nn.init.xavier_uniform_(self.conv11.weight)
        self.relu11 = nn.LeakyReLU(0.1)

        self.conv12 = nn.Conv2d(32, 64, (1, 1))
        nn.init.xavier_uniform_(self.conv12.weight)
        self.relu12 = nn.LeakyReLU(0.1)
        self.conv13 = nn.Conv2d(64, 64, (5, 1), padding=(2, 0))
        nn.init.xavier_uniform_(self.conv13.weight)
        self.relu13 = nn.LeakyReLU(0.1)

        self.maxpool1 = nn.MaxPool2d((3, 1), stride=(1, 1), padding=(1, 0))
        self.conv14 = nn.Conv2d(32, 64, (1, 1))
        nn.init.xavier_uniform_(self.conv14.weight)
        self.rnn1 = nn.LSTM(100, 64)
        self.linear1 = nn.Linear(64, 3)
        self.softmax1 = nn.Softmax(dim=1)

        ## testing layers
        self.linear1 = nn.Linear(3200, 3)

    def forward(self, x):

        ### build the convolutional block ###
        x = self.conv1(x)
        x = self.relu1(x)
        x = F.pad(x, (0, 0, 1, 2))  # [left, right, top, bot]
        # x = F.pad(x, (0, 0, 1, 2))  # [left, right, top, bot]
        ### SHOULD I USE 1,2 or 2,1 padding?
        x = self.conv2(x)
        x = self.relu2(x)
        x = F.pad(x, (0, 0, 1, 2))  # [left, right, top, bot]
        x = self.conv3(x)
        x = self.relu3(x)

        x = self.conv4(x)
        x = self.relu4(x)
        x = F.pad(x, (0, 0, 1, 2))  # [left, right, top, bot]
        x = self.conv5(x)
        x = self.relu5(x)
        x = F.pad(x, (0, 0, 1, 2))  # [left, right, top, bot]
        x = self.conv6(x)
        x = self.relu6(x)

        x = self.conv7(x)
        x = self.relu7(x)
        x = F.pad(x, (0, 0, 1, 2))  # [left, right, top, bot]
        x = self.conv8(x)
        x = self.relu8(x)
        x = F.pad(x, (0, 0, 1, 2))  # [left, right, top, bot]
        x = self.conv9(x)
        x = self.relu9(x)

        ## testing layers
        x = x.view(x.shape[0], x.shape[1] * x.shape[2] * x.shape[3])
        x = self.linear1(x)
        x = self.softmax1(x)

        return x

The bigger the learning rate the shorter the flat part and the steeper the curve once it gets going which makes sense.
image2354×766 62 KB
It seems like the weights and biases of the convolutional layers are small on the flat part
image3376×598 212 KB