@ptrblck Sorry for bothering so much. I changed my model to the following -
class timeSeriesConv(nn.Module):
    def __init__(self, channels, seq_length, kernel_size=3, k=2):
        super(timeSeriesConv, self).__init__()
        self.channels = channels
        self.seq_length = seq_length
        self.kernel = kernel_size
        self.criterion = nn.CrossEntropyLoss().cuda()
        self.conv1 = nn.Conv1d(in_channels=self.channels, out_channels=self.channels, kernel_size=self.kernel, stride=1)
        self.depthwiseConv = nn.Conv1d(in_channels=self.channels,
                                       out_channels=k * self.channels,
                                       kernel_size=self.kernel,
                                       stride=1)

        self.fc1 = nn.Linear(in_features=k * self.channels * 309,
                             out_features=(k * self.channels * self.seq_length))

        self.fc2 = nn.Linear(in_features=(k * self.channels * self.seq_length),
                             out_features=2 * k * self.channels * self.seq_length)

        self.fc3 = nn.Linear(in_features=2 *k* self.channels * self.seq_length,
                             out_features=2 *k* self.channels * self.seq_length)

        self.fc4 = nn.Linear(in_features=2 *k* self.channels * self.seq_length,
                             out_features=(k*self.channels * self.seq_length) // 2)

        self.fc5 = nn.Linear(in_features=(self.channels * self.seq_length) // 2,
                             out_features=(k*self.channels * self.seq_length) // 4)
        self.fc6 = nn.Linear(in_features=(self.channels * self.seq_length) // 4, out_features=4)

    def forward(self, X):
        out = self.conv1(X)
        out = nn.functional.relu(out)
        out = self.depthwiseConv(out)
        out = out.view(out.size(0), -1)
        out = self.fc1(out)
        out = nn.functional.relu(out)
        nn.Dropout(0.5)
        out = self.fc2(out)
        out = nn.functional.relu(out)
        nn.Dropout(0.3)
        out = self.fc3(out)
        out = nn.functional.relu(out)
        out = self.fc4(out)
        out = nn.functional.relu(out)
        nn.Dropout(0.25)
        out =self.fc5(out)
        out = nn.functional.relu(out)
        return self.fc6(out)

whereas def train is -
    def train_model(self, data, model, epochs):
        cudnn.benchmark = True
        model.train()
        model.cuda()
        optimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
        loss = []
        min_loss = 100
        for epoch in range(0, epochs + 1):
            avg_loss = 0
            st = time.time()
            for _, (x, y) in enumerate(data):
                optimizer.zero_grad()
                x = x.reshape(1, 22, 313)
                x = x.float().cuda()
                y = y.long().cuda()
                out = model(x)
                print(out)
                loss = self.criterion(out, y)
                loss.backward()
                optimizer.step()
                avg_loss += loss.item()

As you can see,  everything has .cuda() following it so it should be loaded onto the GPU memory. But now after changing the architecture, system RAM gets filed up all the way upto swap and more. Since the dataset has not changed, it cannot be the issue as I ran the model before.
Could you help  please ?