The calculation is correct and my suggestion should fix this error:
class timeSeriesConv(nn.Module):
    def __init__(self, channels, seq_length, kernel_size=3, K=2):
        super(timeSeriesConv, self).__init__()
        self.channels = channels
        self.seq_length = seq_length
        self.kernel = kernel_size

        self.depthwiseConv = nn.Conv1d(in_channels=self.channels,
                                       out_channels=K * self.channels,
                                       kernel_size=self.kernel,
                                       stride=1)
        self.fc1 = nn.Linear(in_features=K*self.channels * (self.seq_length - 2*(self.kernel//2)), out_features=(K*self.channels*self.seq_length))
        self.fc2 = nn.Linear(in_features=(K*self.channels*self.seq_length), out_features=4)

    def forward(self, X):
        out = nn.functional.elu(X)
        out = self.depthwiseConv(out)
        out = out.view(out.size(0), -1)
        out = nn.functional.elu(out)
        out = self.fc1(out)
        out = nn.functional.elu(out)
        return (self.fc2(out))


x = torch.randn(1, 22, 313)
model = timeSeriesConv(22, 313, K = 3)
output = model(x)