After some reading I came up with these two models @Kzyh @Bhack  – still unsure if this is the right implementation of the architecture described above(image shared in replies and description in the intro) way to go about things.
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.python.ops.gen_array_ops import reshape

model = keras.Sequential(
        [
        keras.Input(shape=(128, 800, 2)),
        layers.Conv2D(16, (3, 3), activation='relu', padding="same"),
        layers.BatchNormalization(),
        layers.MaxPooling2D(pool_size=(5, 4)),
        layers.Conv2D(8, kernel_size=(4,1),strides=(4,1), activation='relu', padding="same"),
        layers.BatchNormalization(),
        layers.MaxPooling2D(pool_size=(6, 5)),
        layers.Reshape((40,8)),
        layers.Bidirectional(layers.LSTM(8, return_sequences=True)),
        layers.Dense(7),
        ]
    )

model.summary()

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers

model = keras.Sequential(
    [
     keras.Input(shape=(128, 800, 2)),
      layers.Conv2D(16, (3, 3), activation='relu', padding='same'),
      layers.BatchNormalization(),
      layers.MaxPooling2D(pool_size=(5, 4)),
      layers.Conv2D(8, (4, 1), activation='relu', strides=(4,1), padding='same'),
      layers.BatchNormalization(),
      layers.MaxPooling2D(pool_size=(6, 5)),
      layers.TimeDistributed(layers.Flatten()),
      layers.Bidirectional(layers.LSTM(8, return_sequences=True)),
      layers.Dense(7),
    ]
)

model.summary()

model.compile(
    loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),
    optimizer = keras.optimizers.Adadelta(),
    metrics = ["accuracy"],
)

If some one can look at the code and architecture described above and let me know in case I am missing something that will be really helpful.