Learning a simple pattern with RNN I am trying to make RNN in tensorflow capture a basic pattern in a simple time series in hours. I am trying to solve a bigger problem involving count time series of customer demand.
The simple time series is as follows:

*

*Every 24 hours (1 day) there will be a small integer number either 1 or 2 from a random uniform distirbution.

*In between these 24 hours will be zero values.

*Every 168 hours (7 days) there will be a high integer number (5 or 6 or 7 or 8 or 9) from a random uniform distirbution.


I tried following the code at https://r2rt.com/recurrent-neural-networks-in-tensorflow-i.html using dynamic_rnn.
Is my test data correct? How can I feed the batches of output from previous times step as input to the next time step? I have 5 hyperparamters to play with

batch_size = 8 num_steps = 192 state_size = 5 learning_rate = 0.00001
num_epochs=1

However, after training each time with the same hyperparameters I am getting different results. Each time the training error is very small. The different results seem quite random (local minima probably??). orange is actual, blue is predicted.



Can my test batch start at any point in the sequence? Does the RNN learn the number of zeros inbetween non-zero values? if the test batch starts with a small non-zero number then the RNN should know that it should output 23 zero value steps after this and then after 167 steps output a high non-zero value. if I start my test sequence at 0 then it should wait 23 more zero value steps before outputing a small non-zero value and after 167 steps output a high non-zero value?
or does it learn another pattern? I am not sure if my method of testing is correct?
Is it better to just pass one time step integer value and let the network generate the remaining time steps integer values by passing the current time step output as input to the next time step?
Currently, I just take a random sequence of X generated by the same method for training and check if my output Y is the shifted version of X by 1 time step. Could you please explain?
My code is given below. you can just copy and paste and it should run. Basically, I just generate the data, build the model, train the network and test it.
from data_generator import gen_data
import tensorflow as tf
tf.compat.v1.disable_eager_execution()
import numpy as np
import time
import matplotlib.pyplot as plt

num_classes = 11

batch_size = 8
num_steps = 192
state_size = 5
learning_rate = 0.00001
num_epochs=1

dem = gen_data(len=1576)


def gen_batch(dem, batch_size, num_steps):
    raw_x = dem[:-1]
    raw_y = dem[1:]
    data_length = len(raw_x)
    
    num_of_win = data_length - num_steps - 1 # 1382 windows
    batch_partition_length =  num_of_win // batch_size # 172 batches


    data_x = []
    data_y = []

    j=0
    for i in range(batch_partition_length):
        windows_x = []
        windows_y = []
        k=0
        while(k<batch_size):
            windows_x.append( raw_x[ j:num_steps + j] )
            windows_y.append( raw_y[ j:num_steps + j] )
            j+=1
            k+=1

        data_x.append(np.array(windows_x)) # each batch is stacked horizontally.
        data_y.append(np.array(windows_y))

    for windows_x, windows_y in zip(data_x,data_x):
        x = windows_x
        y = windows_y
        z = x.shape
        z = y.shape
        yield (x, y)




def gen_epoch(num_epochs,batch_size, num_steps):

    for n in range(num_epochs):
        yield gen_batch(dem, batch_size, num_steps)

def reset_graph():
    # if 'sess' in globals() and sess:
    #     sess.close()
    tf.compat.v1.reset_default_graph()

def build_RNN_model(batch_size, num_classes,state_size,num_steps,learning_rate):
    reset_graph()

    x = tf.compat.v1.placeholder(dtype=tf.int32, shape=(batch_size,num_steps))
    y = tf.compat.v1.placeholder(dtype=tf.int32, shape=(batch_size,num_steps))
    init_state = tf.zeros([batch_size, state_size])
    # with tf.compat.v1.variable_scope('rnn_cell'):
    #     W = tf.compat.v1.get_variable('inp_state_w', shape=(num_classes+state_size,state_size),initializer=tf.compat.v1.initializers.glorot_uniform(10) )
    #     b = tf.compat.v1.get_variable('inp_state_b', shape=(state_size),initializer=tf.compat.v1.initializers.constant(0.0) )

    # def rnn_cell(rnn_input,state):
    #     with tf.compat.v1.variable_scope('rnn_cell', reuse=True):
    #         W = tf.compat.v1.get_variable('inp_state_w', shape=(num_classes+state_size,state_size),initializer=tf.compat.v1.initializers.glorot_uniform(10) )
    #         b = tf.compat.v1.get_variable('inp_state_b', shape=(state_size),initializer=tf.compat.v1.initializers.constant(0.0) )
    #     return tf.tanh( tf.matmul( tf.concat([rnn_input,state], axis=1),W) + b )

    

    #cell = tf.compat.v1.nn.rnn_cell.BasicRNNCell(state_size, reuse=True, name='rnn_cell' )


    rnn_inputs = tf.one_hot(x, num_classes)

    cell = tf.compat.v1.nn.rnn_cell.BasicRNNCell(state_size)
    rnn_outputs, final_state = tf.compat.v1.nn.dynamic_rnn(cell, rnn_inputs, initial_state=init_state)

    with tf.compat.v1.variable_scope('output'):
        W = tf.compat.v1.get_variable('out_state_w', shape=(state_size,num_classes),initializer=tf.compat.v1.initializers.glorot_uniform(10) )
        b = tf.compat.v1.get_variable('out_state_b', shape=(num_classes),initializer=tf.compat.v1.initializers.constant(0.0) )


    logits = tf.reshape( tf.compat.v1.matmul(tf.reshape(rnn_outputs, [-1, state_size]), W) + b, [batch_size, num_steps, num_classes])
    predictions = tf.compat.v1.nn.softmax(logits)
    tru_labels = y

    losses = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)
    total_loss = tf.reduce_mean(losses)
    train_step = tf.compat.v1.train.AdagradOptimizer(learning_rate).minimize(total_loss)

    return dict(
        x=x,
        y=y,
        final_state = final_state,
        total_loss = total_loss,
        train_step = train_step,
        init_state = init_state,
        predictions = predictions,
        tru_labels = tru_labels,
        saver = tf.compat.v1.train.Saver()
        
    )

def train_network(g,num_epochs, batch_size,num_steps, dem,save=' '):
    tf.compat.v1.set_random_seed(2345)
    with tf.compat.v1.Session() as sess:
        sess.run(tf.compat.v1.initialize_all_variables())
        training_losses = []
        
        for idx, epoch in enumerate(gen_epoch(num_epochs,batch_size, num_steps)):
            training_loss = 0
            steps=0 # number of batches
            training_state = None
            for X,Y in epoch:
                steps+=1

                feed_dict = {g['x'] : X, g['y'] : Y}
                
                if training_state is not None:
                    feed_dict[g['init_state']] = training_state
         
                training_loss_, training_state, train_step = \
                    sess.run([g['total_loss'], g['final_state'], g['train_step']], feed_dict)

                training_loss+=training_loss_
                print("Average training loss for Epoch", idx, ":", training_loss/steps)
                print('steps',steps)
                training_losses.append(training_loss/steps)

        if isinstance(save, str):
            g['saver'].save(sess, save)
        

        e = gen_batch(dem, batch_size, num_steps)
        e = gen_batch(dem, batch_size, num_steps)

        for X,Y in e:
            tru_labels, predictions = \
                    sess.run([g['tru_labels'], g['predictions']], feed_dict={g['x'] : X, g['y'] : Y, g['init_state'] : training_state})
    

    pred = np.argmax(predictions, axis=2)
    print(pred.shape)
    pred = pred[0]
    print('predictions',pred)
    tru_labels = tru_labels[0]
    print('tru_labels',tru_labels )

    plt.plot(pred)    
    plt.plot(tru_labels)
    plt.show()


    return training_loss






g = build_RNN_model(batch_size, num_classes,state_size,num_steps,learning_rate)
t = time.time()
train_network(g, num_epochs,batch_size,num_steps, dem,save='saver' )
print("It took", time.time() - t, "seconds to train for 3 epochs.")


I have managed to capture this pattern using pytorch DeepAR for two nbinom values (n=9,p=0.5) and (n=4,p=0.75) with period of 8 hours for each. The two signals are interleaved like below.

Run My_demo.ipynb to see the prediction of means of the two frequencies in the following repo.
https://github.com/JosePeeterson/2_freq_nbinom_LSTM/tree/master
