################################################################################
# Peach - Computational Intelligence for Python
# Jose Alexandre Nalon
#
# This file: tutorial/linear-prediction.py
# Using neural networks to predict number sequences
################################################################################


# A neural network can be used to predict future values of a sequence of
# numbers. Wold's Decomposition Theorem stablishes that any sequence can be
# split in a regular and predictable part and an innovation process (which is
# discrete white noise, and thus impredictable). The goal of this tutorial is
# to show how to use the neural network implementation of Peach to do this.

# We import numpy for arrays and peach for the library. Actually, peach also
# imports the numpy module, but we want numpy in a separate namespace:
from numpy import *
import random
import peach as p

# First, we create the network, with only one layer with only one neuron in it.
# The neuron has many inputs and only one output. The activation function is the
# identity. This kind of neuron is usually known as ADALINE (Adaptive Linear
# Neuron, later Adaptive Linear Element). We use as learning algorithm the LMS
# algorithm.
N = 32
nn = p.FeedForward((N, 1), phi=p.Identity, lrule=p.LMS(0.05))

# The lists below will track the values of the sequence being predicted and of
# the error for plotting.
xlog = [ ]
ylog = [ ]
elog = [ ]

error = 1.
i = 0
x = zeros((N, 1), dtype=float)       # Input is a column-vector.
while i < 2000 and error > 1.e-10:

    # The sequence we will predict is the one generated by a cossinus. The next
    # value of the function is the desired output of the neuron. The neuron will
    # use past values to predict the unknown value. To spice things, we add some
    # gaussian noise (actually, it might help the convergence).
    d = cos(2.*pi/128. * i) + random.gauss(0., 0.01)

    # Here, we activate the network to calculate the prediction.
    y = nn(x)[0, 0]                 # Notice that we need to access the output
    error = abs(d - y)              # as a vector, since that's how the NN work.
    nn.learn(x, d)

    # We store the results to plot later.
    xlog.append(d)
    ylog.append(y)
    elog.append(error)

    # Here, we apply a delay in the sequence by shifting every value one
    # position back. We are using N (=32) samples to make the prediction, but
    # the code here makes no distinction and could be used with any number of
    # coefficients in the prediction. The last value of the sequence is put in
    # the [0] position of the vector.
    x[1:] = x[:-1]
    x[0] = d
    i = i + 1

# If the system has the plot package matplotlib, this tutorial tries to plot
# and save the convergence of synaptic weights and error. The plot is saved in
# the file ``linear-prediction.png``.
try:
    import pylab

    pylab.subplot(211)
    pylab.hold(True)
    pylab.grid(True)
    pylab.plot(array(xlog), 'b--')
    pylab.plot(array(ylog), 'g')
    pylab.plot(array(elog), 'r:')
    pylab.legend([ "$x$", "$y$", "$error$" ])

    pylab.subplot(212)
    pylab.grid(True)
    pylab.stem(arange(0, N), reshape(nn[0].weights, (N,)), "k-", "ko", "k-")
    pylab.xlim([0, N-1])
    pylab.savefig("linear-prediction.png")

except ImportError:
    print "After %d iterations:" % (len(elog),)
    print nn[0].weights