<file_sep>#ifndef TYPES_H__
#define TYPES_H__

typedef unsigned int uint_t;
typedef uint_t uint32_t;

struct point_t {
  uint_t x, y, z;
  point_t() {}
  point_t( uint_t& _x, uint_t& _y, uint_t& _z )
  {
    x = _x;
    y = _y;
    z = _z;
  }
};

struct tsize_t {
  uint_t x, y, z;
  tsize_t() { }
  tsize_t( int _x, int _y, int _z )
  {
    x = (uint_t)_x;
    y = (uint_t)_y;
    z = (uint_t)_z;
  }
};

typedef enum {
  LAYER_FC,
  LAYER_CONV,
  LAYER_RELU,
  LAYER_POOL
} layer_type_t;

#endif
<file_sep>#ifndef FC_LAYER_T_H__
#define FC_LAYER_T_H__

#include <vector>
#include <math.h>
#include <boost/format.hpp>
#include "gradient_t.h"
#include "optimization.h"

struct fc_layer_t {
  layer_type_t type;
  tensor_t<float> in;
  tensor_t<float> out;
  tensor_t<float> weights;
  tensor_t<float> prev_updates;
  tensor_t<float> biases;
  tensor_t<float> prev_bias_updates;
  std::vector<float> sum;
  tensor_t<float> dE_dIn; // partial derivative of network error by n-th layer input

  fc_layer_t( tsize_t _in_size, uint_t _out_size ) :
    in( _in_size.x, _in_size.y, _in_size.z ),
    out( _out_size ),
    weights( _out_size, _in_size.x * _in_size.y * _in_size.z ),
    prev_updates( _out_size, _in_size.x * _in_size.y * _in_size.z ),
    biases( _out_size ),
    prev_bias_updates( _out_size ),
    sum( _out_size ),
    dE_dIn( _in_size.x * _in_size.y * _in_size.z )
  {
    type = LAYER_FC;

    init_weights();
  }

  ~fc_layer_t() {
  }

  // FIXME
  void init_weight_rand_small( float& weight, uint_t maxval ) {
    // random small values
    weight = 2.19722f / maxval * rand() / float( RAND_MAX ); // random small values 
  }

  void init_weight_rand(float& weight) {
    // random values -1..1
    //
    weight = float(rand()) / (float(RAND_MAX) + 1.0);
    /*
    int evn = rand() % 2;
    if (evn == 0) {
      weight = float(rand()) / (float(RAND_MAX) + 1.0);
    } else {
      weight = -1 * float(rand()) / (float(RAND_MAX) + 1.0);
    }
    */
  }

  void init_weights() {
    srand(time(NULL));
    for ( uint_t i = 0 ; i < out.size.x; i++ ) {
      init_weight_rand( biases( i ) );
      for ( uint_t j = 0 ; j < ( in.get_size() ); j++ ) {
        // init_weight_rand_small( weights( i, j ), in.size.x * in.size.y * in.size.z ) ;
        init_weight_rand( weights( i, j ) ) ;
        prev_updates( i, j ) = 0;
        prev_bias_updates( i ) = 0;
      }
    }
  }

  void print_weights()
  {
    for ( uint_t j = 0; j < weights.size.y; j++ ) {
      std::cout << "output neuron " << j << " : ";
      for ( uint_t i = 0; i < weights.size.x; i++ ) {
          std::cout << boost::format("%7.5f") % weights( i, j, 0 ) << "  ";
      }
      std::cout << std::endl;
    }
  }

  uint_t map( point_t p )
  {
    return in.size.z * in.size.y* p.x + 
           in.size.z * p.y + 
           p.z;
           /*
           return p.z * (in.size.x * in.size.y) +
             p.y * (in.size.x) +
             p.x;
           */
  }

  float sig( float x )
  {
    float sig = 1.0f / (1.0f + exp( -x ));
    return sig;
  }

  float activator_function( float x )
  {
    //return tanhf( x );
    return sig(x);
  }

  /*
  float activator_derivative( float x )
  {
    //float t = tanhf( x );
    //return 1 - t * t;
    return sig( x ) * ( 1 - sig( x ) ); // (sig)' = sig * (1 - sig)
  }
  */

  void activate( const tensor_t<float>& _in )
  {
    assert( (_in.size.x == in.size.x) && (_in.size.y == in.size.y) && (_in.size.z == in.size.z) );

    in.copy_from(_in);
    activate();
  }

  void activate()
  {
    for ( uint_t m = 0; m < out.size.x; m++ ) {
      float input_sum = 0;

      for ( uint_t i = 0; i < in.size.x; i++ ) {
        for ( uint_t j = 0; j < in.size.y; j++ ) {
          for ( uint_t z = 0; z < in.size.z; z++ ) {
            int n = map( point_t( i, j, z ) );
            input_sum += in( i, j, z ) * weights( m, n );
          }
        }
      }
      input_sum += biases( m );

      sum[m] = input_sum;
      out( m ) = activator_function( input_sum );
    }
  }

  void update( tensor_t<float>& dE_dIn_next)
  {
    // calculation of deltas vector
    tensor_t<float> deltas( out.get_size() );
    for ( uint_t m = 0; m < out.get_size(); m++ ) {
      deltas( m ) = dE_dIn_next( m ) * out( m ) * ( 1 - out( m ) );
    }

    // weights update
    for ( uint_t m = 0; m < out.get_size(); m++ ) {
      float bias_update = deltas( m ) *  LEARNING_RATE + MOMENTUM * prev_bias_updates( m );
      biases( m ) -= bias_update;
      prev_bias_updates( m ) = bias_update;
      for ( uint_t n = 0; n < in.get_size(); n++ ) {
        float update = deltas( m ) * in( n ) * LEARNING_RATE + MOMENTUM * prev_updates( m, n );
        weights( m, n ) -= update;
        prev_updates( m, n ) = update;

        // or ...
        // weights( m, n ) -= deltas( m ) * in( n );
        // biases( m ) -= deltas( m ); // ( deltas( m ) * 1 )
      }
    }

    // calculation of dE_dIn vector
    for ( uint_t n = 0; n < in.get_size(); n++ ) {
      dE_dIn( n ) = 0;
      for ( uint_t m = 0; m < out.get_size(); m++ ) {
        dE_dIn( n ) += deltas( m ) * weights( m, n ); // to be used in the previous layer
      }
    }
    
  }

};

#endif
<file_sep>#ifndef TENSOR_T_H__
#define TENSOR_T_H__

#include <vector>
#include <assert.h>
#include <string.h>
#include "types.h"

template <typename T>
struct tensor_t {
  tsize_t size;
  T* data;
  
  tensor_t( uint_t _x, uint_t _y, uint_t _z ) {
    assert( _x > 0  && _y > 0 && _z > 0 );
    data = new T[_x * _y * _z];
    size.x = _x;
    size.y = _y;
    size.z = _z;
  }

  tensor_t( uint_t _x, uint_t _y ) {
    assert( _x > 0  && _y > 0 );
    data = new T[_x * _y]; // data = new T[_x * _y * 1]
    size.x = _x;
    size.y = _y;
    size.z = 1;
  }

  tensor_t( uint_t _x ) {
    assert( _x > 0 );
    data = new T[_x]; // data = new T[_x * 1 * 1]
    size.x = _x;
    size.y = 1;
    size.z = 1;
  }

  tensor_t( const tensor_t<T>& other ) {
    data = new T[other.size.x * other.size.y * other.size.z];
    memcpy( 
            this->data,
            other.data,
            other.size.x * other.size.y * other.size.z * sizeof( T )
    );
    this->size = other.size;
  }

  T& get( uint_t _x, uint_t _y, uint_t _z )
  {
    assert( _x < size.x  && _y < size.y && _z < size.z );

    return data[ 
                 size.z * size.y* _x + 
                 size.z * _y + 
                 _z
                 /*
                 _z * (size.x * size.y) +
                 _y * (size.x) +
                 _x
                */
            ];
  }

  T& operator()( uint_t _x, uint_t _y, uint_t _z )
  {
    return get( _x, _y, _z );
  }

  T& operator()( uint_t _x, uint_t _y )
  {
    return get( _x, _y, 0 );
  }

  T& operator()( uint_t _x )
  {
    return get( _x, 0, 0 );
  }

  void copy_from( const std::vector<std::vector<std::vector<T> > >& _data )
  {
    uint_t _size_x = _data.size();
    uint_t _size_y = _data[0].size();
    uint_t _size_z = _data[0][0].size();
    assert( size.x == _size_x && size.y == _size_y && size.z == _size_z);

    for (uint_t i = 0; i < _size_x; i++) {
      for (uint_t j = 0; j < _size_y; j++) {
        for (uint_t k = 0; k < _size_z; k++) {
          this->get(i, j, k) = _data[i][j][k];
          /*
          this->get(i, j, k) = _data[k][j][i];
          */
        }
      }
    }
  }

  void copy_from( const tensor_t<T>& other ) {
    assert( (other.size.x == size.x) && (other.size.y == size.y) && (other.size.z == size.z) );
    memcpy( 
            this->data,
            other.data,
            other.size.x * other.size.y * other.size.z * sizeof( T )
    );
  }

  tensor_t<T> operator+( tensor_t<T>& other )
  {
    assert(this->size.x == other.size.x);
    assert(this->size.y == other.size.y);
    assert(this->size.z == other.size.z);
    tensor_t<T> clone(*this);
    for (int i = 0; i < clone.size.x; i++) {
      clone.data[i] += other.data[i];
    }
    return clone;
  }

  tensor_t<T> operator-( tensor_t<T>& other )
  {
    assert(this->size.x == other.size.x);
    assert(this->size.y == other.size.y);
    assert(this->size.z == other.size.z);
    tensor_t<T> clone(*this);
    for (uint_t i = 0; i < clone.size.x; i++) {
      clone.data[i] -= other.data[i];
    }
    return clone;
  }

  uint_t get_size() {
    return size.x * size.y * size.z;
  }

  ~tensor_t()
  {
    delete[] data;
  }
};


#endif
<file_sep>SOURCES = tensor_t.test.cpp fc_layer_t.test.cpp
TESTS = $(SOURCES:.cpp=)

DEBUG = 1

#---------------------------------
OBJECTS = $(SOURCES:.cpp=.o)

LDFLAGS += -L/usr/lib
LDFLAGS += -L/lib/boost/lib
LDFLAGS += -lboost_unit_test_framework.dll
CPPFLAGS += -Wall
# CPPFLAGS += ...
CPPFLAGS += -I/usr/local/include

ifdef DEBUG
CPPFLAGS += -g
endif

all: $(TESTS)

$(TESTS): $(OBJECTS)
	echo TESTS = $(TESTS);
	$(foreach var, $(OBJECTS), $(CXX) -o $(var:.o=) $(var) $(LDFLAGS);)

$(OBJECTS): $(SOURCES)
	$(foreach var, $(SOURCES), $(CXX) -c $(CPPFLAGS) $(var) -o $(var:.cpp=.o);)

clean: 
	rm -f *.o

build:
	$(MAKE) clean
	$(MAKE)
<file_sep>#include <iostream>
#include "tensor_t.h"

#define BOOST_TEST_DYN_LINK
#define BOOST_TEST_MODULE test_tensor_t

#include <boost/test/unit_test.hpp>

BOOST_AUTO_TEST_CASE( test_tensor_t_constructor )
{
  tensor_t<float> t1(1, 2, 3);
  BOOST_CHECK(t1.size.x == 1);
  BOOST_CHECK(t1.size.y == 2);
  BOOST_CHECK(t1.size.z == 3);

  tensor_t<float> t2(t1);
  BOOST_CHECK(t1.size.x == 1);
  BOOST_CHECK(t1.size.y == 2);
  BOOST_CHECK(t1.size.z == 3);
}

BOOST_AUTO_TEST_CASE( test_tensor_t_get )
{
  tensor_t<float> t(2, 2, 3);
  t.data[0] = 0;
  t.data[1] = 1.1f;
  t.data[2] = 2.2f;
  t.data[3] = 3.3f;
  t.data[4] = 4.4f;
  t.data[5] = 5.5f;
  t.data[6] = 6.6f;
  t.data[7] = 7.7f;
  t.data[8] = 8.8f;
  t.data[9] = 9.9f;
  t.data[10] = 10.10f;
  t.data[11] = 11.11f;

  BOOST_CHECK(t.get(0, 0, 0) == 0);
  BOOST_CHECK(t.get(1, 0, 1) == 7.7f);
  BOOST_CHECK(t.get(0, 1, 1) == 4.4f);
  BOOST_CHECK(t.get(1, 1, 2) == 11.11f);
}

BOOST_AUTO_TEST_CASE( test_tensor_t_operatot_get )
{
  tensor_t<float> t(2, 2, 3);
  t.data[0] = 0;
  t.data[1] = 1.1f;
  t.data[2] = 2.2f;
  t.data[3] = 3.3f;
  t.data[4] = 4.4f;
  t.data[5] = 5.5f;
  t.data[6] = 6.6f;
  t.data[7] = 7.7f;
  t.data[8] = 8.8f;
  t.data[9] = 9.9f;
  t.data[10] = 10.10f;
  t.data[11] = 11.11f;

  BOOST_CHECK(t(0, 0, 0) == 0);
  BOOST_CHECK(t(1, 0, 1) == 7.7f);
  BOOST_CHECK(t(0, 1, 1) == 4.4f);
  BOOST_CHECK(t(1, 1, 2) == 11.11f);
}

BOOST_AUTO_TEST_CASE( test_tensor_t_copy_from )
{
  std::vector<std::vector<std::vector<float> > > data;

  std::vector<std::vector<float> > y;

  std::vector<float> z;
  z.push_back(0);
  z.push_back(1.1f);
  z.push_back(2.2f);
  y.push_back(z);

  z.clear();
  z.push_back(3.3f);
  z.push_back(4.4f);
  z.push_back(5.5f);
  y.push_back(z);

  data.push_back(y);
  y.clear();

  z.clear();
  z.push_back(6.6f);
  z.push_back(7.7f);
  z.push_back(8.8f);
  y.push_back(z);

  z.clear();
  z.push_back(9.9f);
  z.push_back(10.10f);
  z.push_back(11.11f);
  y.push_back(z);

  data.push_back(y);

  tensor_t<float> t(2, 2, 3);
  t.copy_from(data);

  BOOST_CHECK(t(0, 0, 0) == 0);
  BOOST_CHECK(t(1, 0, 1) == 7.7f);
  BOOST_CHECK(t(0, 1, 1) == 4.4f);
  BOOST_CHECK(t(1, 1, 2) == 11.11f);
}

<file_sep>#include <iostream>
#include <vector>
#include <cstdlib>
#include "types.h"
#include "tensor_t.h"
#include "fc_layer_t.h"

#define EPOCHS_NUM 5000

struct case_t
{
  struct tensor_t<float> data;
  struct tensor_t<float> out;
  case_t() :
    data( 2 ),
    out( 1 )
  {}
};

void add_case(std::vector<case_t> &cases, float x1, float x2, float y)
{
  case_t c;
  c.data( 0 ) = x1;
  c.data( 1 ) = x2;
  c.out( 0 ) = y;
  cases.push_back( c );
}

void add_cases(std::vector<case_t> &cases) {
  add_case(cases, 0, 0, 0);
  add_case(cases, 0, 1, 1);
  add_case(cases, 1, 0, 1);
  add_case(cases, 1, 1, 0);
}

void train ( std::vector<fc_layer_t*>& layers, std::vector<case_t> cases ) // FIXME: const cases
{

  for (uint_t i = 0; i < EPOCHS_NUM; i++) {

    std::cout << std::endl << std::endl << "EPOCH " << i << std::endl;

    for ( uint_t j = 0; j < cases.size(); j++ ) {

      std::cout << std::endl << "case " << j << "  ";

      std::cout << "( " << cases[j].data(0) << ", " << cases[j].data(1) << " ) ->" << cases[j].out(0) << " | ";

      struct tensor_t<float> expected = cases[j].out;

      // forward pass
      for ( uint_t k = 0; k < layers.size(); k++ ) {

        // activate layer k
        if ( k == 0 ) {
          layers[k]->activate(cases[j].data);
        } else {
          layers[k]->activate(layers[k-1]->out);
        }
      }

      std::cout << layers.back()->out(0) << std::endl;

      // backpropagation
      tensor_t<float> grads = layers.back()->out - expected;

      for ( int i = (int)(layers.size() - 1); i >= 0; i-- )
      {
        if ( i == (int)layers.size() - 1 ) {
          layers[i]->update( grads );
        } else {
          layers[i]->update( layers[i + 1]->dE_dIn);
        }
      }

    } // cases loop
  } // epochs loop
}

int main()
{
  std::vector<fc_layer_t*> nn;
  fc_layer_t* layer1 = new fc_layer_t( tsize_t( 2, 1, 1 ), 5);
  fc_layer_t* layer2 = new fc_layer_t( tsize_t( 5, 1, 1 ), 1);
  nn.push_back(layer1);
  nn.push_back(layer2);

  std::vector<case_t> cases;
  add_cases(cases);

  train( nn, cases );

  delete layer1;
  delete layer2;
}
<file_sep>#ifndef GRADIENT_T_H__
#define GRADIENT_T_H__

struct gradient_t
{
  float grad;
  float oldgrad;
  gradient_t()
  {
    grad = 0;
    oldgrad = 0;
  }
};

#endif
<file_sep>#ifndef OPTIMIZATION_H__
#define OPTIMIZATION_H__

#include "gradient_t.h"

// #define LEARNING_RATE 0.01
// #define MOMENTUM 0.6
// #define WEIGHT_DECAY 0.001

#define LEARNING_RATE 0.5
#define MOMENTUM 0.6
#define WEIGHT_DECAY 0.001


#endif
<file_sep>#include <iostream>
#include "types.h"
#include "tensor_t.h"
#include "fc_layer_t.h"

#define BOOST_TEST_DYN_LINK
#define BOOST_TEST_MODULE test_tensor_t

#include <boost/test/unit_test.hpp>

BOOST_AUTO_TEST_CASE( test_fc_layer_t_constructor )
{
  tsize_t tsize = {2, 2, 3};
  fc_layer_t l(tsize, 3);

  tensor_t<float> in_t(2, 2, 3);
  in_t.data[0] = 0;
  in_t.data[1] = 1.1f;
  in_t.data[2] = 2.2f;
  in_t.data[3] = 3.3f;
  in_t.data[4] = 4.4f;
  in_t.data[5] = 5.5f;
  in_t.data[6] = 6.6f;
  in_t.data[7] = 7.7f;
  in_t.data[8] = 8.8f;
  in_t.data[9] = 9.9f;
  in_t.data[10] = 10.10f;
  in_t.data[11] = 11.11f;

  for( uint_t i = 0; i < (l.in.size.x * l.in.size.y * l.in.size.z); i++ ) {
    l.weights( i, 0, 0 ) = 0.1;
    l.weights( i, 1, 0 ) = 0.5;
    l.weights( i, 2, 0 ) = 1;
  }

  l.print_weights();

  l.activate(in_t);

  // std::cout << "in_t.data[6] = " << in_t.data[6] << std::endl;
  // std::cout << "weights( 6, 0, 0 ) = " << (*l.weights)( 6, 0, 0 ) << std::endl;

  // FIXME
  // BOOST_CHECK((*l.out)( 2, 0, 0 ) == 66 );
}

