<|start_filename|>RNNSharp/RNNDecoder.cs<|end_filename|>
﻿using AdvUtils;
using RNNSharp.Networks;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Threading;

/// <summary>
/// RNNSharp written by <NAME> (<EMAIL>)
/// </summary>

namespace RNNSharp
{
    public class RNNDecoder
    {
       private ConcurrentQueue<RNN<Sequence>> qRNNs;
        public Config Config;
        private RNN<Sequence> rnn;

        public RNNDecoder(Config config)
        {
            Config = config;
            rnn = RNN<Sequence>.CreateRNN(Config.NetworkType);
            rnn.LoadModel(config.ModelFilePath);
            rnn.MaxSeqLength = config.MaxSequenceLength;

            Logger.WriteLine("CRF Model: {0}", rnn.IsCRFTraining);
            Logger.WriteLine($"Max Sequence Length: {rnn.MaxSeqLength}");
            Logger.WriteLine($"Processor Count: {Environment.ProcessorCount}");

            qRNNs = new ConcurrentQueue<RNN<Sequence>>();
            for (var i = 0; i < Environment.ProcessorCount; i++)
            {
                qRNNs.Enqueue(rnn.Clone());
            }
        }

        private RNN<Sequence> GetRNNInstance()
        {
            RNN<Sequence> r = null;
            if (qRNNs.TryDequeue(out r) == false)
            {
                r = rnn.Clone();
            }

            return r;
        }

        private void FreeRNNInstance(RNN<Sequence> r)
        {
            qRNNs.Enqueue(r);
        }

        public NETWORKTYPE NetworkType => Config.NetworkType;

        public int[][] ProcessNBest(Sentence sent, int nbest)
        {
            var rnn = GetRNNInstance();
            if (rnn.IsCRFTraining == false)
            {
                throw new ArgumentException("N-best result is only for RNN-CRF model.");
            }

            var predicted = rnn.DecodeNBestCRF(sent, Config, nbest);

            FreeRNNInstance(rnn);

            return predicted;
        }

        public int[] Process(Sentence sent)
        {
            int[] predicted = null;
            var rnn = GetRNNInstance();

            if (sent.TokensList.Count >= rnn.MaxSeqLength)
            {
                Logger.WriteLine($"The length of given sentnce is larger than {rnn.MaxSeqLength}, so ignore it: {sent.ToString()}");
            }
            else
            {
                predicted = rnn.IsCRFTraining ? rnn.DecodeCRF(sent, Config) : rnn.DecodeNN(sent, Config);
            }

            FreeRNNInstance(rnn);

            return predicted;
        }

        public float[][] ComputeTopHiddenLayerOutput(Sentence sent)
        {
            var rnn = GetRNNInstance();
            var seq = Config.BuildSequence(sent);
            float[][] output = rnn.ComputeTopHiddenLayerOutput(seq);
            FreeRNNInstance(rnn);

            return output;

        }


        public float[][] ComputeTopHiddenLayerOutput(Sequence seq)
        {

            var rnn = GetRNNInstance();
            float[][] output = rnn.ComputeTopHiddenLayerOutput(seq);
            FreeRNNInstance(rnn);
            return output;
        }

        public int GetTopHiddenLayerSize()
        {
            var rnn = GetRNNInstance();
            int n = rnn.GetTopHiddenLayerSize();
            FreeRNNInstance(rnn);
            return n;
        }
    }
}

<|start_filename|>RNNSharp/Layers/SampledSoftmaxLayer.cs<|end_filename|>
﻿using AdvUtils;
using RNNSharp.Layers;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Numerics;
using System.Threading.Tasks;

namespace RNNSharp
{
    internal class SampledSoftmaxLayer : SoftmaxLayer
    {
        private int negativeSampleSize => config.NegativeSampleSize;
        public HashSet<int> negativeSampleWordList = new HashSet<int>();
        public Random rand = new Random();
        SampledSoftmaxLayerConfig config;

        public SampledSoftmaxLayer() { }
        public SampledSoftmaxLayer(SampledSoftmaxLayerConfig config) : base(config)
        {
            this.config = config;
            if (negativeSampleSize > LayerSize)
            {
                throw new ArgumentException(
                    $"The size of negative sampling('{negativeSampleSize}') cannot be greater than the hidden layer size('{LayerSize}').");
            }
        }

        public override ILayer CreateLayerSharedWegiths()
        {
            SampledSoftmaxLayer layer = new SampledSoftmaxLayer(config);
            ShallowCopyWeightTo(layer);
            layer.InitializeInternalTrainingParameters();

            return layer;
        }


        public override void ForwardPass(List<SparseVector> sparseFeatureGroups, List<float[]> denseFeatureGroups)
        {
            if (runningMode == RunningMode.Training)
            {
                negativeSampleWordList.Clear();

                foreach (var labelId in LabelShortList)
                {
                    negativeSampleWordList.Add(labelId);
                }

                for (var i = 0; i < negativeSampleSize; i++)
                {
                    var wordId = rand.Next() % LayerSize;
                    while (negativeSampleWordList.Contains(wordId))
                    {
                        wordId = (wordId + 1) % LayerSize;
                    }
                    negativeSampleWordList.Add(wordId);
                }

                if (DenseFeatureSize > 0)
                {
                    DenseFeatureGroups = denseFeatureGroups;
                    RNNHelper.matrixXvectorADD(Cells, DenseFeatureGroups, DenseWeights, negativeSampleWordList);
                }

                if (SparseFeatureSize > 0)
                {
                    //Apply sparse features
                    SparseFeatureGroups = sparseFeatureGroups;
                    foreach (var b in negativeSampleWordList)
                    {
                        float score = 0;
                        var vector_b = SparseWeights[b];

                        foreach (var sparseFeature in SparseFeatureGroups)
                        {
                            foreach (var pair in sparseFeature)
                            {
                                score += pair.Value * vector_b[pair.Key];
                            }
                        }
                        Cells[b] += score;
                    }
                }

                //Softmax
                double sum = 0;
                foreach (var c in negativeSampleWordList)
                {
                    var cell = Cells[c];
                    if (cell > 50) cell = 50;
                    if (cell < -50) cell = -50;
                    var val = (float)Math.Exp(cell);
                    sum += val;
                    Cells[c] = val;
                }

                foreach (var c in negativeSampleWordList)
                {
                    Cells[c] /= (float)sum;
                }
            }
            else
            {
                base.ForwardPass(sparseFeatureGroups, denseFeatureGroups);
            }
        }

        public override void ComputeLayerErr(float[] destErrs, bool cleanDest = true)
        {
            RNNHelper.matrixXvectorADDErr(destErrs, Errs, DenseWeights, negativeSampleWordList, cleanDest);
        }

        public override void ComputeLayerErr(ILayer prevLayer)
        {
            ComputeLayerErr(prevLayer.Errs);
        }

        public override int GetBestOutputIndex()
        {
            if (runningMode == RunningMode.Training)
            {
                var imax = 0;
                var dmax = double.MinValue;
                foreach (var k in negativeSampleWordList.Where(k => Cells[k] > dmax))
                {
                    dmax = Cells[k];
                    imax = k;
                }
                return imax;
            }
            else
            {
                return base.GetBestOutputIndex();
            }
        }

        public override void BackwardPass()
        {
            //Update hidden-output weights

                       foreach (var c in negativeSampleWordList)
            {
                UpdateLayerAt(c);
            }

        }

        public override void ComputeOutputLoss(Matrix<float> CRFSeqOutput, State state, int timeat)
        {
            if (CRFSeqOutput != null)
            {
                //For RNN-CRF, use joint probability of output layer nodes and transition between contigous nodes
                foreach (var c in negativeSampleWordList)
                {
                    Errs[c] = -CRFSeqOutput[timeat][c];
                }
                Errs[state.Label] = (float)(1.0 - CRFSeqOutput[timeat][state.Label]);
            }
            else
            {
                //For standard RNN
                foreach (var c in negativeSampleWordList)
                {
                    Errs[c] = -Cells[c];
                }
                Errs[state.Label] = (float)(1.0 - Cells[state.Label]);
            }
        }
    }
}

<|start_filename|>RNNSharp/Layers/SimpleLayer.cs<|end_filename|>
﻿using AdvUtils;
using RNNSharp.Layers;
using System;
using System.Collections.Generic;
using System.IO;
using System.Numerics;
using System.Threading.Tasks;

namespace RNNSharp
{
    public class Neuron
    {
        public float[] Cells { get; set; }
        public float[] Errs { get; set; }

        public Neuron(int length)
        {
            Cells = new float[length];
            Errs = new float[length];
        }

        public virtual int GetMaxCellIndex()
        {
            var imax = 0;
            var dmax = Cells[0];
            for (var k = 1; k < Cells.Length; k++)
            {
                if (Cells[k] > dmax)
                {
                    dmax = Cells[k];
                    imax = k;
                }
            }
            return imax;
        }
    }

    public class SimpleLayer : IOutputLayer
    {
        public float[] Cells { get; set; }
        public float[] Errs { get; set; }
        public int SparseFeatureSize { get; set; }
        public int DenseFeatureSize { get; set; }

        public List<int> LabelShortList { get; set; }
        public List<SparseVector> SparseFeatureGroups { get; set; }
        public List<float[]> DenseFeatureGroups { get; set; }

        protected ParallelOptions parallelOption = new ParallelOptions();
        protected RunningMode runningMode;

        public SimpleLayer() { }

        public SimpleLayer(LayerConfig config)
        {
            LayerConfig = config;
            Cells = new float[LayerSize];
            Errs = new float[LayerSize];
            LabelShortList = new List<int>();
        }

        public int LayerSize
        {
            get { return LayerConfig.LayerSize; }
            set { LayerConfig.LayerSize = value; }
        }

        public LayerType LayerType
        {
            get { return LayerConfig.LayerType; }
            set { LayerConfig.LayerType = value; }
        }

        public LayerConfig LayerConfig { get; set; }
        protected Matrix<float> SparseWeights { get; set; }
        Matrix<float> SparseWeightsLearningRate { get; set; }

        Matrix<float> SparseWeightsDelta { get; set; }

        protected Matrix<float> DenseWeights { get; set; }
        Matrix<float> DenseWeightsLearningRate { get; set; }



        public Matrix<float> DenseWeightsDelta { get; set; }

        public void SetRunningMode(RunningMode mode)
        {
            runningMode = mode;
        }

        public virtual Neuron CopyNeuronTo(Neuron neuron)
        {
            Cells.CopyTo(neuron.Cells, 0);

            return neuron;
        }

        public virtual ILayer CreateLayerSharedWegiths()
        {
            SimpleLayer layer = new SimpleLayer(LayerConfig);
            ShallowCopyWeightTo(layer);
            return layer;
        }

        public virtual void ShallowCopyWeightTo(ILayer layer)
        {
            SimpleLayer destLayer = layer as SimpleLayer;

            destLayer.DenseWeights = DenseWeights;
            destLayer.DenseWeightsDelta = DenseWeightsDelta;
            destLayer.DenseWeightsLearningRate = DenseWeightsLearningRate;
            destLayer.DenseFeatureSize = DenseFeatureSize;

            destLayer.SparseWeights = SparseWeights;
            destLayer.SparseWeightsDelta = SparseWeightsDelta;
            destLayer.SparseWeightsLearningRate = SparseWeightsLearningRate;
            destLayer.SparseFeatureSize = SparseFeatureSize;

            destLayer.InitializeInternalTrainingParameters();
        }

        public virtual void SetNeuron(Neuron neuron)
        {
            neuron.Cells.CopyTo(Cells, 0);
            //  neuron.Errs.CopyTo(Errs, 0);

            Errs = neuron.Errs;
        }

        /// <summary>
        /// Initialize parameters only used for training
        /// </summary>
        public virtual void InitializeInternalTrainingParameters()
        {
            if (DenseFeatureSize > 0 && DenseWeightsLearningRate == null)
            {
                DenseWeightsLearningRate = new Matrix<float>(LayerSize, DenseFeatureSize);
                DenseWeightsDelta = new Matrix<float>(LayerSize, DenseFeatureSize);
            }

            if (SparseFeatureSize > 0 && SparseWeightsLearningRate == null)
            {
                SparseWeightsLearningRate = new Matrix<float>(LayerSize, SparseFeatureSize);
                SparseWeightsDelta = new Matrix<float>(LayerSize, SparseFeatureSize);
            }
        }

        public virtual void InitializeWeights(int sparseFeatureSize, int denseFeatureSize)
        {
            if (denseFeatureSize > 0)
            {
                Logger.WriteLine("Initializing dense feature matrix. layer size = {0}, feature size = {1}", LayerSize, denseFeatureSize);
                DenseFeatureSize = denseFeatureSize;
                DenseWeights = new Matrix<float>(LayerSize, denseFeatureSize);
                for (var i = 0; i < DenseWeights.Height; i++)
                {
                    for (var j = 0; j < DenseWeights.Width; j++)
                    {
                        DenseWeights[i][j] = RNNHelper.RandInitWeight();
                    }
                }
            }

            if (sparseFeatureSize > 0)
            {
                Logger.WriteLine("Initializing sparse feature matrix. layer size = {0}, feature size = {1}", LayerSize, sparseFeatureSize);
                SparseFeatureSize = sparseFeatureSize;
                SparseWeights = new Matrix<float>(LayerSize, SparseFeatureSize);
                for (var i = 0; i < SparseWeights.Height; i++)
                {
                    for (var j = 0; j < SparseWeights.Width; j++)
                    {
                        SparseWeights[i][j] = RNNHelper.RandInitWeight();
                    }
                }            
            }

            InitializeInternalTrainingParameters();
        }

        public virtual void Save(BinaryWriter fo)
        {
            fo.Write(LayerSize);
            fo.Write(SparseFeatureSize);
            fo.Write(DenseFeatureSize);

            Logger.WriteLine(
                $"Saving simple layer, size = '{LayerSize}', sparse feature size = '{SparseFeatureSize}', dense feature size = '{DenseFeatureSize}'");

            if (SparseFeatureSize > 0)
            {
                Logger.WriteLine("Saving sparse feature weights...");
                RNNHelper.SaveMatrix(SparseWeights, fo);
            }

            if (DenseFeatureSize > 0)
            {
                //weight fea->hidden
                Logger.WriteLine("Saving dense feature weights...");
                RNNHelper.SaveMatrix(DenseWeights, fo);
            }
        }

        public virtual void Load(BinaryReader br, LayerType layerType, bool forTraining = false)
        {
            LayerConfig = new LayerConfig();
            LayerConfig.LayerSize = br.ReadInt32();
            LayerConfig.LayerType = layerType;

            Cells = new float[LayerSize];
            Errs = new float[LayerSize];
            LabelShortList = new List<int>();

            SparseFeatureSize = br.ReadInt32();
            DenseFeatureSize = br.ReadInt32();

            if (SparseFeatureSize > 0)
            {
                Logger.WriteLine("Loading sparse feature weights...");
                SparseWeights = RNNHelper.LoadMatrix(br);
            }

            if (DenseFeatureSize > 0)
            {
                Logger.WriteLine("Loading dense feature weights...");
                DenseWeights = RNNHelper.LoadMatrix(br);
            }

            if (forTraining)
            {
                InitializeInternalTrainingParameters();
            }
        }

        public virtual void CleanForTraining()
        {
            if (SparseWeightsLearningRate != null)
            {
                SparseWeightsLearningRate.Clean();
                SparseWeightsDelta.Clean();
            }

            if (DenseWeightsLearningRate != null)
            {
                DenseWeightsLearningRate.Clean();
                DenseWeightsDelta.Clean();
            }
        }


        public virtual void ForwardPass(SparseVector sparseFeature, float[] denseFeature)
        {
            List<SparseVector> sparseFeatureGroups = new List<SparseVector>();
            sparseFeatureGroups.Add(sparseFeature);

            List<float[]> denseFeatureGroups = new List<float[]>();
            denseFeatureGroups.Add(denseFeature);

            ForwardPass(sparseFeatureGroups, denseFeatureGroups);
        }


        public virtual void ForwardPass(List<SparseVector> sparseFeatureGroups, List<float[]> denseFeatureGroups)
        {
            if (DenseFeatureSize > 0)
            {
                DenseFeatureGroups = denseFeatureGroups;
                RNNHelper.matrixXvectorADD(Cells, DenseFeatureGroups, DenseWeights, LayerSize);
            }

            if (SparseFeatureSize > 0)
            {
                //Apply sparse features
                SparseFeatureGroups = sparseFeatureGroups;
                for (var b = 0; b < LayerSize; b++)
                {
                    float score = 0;
                    var vector_b = SparseWeights[b];

                    foreach (var sparseFeature in SparseFeatureGroups)
                    {
                        foreach (var pair in sparseFeature)
                        {
                            score += pair.Value * vector_b[pair.Key];
                        }
                    }
                    Cells[b] += score;
                }
            }
        }

        public virtual void BackwardPass()
        {
            //Update hidden-output weights
            for (var c = 0; c < LayerSize; c++)
            {
                UpdateLayerAt(c);
            }
        }

        protected void UpdateLayerAt(int c)
        {
            var err = Errs[c];
            if (DenseFeatureSize > 0)
            {
                var denseWeightsDelta_c = DenseWeightsDelta[c];
                var j = 0;

                foreach (var denseFeature in DenseFeatureGroups)
                {
                    int k = 0;
                    var denseFeatureSize = denseFeature.Length;
                    var moreItems = (denseFeatureSize % Vector<float>.Count);
                    while (k < denseFeatureSize - moreItems)
                    {
                        var vecDenseFeature = new Vector<float>(denseFeature, k);
                        var vecDelta = vecDenseFeature * err;

                        var vecVector_C = new Vector<float>(denseWeightsDelta_c, j);
                        vecVector_C += vecDelta;
                        vecVector_C.CopyTo(denseWeightsDelta_c, j);

                        j += Vector<float>.Count;
                        k += Vector<float>.Count;
                    }

                    while (k < denseFeatureSize)
                    {
                        denseWeightsDelta_c[j] += err * denseFeature[k];

                        j++;
                        k++;
                    }
                }
            }

            if (SparseFeatureSize > 0)
            {
                var sparseWeightsDelta_c = SparseWeightsDelta[c];

                foreach (var sparseFeature in SparseFeatureGroups)
                {
                    foreach (var pair in sparseFeature)
                    {
                        var pos = pair.Key;
                        var val = pair.Value;
                        sparseWeightsDelta_c[pos] += err * val;
                    }
                }
            }
        }

        public virtual void UpdateWeights()
        {
            Vector<float> vecMiniBatchSize = new Vector<float>(RNNHelper.MiniBatchSize);
            for (var i = 0; i < LayerSize; i++)
            {
                if (SparseFeatureSize > 0)
                {
                    var sparseWeights_i = SparseWeights[i];
                    var sparseDelta_i = SparseWeightsDelta[i];
                    var sparseLearningRate_i = SparseWeightsLearningRate[i];
                    var j = 0;

                    var moreItems = (SparseFeatureSize % Vector<float>.Count);
                    while (j < SparseFeatureSize - moreItems)
                    {
                        Vector<float> vecDelta = new Vector<float>(sparseDelta_i, j);
                        Vector<float>.Zero.CopyTo(sparseDelta_i, j);

                        if (vecDelta != Vector<float>.Zero)
                        {
                            vecDelta = vecDelta / vecMiniBatchSize;
                            vecDelta = RNNHelper.NormalizeGradient(vecDelta);

                            var wlr_i = new Vector<float>(sparseLearningRate_i, j);
                            var vecLearningRate = RNNHelper.ComputeLearningRate(vecDelta, ref wlr_i);
                            wlr_i.CopyTo(sparseLearningRate_i, j);

                            vecDelta = vecLearningRate * vecDelta;
                            Vector<float> vecWeights = new Vector<float>(sparseWeights_i, j);
                            vecWeights += vecDelta;
                            vecWeights.CopyTo(sparseWeights_i, j);

                        }

                        j += Vector<float>.Count;
                    }

                    while (j < SparseFeatureSize)
                    {
                        var delta = sparseDelta_i[j];
                        sparseDelta_i[j] = 0;
                        delta = delta / RNNHelper.MiniBatchSize;
                        delta = RNNHelper.NormalizeGradient(delta);

                        var newLearningRate = RNNHelper.ComputeLearningRate(SparseWeightsLearningRate, i, j, delta);
                        sparseWeights_i[j] += newLearningRate * delta;

                        j++;
                    }

                }

                if (DenseFeatureSize > 0)
                {
                    var denseWeights_i = DenseWeights[i];
                    var denseDelta_i = DenseWeightsDelta[i];
                    var denseLearningRate_i = DenseWeightsLearningRate[i];
                    var j = 0;

                    var moreItems = (DenseFeatureSize % Vector<float>.Count);
                    while (j < DenseFeatureSize - moreItems)
                    {
                        Vector<float> vecDelta = new Vector<float>(denseDelta_i, j);
                        Vector<float>.Zero.CopyTo(denseDelta_i, j);

                        vecDelta = vecDelta / vecMiniBatchSize;

                        vecDelta = RNNHelper.NormalizeGradient(vecDelta);

                        var wlr_i = new Vector<float>(denseLearningRate_i, j);
                        var vecLearningRate = RNNHelper.ComputeLearningRate(vecDelta, ref wlr_i);
                        wlr_i.CopyTo(denseLearningRate_i, j);

                        vecDelta = vecLearningRate * vecDelta;
                        Vector<float> vecWeights = new Vector<float>(denseWeights_i, j);
                        vecWeights += vecDelta;
                        vecWeights.CopyTo(denseWeights_i, j);

                        j += Vector<float>.Count;
                    }

                    while (j < DenseFeatureSize)
                    {
                        var delta = denseDelta_i[j];
                        denseDelta_i[j] = 0;

                        delta = delta / RNNHelper.MiniBatchSize;
                        delta = RNNHelper.NormalizeGradient(delta);

                        var newLearningRate = RNNHelper.ComputeLearningRate(DenseWeightsLearningRate, i, j, delta);
                        denseWeights_i[j] += newLearningRate * delta;

                        j++;
                    }
                }
            }

        }

        public virtual void Reset()
        {
        }

        public virtual void ComputeLayerErr(float[] destErrLayer, bool cleanDest = true)
        {
            RNNHelper.matrixXvectorADDErr(destErrLayer, Errs, DenseWeights, cleanDest);
        }

        public virtual void ComputeLayerErr(List<float[]> destErrsList, bool cleanDest = true)
        {
            RNNHelper.matrixXvectorADDErr(destErrsList, Errs, DenseWeights, cleanDest);
        }

        public virtual void ComputeLayerErr(ILayer prevLayer)
        {
            ComputeLayerErr(prevLayer.Errs);
        }

        public virtual void ComputeOutputLoss(Matrix<float> CRFSeqOutput, State state, int timeat)
        {
            if (CRFSeqOutput != null)
            {
                //For RNN-CRF, use joint probability of output layer nodes and transition between contigous nodes
                for (var c = 0; c < LayerSize; c++)
                {
                    Errs[c] = -CRFSeqOutput[timeat][c];
                }
                Errs[state.Label] = (float)(1.0 - CRFSeqOutput[timeat][state.Label]);
            }
            else
            {
                //For standard RNN
                for (var c = 0; c < LayerSize; c++)
                {
                    Errs[c] = -Cells[c];
                }
                Errs[state.Label] = (float)(1.0 - Cells[state.Label]);
            }
        }

        public virtual int GetBestOutputIndex()
        {
            var imax = 0;
            var dmax = Cells[0];
            for (var k = 1; k < LayerSize; k++)
            {
                if (Cells[k] > dmax)
                {
                    dmax = Cells[k];
                    imax = k;
                }
            }
            return imax;
        }
    }
}

<|start_filename|>RNNSharp/RNNEncoder.cs<|end_filename|>
﻿using AdvUtils;
using RNNSharp.Networks;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Numerics;
using System.Threading;
using System.Threading.Tasks;

/// <summary>
/// RNNSharp written by <NAME> (<EMAIL>)
/// </summary>

namespace RNNSharp
{
    public class RNNEncoder<T> where T : ISequence
    {
        private readonly Config featurizer;
        private readonly ModelSetting ModelSettings;

        public RNNEncoder(ModelSetting modelSetting, Config featurizer)
        {
            ModelSettings = modelSetting;
            this.featurizer = featurizer;
        }

        private NETWORKTYPE networkType => featurizer.NetworkType;

        private string modelFilePath => featurizer.ModelFilePath;

        private List<LayerConfig> hiddenLayersConfig => featurizer.HiddenLayersConfig;

        private LayerConfig outputLayerConfig => featurizer.OutputLayerConfig;

        public bool IsCRFTraining => featurizer.IsCRFTraining;
        private int maxSequenceLength => featurizer.MaxSequenceLength;

        public DataSet<T> TrainingSet { get; set; }
        public DataSet<T> ValidationSet { get; set; }

        int processedSequence = 0;
        int processedWordCnt = 0;
        int tknErrCnt = 0;
        int sentErrCnt = 0;

        int updatingWeights = 0;
        int processMiniBatch = 0;

        ParallelOptions parallelOptions;

        /// <summary>
        /// Process entire corpus set by given RNN
        /// </summary>
        /// <param name="rnns"></param>
        /// <param name="corpusSet"></param>
        /// <param name="runningMode"></param>
        public void Process(List<RNN<T>> rnns, DataSet<T> corpusSet, RunningMode runningMode)
        {
            parallelOptions = new ParallelOptions();
            parallelOptions.MaxDegreeOfParallelism = Environment.ProcessorCount;
            processedSequence = 0;
            processedWordCnt = 0;
            tknErrCnt = 0;
            sentErrCnt = 0;

            corpusSet.Shuffle();

            //Add RNN instance into job queue
            ConcurrentQueue<RNN<T>> qRNNs = new ConcurrentQueue<RNN<T>>();
            foreach (var rnn in rnns)
            {
                qRNNs.Enqueue(rnn);
            }

            Parallel.For(0, corpusSet.SequenceList.Count, parallelOptions, i =>
            {
                //Get a free RNN instance for running
                RNN<T> rnn;
                if (qRNNs.TryDequeue(out rnn) == false)
                {
                    //The queue is empty, so we clone a new one
                    rnn = rnns[0].Clone();
                    Logger.WriteLine("Cloned a new RNN instance for training.");
                }

                var pSequence = corpusSet.SequenceList[i];

                //Calcuate how many tokens we are going to process in this sequence
                int tokenCnt = 0;
                if (pSequence is Sequence)
                {
                    tokenCnt = (pSequence as Sequence).States.Length;
                }
                else
                {
                    SequencePair sp = pSequence as SequencePair;
                    if (sp.srcSentence.TokensList.Count > rnn.MaxSeqLength)
                    {
                        qRNNs.Enqueue(rnn);
                        return;
                    }
                    tokenCnt = sp.tgtSequence.States.Length;
                }

                //This sequence is too long, so we ignore it
                if (tokenCnt > rnn.MaxSeqLength)
                {
                    qRNNs.Enqueue(rnn);
                    return;
                }

                //Run neural network
                int[] predicted;
                if (IsCRFTraining)
                {
                    predicted = rnn.ProcessSequenceCRF(pSequence as Sequence, runningMode);
                }
                else
                {
                    Matrix<float> m;
                    predicted = rnn.ProcessSequence(pSequence, runningMode, false, out m);
                }

                //Update counters
                Interlocked.Add(ref processedWordCnt, tokenCnt);
                Interlocked.Increment(ref processedSequence);
                Interlocked.Increment(ref processMiniBatch);

                int newTknErrCnt;
                if (pSequence is Sequence)
                {
                    newTknErrCnt = GetErrorTokenNum(pSequence as Sequence, predicted);
                }
                else
                {
                    newTknErrCnt = GetErrorTokenNum((pSequence as SequencePair).tgtSequence, predicted);
                }

                Interlocked.Add(ref tknErrCnt, newTknErrCnt);
                if (newTknErrCnt > 0)
                {
                    Interlocked.Increment(ref sentErrCnt);
                }

                //Update weights
                //We only allow one thread to update weights, and other threads keep running to train or predict given sequences
                //Note: we don't add any lock when updating weights and deltas for weights in order to improve performance singificantly, 
                //so that means race condition will happen and it's okay for us.
                if (runningMode == RunningMode.Training && processMiniBatch > 0 && processMiniBatch % ModelSettings.MiniBatchSize == 0 && updatingWeights == 0)
                {
                    Interlocked.Increment(ref updatingWeights);
                    if (updatingWeights == 1)
                    {
                        rnn.UpdateWeights();
                        Interlocked.Exchange(ref processMiniBatch, 0);
                    }
                    Interlocked.Decrement(ref updatingWeights);
                }

                //Show progress information
                if (processedSequence % 1000 == 0)
                {
                    Logger.WriteLine("Progress = {0} ", processedSequence / 1000 + "K/" + corpusSet.SequenceList.Count / 1000.0 + "K");
                    Logger.WriteLine(" Error token ratio = {0}%", (double)tknErrCnt / (double)processedWordCnt * 100.0);
                    Logger.WriteLine(" Error sentence ratio = {0}%", (double)sentErrCnt / (double)processedSequence * 100.0);
                }

                //Save intermediate model file
                if (ModelSettings.SaveStep > 0 && processedSequence % ModelSettings.SaveStep == 0)
                {
                    //After processed every m_SaveStep sentences, save current model into a temporary file
                    Logger.WriteLine("Saving temporary model into file...");
                    try
                    {
                        rnn.SaveModel("model.tmp");
                    }
                    catch (Exception err)
                    {
                        Logger.WriteLine($"Fail to save temporary model into file. Error: {err.Message.ToString()}");
                    }
                }

                qRNNs.Enqueue(rnn);
            });

        }

        

        private int GetErrorTokenNum(Sequence seq, int[] predicted)
        {
            var tknErrCnt = 0;
            var numStates = seq.States.Length;
            for (var curState = 0; curState < numStates; curState++)
            {
                if (predicted[curState] != seq.States[curState].Label)
                {
                    tknErrCnt++;
                }
            }

            return tknErrCnt;
        }

        public void Train()
        {
            //Create neural net work
            Logger.WriteLine("Create a new network according settings in configuration file.");
            Logger.WriteLine($"Processor Count = {Environment.ProcessorCount}");

            RNN<T> rnn = RNN<T>.CreateRNN(networkType);
            if (ModelSettings.IncrementalTrain)
            {
                Logger.WriteLine($"Loading previous trained model from {modelFilePath}.");
                rnn.LoadModel(modelFilePath, true);
            }
            else
            {
                Logger.WriteLine("Create a new network.");
                rnn.CreateNetwork(hiddenLayersConfig, outputLayerConfig, TrainingSet, featurizer);
                //Create tag-bigram transition probability matrix only for sequence RNN mode
                if (IsCRFTraining)
                {
                    Logger.WriteLine("Initialize bigram transition for CRF output layer.");
                    rnn.InitializeCRFWeights(TrainingSet.CRFLabelBigramTransition);
                }
            }

            rnn.MaxSeqLength = maxSequenceLength;
            rnn.bVQ = ModelSettings.VQ != 0 ? true : false;
            rnn.IsCRFTraining = IsCRFTraining;
            if (rnn.IsCRFTraining)
            {
                rnn.InitializeCRFVariablesForTraining();
            }

            int N = Environment.ProcessorCount * 2;
            List<RNN<T>> rnns = new List<RNN<T>>();
            rnns.Add(rnn);

            for (int i = 1; i < N; i++)
            {
                rnns.Add(rnn.Clone());
            }

            //Initialize RNNHelper
            RNNHelper.LearningRate = ModelSettings.LearningRate;
            RNNHelper.vecNormalLearningRate = new Vector<float>(RNNHelper.LearningRate);

            RNNHelper.GradientCutoff = ModelSettings.GradientCutoff;
            RNNHelper.vecMaxGrad = new Vector<float>(RNNHelper.GradientCutoff);
            RNNHelper.vecMinGrad = new Vector<float>(-RNNHelper.GradientCutoff);
            RNNHelper.IsConstAlpha = ModelSettings.IsConstAlpha;
            RNNHelper.MiniBatchSize = ModelSettings.MiniBatchSize;

            Logger.WriteLine("");

            Logger.WriteLine("Iterative training begins ...");
            var bestTrainTknErrCnt = long.MaxValue;
            var bestValidTknErrCnt = long.MaxValue;
            var lastAlpha = RNNHelper.LearningRate;
            var iter = 0;
            ParallelOptions parallelOption = new ParallelOptions();
            parallelOption.MaxDegreeOfParallelism = -1;
            while (true)
            {
                if (ModelSettings.MaxIteration > 0 && iter > ModelSettings.MaxIteration)
                {
                    Logger.WriteLine("We have trained this model {0} iteration, exit.");
                    break;
                }

                var start = DateTime.Now;
                Logger.WriteLine($"Start to training {iter} iteration. learning rate = {RNNHelper.LearningRate}");

                //Clean all RNN instances' status for training
                foreach (var r in rnns)
                {
                    r.CleanStatusForTraining();
                }
                Process(rnns, TrainingSet, RunningMode.Training);

                var duration = DateTime.Now.Subtract(start);

                Logger.WriteLine($"End {iter} iteration. Time duration = {duration}");
                Logger.WriteLine("");

                if (tknErrCnt >= bestTrainTknErrCnt && lastAlpha != RNNHelper.LearningRate)
                {
                    //Although we reduce alpha value, we still cannot get better result.
                    Logger.WriteLine(
                        $"Current token error count({(double)tknErrCnt / (double)processedWordCnt * 100.0}%) is larger than the previous one({(double)bestTrainTknErrCnt / (double)processedWordCnt * 100.0}%) on training set. End training early.");
                    Logger.WriteLine("Current alpha: {0}, the previous alpha: {1}", RNNHelper.LearningRate, lastAlpha);
                    break;
                }
                lastAlpha = RNNHelper.LearningRate;

                int trainTknErrCnt = tknErrCnt;
                //Validate the model by validated corpus
                if (ValidationSet != null)
                {
                    Logger.WriteLine("Verify model on validated corpus.");
                    Process(rnns, ValidationSet, RunningMode.Validate);
                    Logger.WriteLine("End model verification.");
                    Logger.WriteLine("");

                    if (tknErrCnt < bestValidTknErrCnt)
                    {
                        //We got better result on validated corpus, save this model
                        Logger.WriteLine($"Saving better model into file {modelFilePath}, since we got a better result on validation set.");
                        Logger.WriteLine($"Error token percent: {(double)tknErrCnt / (double)processedWordCnt * 100.0}%, Error sequence percent: {(double)sentErrCnt / (double)processedSequence * 100.0}%");

                        rnn.SaveModel(modelFilePath);
                        bestValidTknErrCnt = tknErrCnt;
                    }
                }
                else if (trainTknErrCnt < bestTrainTknErrCnt)
                {
                    //We don't have validate corpus, but we get a better result on training corpus
                    //We got better result on validated corpus, save this model
                    Logger.WriteLine($"Saving better model into file {modelFilePath}, although validation set doesn't exist, we have better result on training set.");
                    Logger.WriteLine($"Error token percent: {(double)trainTknErrCnt / (double)processedWordCnt * 100.0}%, Error sequence percent: {(double)sentErrCnt / (double)processedSequence * 100.0}%");

                    rnn.SaveModel(modelFilePath);
                }

                Logger.WriteLine("");

                if (trainTknErrCnt >= bestTrainTknErrCnt)
                {
                    //We don't have better result on training set, so reduce learning rate
                    RNNHelper.LearningRate = RNNHelper.LearningRate / 2.0f;
                    RNNHelper.vecNormalLearningRate = new Vector<float>(RNNHelper.LearningRate);
                }
                else
                {
                    bestTrainTknErrCnt = trainTknErrCnt;
                }

                iter++;
            }
        }
    }
}

<|start_filename|>RNNSharp/DataSet.cs<|end_filename|>
﻿using System;
using System.Collections.Generic;

/// <summary>
/// RNNSharp written by <NAME> (<EMAIL>)
/// </summary>

namespace RNNSharp
{
    public class DataSet<T> where T : ISequence
    {
        public DataSet(int tagSize)
        {
            TagSize = tagSize;
            SequenceList = new List<T>();
            CRFLabelBigramTransition = new List<List<float>>();
        }

        public List<T> SequenceList { get; set; }
        public int TagSize { get; set; }
        public List<List<float>> CRFLabelBigramTransition { get; set; }

        public int DenseFeatureSize => 0 == SequenceList.Count ? 0 : SequenceList[0].DenseFeatureSize;

        public int SparseFeatureSize => 0 == SequenceList.Count ? 0 : SequenceList[0].SparseFeatureSize;

        public void Shuffle()
        {
            var rnd = new Random(DateTime.Now.Millisecond);
            for (var i = 0; i < SequenceList.Count; i++)
            {
                var m = rnd.Next() % SequenceList.Count;
                var tmp = SequenceList[i];
                SequenceList[i] = SequenceList[m];
                SequenceList[m] = tmp;
            }
        }


        public List<DataSet<T>> Split(int n)
        {
            List<DataSet<T>> dataSets = new List<DataSet<T>>();
            for (int i = 0; i < n; i++)
            {
                dataSets.Add(new DataSet<T>(TagSize));
            }

            for (int i = 0; i < SequenceList.Count; i++)
            {
                dataSets[i % n].SequenceList.Add(SequenceList[i]);
            }

            return dataSets;
        }

        public void BuildLabelBigramTransition(float smooth = 1.0f)
        {
            CRFLabelBigramTransition = new List<List<float>>();

            for (var i = 0; i < TagSize; i++)
            {
                CRFLabelBigramTransition.Add(new List<float>());
            }
            for (var i = 0; i < TagSize; i++)
            {
                for (var j = 0; j < TagSize; j++)
                {
                    CRFLabelBigramTransition[i].Add(smooth);
                }
            }

            for (var i = 0; i < SequenceList.Count; i++)
            {
                var sequence = SequenceList[i] as Sequence;
                if (sequence.States.Length <= 1)
                    continue;

                var pLabel = sequence.States[0].Label;
                for (var j = 1; j < sequence.States.Length; j++)
                {
                    var label = sequence.States[j].Label;
                    CRFLabelBigramTransition[label][pLabel]++;
                    pLabel = label;
                }
            }

            for (var i = 0; i < TagSize; i++)
            {
                float sum = 0;
                for (var j = 0; j < TagSize; j++)
                {
                    sum += CRFLabelBigramTransition[i][j];
                }

                for (var j = 0; j < TagSize; j++)
                {
                    CRFLabelBigramTransition[i][j] = (float)Math.Log(CRFLabelBigramTransition[i][j] / sum);
                }
            }
        }
    }
}

<|start_filename|>RNNSharp/Sentence.cs<|end_filename|>
﻿using AdvUtils;
using System;
using System.Collections.Generic;
using System.Text;

/// <summary>
/// RNNSharp written by <NAME> (<EMAIL>)
/// </summary>

namespace RNNSharp
{
    public class Sentence : ISentence
    {
        public Sentence(List<string[]> tokensList, bool addSentBE = true)
        {
            var dim = 0;
            TokensList = new List<string[]>();

            if (tokensList.Count == 0)
            {
                return;
            }

            //Check if dimension is consistent inside the sentence
            foreach (var tokens in tokensList)
            {
                if (dim > 0 && tokens.Length != dim)
                {
                    var err = ReportInvalidateTokens(tokensList, dim, tokens);
                    throw new FormatException($"Invalidated record: {err}");
                }

                dim = tokens.Length;
                TokensList.Add(tokens);
            }

            if (addSentBE)
            {
                //Add begin/end of sentence flag into feature
                var beginFeatures = new string[dim];
                var endFeatures = new string[dim];

                for (var i = 0; i < dim - 1; i++)
                {
                    beginFeatures[i] = "<s>";
                    endFeatures[i] = "</s>";
                }

                beginFeatures[dim - 1] = TagSet.DefaultTag;
                endFeatures[dim - 1] = TagSet.DefaultTag;

                TokensList.Insert(0, beginFeatures);
                TokensList.Add(endFeatures);
            }
        }

        public List<string[]> TokensList { get; }

        public override string ToString()
        {
            var sb = new StringBuilder();
            foreach (var tokens in TokensList)
            {
                foreach (var token in tokens)
                {
                    sb.Append(token);
                    sb.Append('\t');
                }
                sb.AppendLine();
            }

            return sb.ToString();
        }

        private string ReportInvalidateTokens(List<string[]> tokenList, int dim, string[] badTokens)
        {
            var sb = new StringBuilder();
            sb.AppendLine(
                $"Inconsistent feature dimension in the record.It's {badTokens.Length}, but it should be {dim}");
            sb.AppendLine(ToString());
            Logger.WriteLine(Logger.Level.err, sb.ToString());

            return sb.ToString();
        }
    }
}

<|start_filename|>RNNSharp/SentencePair.cs<|end_filename|>
﻿namespace RNNSharp
{
    public class SentencePair :ISentence
    {
        public Sentence srcSentence;
        public Sentence tgtSentence;
    }
}

<|start_filename|>RNNSharp/Layers/DropoutLayer.cs<|end_filename|>
﻿using AdvUtils;
using RNNSharp.Layers;
using System;
using System.Collections.Generic;
using System.IO;
using System.Numerics;
using System.Threading.Tasks;

namespace RNNSharp
{
    public class DropoutNeuron : Neuron
    {
        public bool[] mask;
        public DropoutNeuron(int length) : base(length)
        {
            mask = new bool[length];
        }
    }

    internal class DropoutLayer : SimpleLayer
    {
        private float dropoutRatio => config.DropoutRatio;
        private Random rnd;
        private bool[] mask;
        DropoutLayerConfig config;

        public DropoutLayer() { }
        public DropoutLayer(DropoutLayerConfig config) : base(config)
        {
            this.config = config;
            mask = new bool[LayerSize];
            rnd = new Random();
        }

        public override ILayer CreateLayerSharedWegiths()
        {
            DropoutLayer layer = new DropoutLayer(config);
            ShallowCopyWeightTo(layer);
            layer.InitializeInternalTrainingParameters();

            return layer;
        }

        public override Neuron CopyNeuronTo(Neuron neuron)
        {
            DropoutNeuron dropoutNeuron = neuron as DropoutNeuron;
            mask.CopyTo(dropoutNeuron.mask, 0);
            Cells.CopyTo(dropoutNeuron.Cells, 0);

            return dropoutNeuron;
        }

        public override void SetNeuron(Neuron neuron)
        {
            DropoutNeuron dropoutNeuron = neuron as DropoutNeuron;
            dropoutNeuron.Cells.CopyTo(Cells, 0);
            for (int i = 0; i < LayerSize; i++)
            {
                if (dropoutNeuron.mask[i])
                {
                    Errs[i] = 0;
                }
                else
                {
                    Errs[i] = neuron.Errs[i];
                }
            }
        }

        public override void InitializeWeights(int sparseFeatureSize, int denseFeatureSize)
        {
            if (denseFeatureSize > 0)
            {
                Logger.WriteLine("Initializing dense feature matrix. layer size = {0}, feature size = {1}", LayerSize, denseFeatureSize);
                DenseFeatureSize = denseFeatureSize;
                DenseWeights = new Matrix<float>(LayerSize, denseFeatureSize);
                for (var i = 0; i < DenseWeights.Height; i++)
                {
                    for (var j = 0; j < DenseWeights.Width; j++)
                    {
                        DenseWeights[i][j] = 1.0f;
                    }
                }
            }

            if (sparseFeatureSize > 0)
            {
                Logger.WriteLine("Initializing sparse feature matrix. layer size = {0}, feature size = {1}", LayerSize, sparseFeatureSize);
                SparseFeatureSize = sparseFeatureSize;
                SparseWeights = new Matrix<float>(LayerSize, SparseFeatureSize);
                for (var i = 0; i < SparseWeights.Height; i++)
                {
                    for (var j = 0; j < SparseWeights.Width; j++)
                    {
                        SparseWeights[i][j] = 1.0f;
                    }
                }
            }

            InitializeInternalTrainingParameters();
        }

        public override void UpdateWeights()
        {

        }

        public override void ForwardPass(List<SparseVector> sparseFeatureGroups, List<float[]> denseFeatureGroups)
        {
            if (runningMode == RunningMode.Training)
            {
                int i = 0;
                foreach (var denseFeature in denseFeatureGroups)
                {
                    for (var j = 0; j < denseFeature.Length; j++)
                    {
                        var r = (float)rnd.NextDouble();
                        if (r < dropoutRatio)
                        {
                            mask[i] = true;
                            Cells[i] = 0;
                        }
                        else
                        {
                            mask[i] = false;
                            Cells[i] = denseFeature[j];
                        }
                        i++;
                    }
                }
            }
            else
            {
                int i = 0;
                foreach (var denseFeature in denseFeatureGroups)
                {
                    for (var j = 0; j < denseFeature.Length; j++)
                    {
                        Cells[i] = (float)(1.0 - dropoutRatio) * denseFeature[j];
                        i++;
                    }
                }
            }
        }

        public override void BackwardPass()
        {
        }

        public override void ComputeLayerErr(ILayer prevLayer)
        {
            base.ComputeLayerErr(prevLayer);
            //Apply drop out on error in hidden layer
            for (var i = 0; i < LayerSize; i++)
            {
                if (mask[i])
                {
                    Errs[i] = 0;
                }
            }
        }

        public override void Save(BinaryWriter fo)
        {
            fo.Write(LayerSize);
            fo.Write(SparseFeatureSize);
            fo.Write(DenseFeatureSize);
            fo.Write(dropoutRatio);
        }

        public override void Load(BinaryReader br, LayerType layerType, bool forTraining = false)
        {
            config = new DropoutLayerConfig();
            config.LayerSize = br.ReadInt32();

            mask = new bool[LayerSize];
            rnd = new Random();

            int sparseFeatureSize = br.ReadInt32();
            int denseFeatureSize = br.ReadInt32();
            config.DropoutRatio = br.ReadSingle();

            InitializeWeights(sparseFeatureSize, denseFeatureSize);

            if (forTraining)
            {
                InitializeInternalTrainingParameters();
            }
        }
    }
}

<|start_filename|>RNNSharp/Enums.cs<|end_filename|>
﻿namespace RNNSharp
{
    public enum TFEATURE_WEIGHT_TYPE_ENUM
    {
        BINARY,
        FREQUENCY
    }

    public enum LayerType
    {
        Softmax,
        SampledSoftmax,
        LSTM,
        DropOut,
        Simple,
        None
    }

    public enum PRETRAIN_TYPE
    {
        AutoEncoder,
        Embedding,
        None
    }

    public enum NETWORKTYPE
    {
        Forward = 0,
        ForwardSeq2Seq,
        BiDirectional,
        BiDirectionalAverage
    }

    public enum LAYERTYPE
    {
        BPTT = 0,
        LSTM
    }

    public enum RunningMode
    {
        Training = 0,
        Validate = 1,
        Test = 2
    }
}

<|start_filename|>RNNSharp/Matrix.cs<|end_filename|>
﻿
using System;
/// <summary>
/// RNNSharp written by <NAME> (<EMAIL>)
/// </summary>
namespace RNNSharp
{
    public class Matrix<T> where T : struct
    {
        private readonly T[][] m_saData;

        public Matrix(int h, int w)
        {
            Height = h;
            Width = w;
            m_saData = new T[Height][];
            for (var i = 0; i < Height; i++)
            {
                m_saData[i] = new T[Width];
            }
        }

        public int Height { get; set; } // the number of rows
        public int Width { get; set; } // the number of columns

        public T[] this[int i]
        {
            get { return m_saData[i]; }
            set { m_saData[i] = value; }
        }

        public void Clean()
        {
            for (var i = 0; i < Height; i++)
            {
                Array.Clear(m_saData[i], 0, Width);
            }
        }
    }
}

<|start_filename|>RNNSharp/RNNHelper.cs<|end_filename|>
﻿using AdvUtils;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;

namespace RNNSharp
{
    public class RNNHelper
    {
        public static Vector<float> vecMaxGrad;
        public static Vector<float> vecMinGrad;
        public static Vector<float> vecNormalLearningRate;
        public static Random rand = new Random(DateTime.Now.Millisecond);
        public static float GradientCutoff { get; set; }
        public static float LearningRate { get; set; }
        public static bool IsConstAlpha { get; set; }

        public static int MiniBatchSize { get; set; }

        private static float random(float min, float max)
        {
            return (float)(rand.NextDouble() * (max - min) + min);
        }

        public static float RandInitWeight()
        {
            return random(-0.1f, 0.1f) + random(-0.1f, 0.1f) + random(-0.1f, 0.1f);
        }

        public static float NormalizeGradient(float err)
        {
            if (err > GradientCutoff)
            {
                err = GradientCutoff;
            }
            else if (err < -GradientCutoff)
            {
                err = -GradientCutoff;
            }
            return err;
        }

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public static Vector<float> NormalizeGradient(Vector<float> v)
        {
            v = Vector.Min(v, vecMaxGrad);
            v = Vector.Max(v, vecMinGrad);

            return v;
        }

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public static float ComputeLearningRate(Matrix<float> m, int i, int j, float delta)
        {
            if (IsConstAlpha)
            {
                return LearningRate;
            }
            var dg = m[i][j] + delta * delta;
            m[i][j] = dg;

            return (float)(LearningRate / (1.0 + Math.Sqrt(dg)));
        }

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public static Vector<float> ComputeLearningRate(Vector<float> vecDelta, ref Vector<float> vecWeightLearningRate)
        {
            if (RNNHelper.IsConstAlpha)
            {
                return RNNHelper.vecNormalLearningRate;
            }

            vecWeightLearningRate += vecDelta * vecDelta;
            return vecNormalLearningRate / (Vector.SquareRoot(vecWeightLearningRate) + Vector<float>.One);

        }

        //Save matrix into file as binary format
        public static void SaveMatrix(Matrix<float> mat, BinaryWriter fo, bool bVQ = false)
        {
            //Save the width and height of the matrix
            fo.Write(mat.Width);
            fo.Write(mat.Height);

            if (bVQ == false)
            {
                Logger.WriteLine("Saving matrix without VQ...");
                fo.Write(0); // non-VQ

                //Save the data in matrix
                for (var r = 0; r < mat.Height; r++)
                {
                    for (var c = 0; c < mat.Width; c++)
                    {
                        fo.Write(mat[r][c]);
                    }
                }
            }
            else
            {
                //Build vector quantization matrix
                var vqSize = 256;
                var vq = new VectorQuantization();
                Logger.WriteLine("Saving matrix with VQ {0}...", vqSize);

                var valSize = 0;
                for (var i = 0; i < mat.Height; i++)
                {
                    for (var j = 0; j < mat.Width; j++)
                    {
                        vq.Add(mat[i][j]);
                        valSize++;
                    }
                }

                if (vqSize > valSize)
                {
                    vqSize = valSize;
                }

                var distortion = vq.BuildCodebook(vqSize);
                Logger.WriteLine("Distortion: {0}, vqSize: {1}", distortion, vqSize);

                //Save VQ codebook into file
                fo.Write(vqSize);
                for (var j = 0; j < vqSize; j++)
                {
                    fo.Write(vq.CodeBook[j]);
                }

                //Save the data in matrix
                for (var r = 0; r < mat.Height; r++)
                {
                    for (var c = 0; c < mat.Width; c++)
                    {
                        fo.Write((byte)vq.ComputeVQ(mat[r][c]));
                    }
                }
            }
        }

        public static void matrixXvectorADD(float[] dest, List<float[]> srcVecGroup, Matrix<float> srcmatrix,
            HashSet<int> setSkipSampling)
        {

            foreach (var i in setSkipSampling)
            {
                var vector_i = srcmatrix[i];
                float cellOutput = 0;
                var k = 0;
                foreach (float[] srcVec in srcVecGroup)
                {
                    var j = 0;
                    var SrcSize = srcVec.Length;
                    var moreItems = (SrcSize % Vector<float>.Count);
                    while (j < SrcSize - moreItems)
                    {
                        var v1 = new Vector<float>(srcVec, j);
                        var v2 = new Vector<float>(vector_i, k);
                        cellOutput += Vector.Dot(v1, v2);

                        j += Vector<float>.Count;
                        k += Vector<float>.Count;
                    }

                    while (j < SrcSize)
                    {
                        cellOutput += srcVec[j] * vector_i[k];
                        j++;
                        k++;
                    }
                }

                dest[i] = cellOutput;
            }
        }

        public static void matrixXvectorADD(float[] dest, List<float[]> srcVecGroup, Matrix<float> srcmatrix, int DestSize)
        {
            for (var i = 0; i < DestSize; i++)
            {
                var vector_i = srcmatrix[i];
                float cellOutput = 0;
                var k = 0;
                foreach (float[] srcVec in srcVecGroup)
                {
                    var j = 0;
                    var SrcSize = srcVec.Length;
                    var moreItems = (SrcSize % Vector<float>.Count);
                    while (j < SrcSize - moreItems)
                    {
                        var v1 = new Vector<float>(srcVec, j);
                        var v2 = new Vector<float>(vector_i, k);
                        cellOutput += Vector.Dot(v1, v2);

                        j += Vector<float>.Count;
                        k += Vector<float>.Count;
                    }

                    while (j < SrcSize)
                    {
                        cellOutput += srcVec[j] * vector_i[k];
                        j++;
                        k++;
                    }
                }

                dest[i] = cellOutput;
            }
        }

        public static Matrix<float> LoadMatrix(BinaryReader br)
        {
            var width = br.ReadInt32();
            var height = br.ReadInt32();
            var vqSize = br.ReadInt32();
            Logger.WriteLine("Loading matrix. width: {0}, height: {1}, vqSize: {2}", width, height, vqSize);

            var m = new Matrix<float>(height, width);
            if (vqSize == 0)
            {
                for (var r = 0; r < height; r++)
                {
                    for (var c = 0; c < width; c++)
                    {
                        m[r][c] = br.ReadSingle();
                    }
                }
            }
            else
            {
                var codeBook = new List<float>();

                for (var i = 0; i < vqSize; i++)
                {
                    codeBook.Add(br.ReadSingle());
                }

                for (var r = 0; r < height; r++)
                {
                    for (var c = 0; c < width; c++)
                    {
                        int vqIndex = br.ReadByte();
                        m[r][c] = codeBook[vqIndex];
                    }
                }
            }

            return m;
        }

        public static void matrixXvectorADDErr(List<float[]> destList, float[] srcvec, Matrix<float> srcmatrix, bool cleanDest = true)
        {
            if (cleanDest == true)
            {
                foreach (float[] dest in destList)
                {
                    Array.Clear(dest, 0, dest.Length);
                }
            }

            for (var j = 0; j < srcmatrix.Height; j++)
            {
                int i = 0;
                float src = srcvec[j];
                float[] srcVector = srcmatrix[j];

                foreach (float[] dest in destList)
                {
                    int k = 0;
                    var moreItems = (dest.Length % Vector<float>.Count);
                    while (k < dest.Length - moreItems)
                    {
                        Vector<float> vecSrc = new Vector<float>(srcVector, i);
                        Vector<float> vecDest = new Vector<float>(dest, k);
                        vecDest += src * vecSrc;

                        vecDest.CopyTo(dest, k);

                        i += Vector<float>.Count;
                        k += Vector<float>.Count;
                    }

                    while (k < dest.Length)
                    {
                        dest[k] += src * srcVector[i];
                        i++;
                        k++;
                    }
                }
                
            }
        }

        public static void matrixXvectorADDErr(float[] dest, float[] srcvec, Matrix<float> srcmatrix, bool cleanDest = true)
        {
            if (cleanDest == true)
            {
                Array.Clear(dest, 0, dest.Length);
            }
            for (var j = 0; j < srcmatrix.Height; j++)
            {
                int i = 0;
                float src = srcvec[j];
                float[] srcVector = srcmatrix[j];

                var moreItems = (srcmatrix.Width % Vector<float>.Count);
                while (i < srcmatrix.Width - moreItems)
                {
                    Vector<float> vecSrc = new Vector<float>(srcVector, i);
                    Vector<float> vecDest = new Vector<float>(dest, i);
                    vecDest += src * vecSrc;

                    vecDest.CopyTo(dest, i);
                    i += Vector<float>.Count;
                }

                while (i < srcmatrix.Width)
                {
                    dest[i] += src * srcVector[i];
                    i++;
                }
            }
        }

        public static void matrixXvectorADDErr(float[] dest, float[] srcvec, Matrix<float> srcmatrix, HashSet<int> setSkipSampling, bool cleanDest = true)
        {
            if (cleanDest == true)
            {
                Array.Clear(dest, 0, dest.Length);
            }
            int cnt = 0;
            foreach (int j in setSkipSampling)
            {
                int i = 0;
                float src = srcvec[j];
                float[] srcVector = srcmatrix[j];
                int weight = srcVector.Length;

                var moreItems = (weight % Vector<float>.Count);
                while (i < weight - moreItems)
                {
                    Vector<float> vecSrc = new Vector<float>(srcVector, i);
                    Vector<float> vecDest = new Vector<float>(dest, i);
                    vecDest += src * vecSrc;

                    vecDest.CopyTo(dest, i);
                    i += Vector<float>.Count;
                }

                while (i < weight)
                {
                    dest[i] += src * srcVector[i];
                    i++;
                }

                cnt++;
            }
        }
    }
}

<|start_filename|>RNNSharpConsole/Program.cs<|end_filename|>
﻿using AdvUtils;
using RNNSharp;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;

namespace RNNSharpConsole
{
    /// <summary>
    ///     RNNSharp written by <NAME> (<EMAIL>)
    /// </summary>
    internal class Program
    {
        private static string testFilePath = "";
        private static string outputFilePath = "";
        private static string tagFilePath = "";
        private static string configFilePath = "";
        private static string trainFilePath = "";
        private static string validFilePath = "";
        private static bool incrementalTrain = false;
        private static int maxIter = 20;
        private static long savestep;
        private static float alpha = 0.1f;
        private static int nBest = 1;
        private static int iVQ;
        private static float gradientCutoff = 1.0f;
        private static bool constAlpha;
        private static int miniBatchSize = 1;

        private static void UsageTitle()
        {
            Console.WriteLine("Deep Recurrent Neural Network Toolkit v2.3.0 by <NAME> (<EMAIL>)");
        }

        private static void Usage()
        {
            UsageTitle();
            Console.WriteLine("RNNSharpConsole.exe <parameters>");
            Console.WriteLine(" -mode <string>: train/test");
        }

        private static void UsageTrain()
        {
            UsageTitle();
            Console.WriteLine("RNNSharpConsole.exe -mode train <parameters>");
            Console.WriteLine("Parameters for Deep-RNN training");

            Console.WriteLine(" -trainfile <string>");
            Console.WriteLine("\tTraining corpus file");

            Console.WriteLine(" -validfile <string>");
            Console.WriteLine("\tValidated corpus file for training");

            Console.WriteLine(" -cfgfile <string>");
            Console.WriteLine("\tConfiguration file");

            Console.WriteLine(" -tagfile <string>");
            Console.WriteLine("\tSupported output tagid-name list file");

            Console.WriteLine(" -inctrain <boolean>");
            Console.WriteLine("\tincremental training. Starting from output model specified in configuration file. Default is false");

            Console.WriteLine(" -alpha <float>");
            Console.WriteLine("\tInitializing learning rate. Default is 0.1");

            Console.WriteLine(" -constalpha <boolean>");
            Console.WriteLine("\tUse const learning rat. Default is false");

            Console.WriteLine(" -maxiter <int>");
            Console.WriteLine("\tMaximum iteration for training, 0 is unlimited. Default is 20");

            Console.WriteLine(" -savestep <int>");
            Console.WriteLine("\tSave temporary model after every <int> sentences. Default is 0");

            Console.WriteLine(" -vq <int>");
            Console.WriteLine("\tModel vector quantization, 0 is disable, 1 is enable. Default is 0");

            Console.WriteLine(" -grad <float>");
            Console.WriteLine("\tGradient cut-off. Default is 1.0f");

            Console.WriteLine(" -minibatch <int>");
            Console.WriteLine("\tUpdating weights every <int> sequences. Default is 1");

            Console.WriteLine();
            Console.WriteLine(
                "Example: RNNSharpConsole.exe -mode train -trainfile train.txt -validfile valid.txt -cfgfile config.txt -tagfile tags.txt -alpha 0.1 -maxiter 20 -savestep 200K");
        }

        private static void UsageTest()
        {
            UsageTitle();
            Console.WriteLine("RNNSharpConsole.exe -mode test <parameters>");
            Console.WriteLine("Parameters to predict tags from given corpus by using trained model");
            Console.WriteLine(" -testfile <string>");
            Console.WriteLine("\tInput file for testing");

            Console.WriteLine(" -tagfile <string>");
            Console.WriteLine("\tSupported output tagid-name list file");

            Console.WriteLine(" -cfgfile <string>");
            Console.WriteLine("\tConfiguration file");

            Console.WriteLine(" -outfile <string>");
            Console.WriteLine("\tResult output file");

            Console.WriteLine(" -nbest <int>");
            Console.WriteLine("\tGet N-best result. Default is 1");

            Console.WriteLine();
            Console.WriteLine(
                "Example: RNNSharpConsole.exe -mode test -testfile test.txt -modelfile model.bin -tagfile tags.txt -ftrfile features.txt -outfile result.txt -nbest 3");
        }

        private static void InitParameters(string[] args)
        {
            int i;
            if ((i = ArgPos("-trainfile", args)) >= 0) trainFilePath = args[i + 1];
            if ((i = ArgPos("-validfile", args)) >= 0) validFilePath = args[i + 1];
            if ((i = ArgPos("-testfile", args)) >= 0) testFilePath = args[i + 1];
            if ((i = ArgPos("-outfile", args)) >= 0) outputFilePath = args[i + 1];
            if ((i = ArgPos("-tagfile", args)) >= 0) tagFilePath = args[i + 1];
            if ((i = ArgPos("-cfgfile", args)) >= 0) configFilePath = args[i + 1];
            if ((i = ArgPos("-inctrain", args)) >= 0) incrementalTrain = bool.Parse(args[i + 1]);
            if ((i = ArgPos("-maxiter", args)) >= 0) maxIter = int.Parse(args[i + 1], CultureInfo.InvariantCulture);
            if ((i = ArgPos("-alpha", args)) >= 0) alpha = float.Parse(args[i + 1], CultureInfo.InvariantCulture);
            if ((i = ArgPos("-nbest", args)) >= 0) nBest = int.Parse(args[i + 1], CultureInfo.InvariantCulture);
            if ((i = ArgPos("-vq", args)) >= 0) iVQ = int.Parse(args[i + 1], CultureInfo.InvariantCulture);
            if ((i = ArgPos("-grad", args)) >= 0) gradientCutoff = float.Parse(args[i + 1], CultureInfo.InvariantCulture);
            if ((i = ArgPos("-constalpha", args)) >= 0) constAlpha = bool.Parse(args[i + 1]);
            if ((i = ArgPos("-minibatch", args)) >= 0) miniBatchSize = int.Parse(args[i + 1], CultureInfo.InvariantCulture);

            if ((i = ArgPos("-savestep", args)) >= 0)
            {
                var str = args[i + 1].ToLower();
                if (str.EndsWith("k"))
                {
                    savestep = long.Parse(str.Substring(0, str.Length - 1), CultureInfo.InvariantCulture) * 1024;
                }
                else if (str.EndsWith("m"))
                {
                    savestep = long.Parse(str.Substring(0, str.Length - 1), CultureInfo.InvariantCulture) * 1024 * 1024;
                }
                else if (str.EndsWith("g"))
                {
                    savestep = long.Parse(str.Substring(0, str.Length - 1), CultureInfo.InvariantCulture) * 1024 * 1024 * 1024;
                }
                else
                {
                    savestep = long.Parse(str, CultureInfo.InvariantCulture);
                }
            }
        }

        //Parse parameters in command line
        private static int ArgPos(string str, string[] args)
        {
            str = str.ToLower();
            for (var a = 0; a < args.Length; a++)
            {
                if (str == args[a].ToLower())
                {
                    if (a == args.Length - 1)
                    {
                        Logger.WriteLine("Argument missing for {0}", str);
                        return -1;
                    }
                    return a;
                }
            }
            return -1;
        }

        //Check if the corpus is validated and get the number of record in total
        private static void CheckCorpus(string strFileName)
        {
            var sr = new StreamReader(strFileName);
            var RecordCount = 0;

            var tokenList = new List<string>();
            while (true)
            {
                var strLine = sr.ReadLine();
                if (strLine == null)
                {
                    break;
                }

                strLine = strLine.Trim();
                if (strLine.Length == 0)
                {
                    RecordCount++;
                }
            }

            Logger.WriteLine("Record set size in {0}: {1}", strFileName, RecordCount);
            sr.Close();
        }

        private static void LoadSeq2SeqDataSet(string strFileName, Config featurizer, DataSet<SequencePair> dataSet)
        {
            Logger.WriteLine("Loading data set for seq2seq2 training...");
            var sr = new StreamReader(strFileName);
            var RecordCount = 0;
            while (true)
            {
                var sentPair = new SentencePair
                {
                    srcSentence = new Sentence(ReadRecord(sr)),
                    tgtSentence = new Sentence(ReadRecord(sr), false)
                };

                //Extract features from it and convert it into sequence

                if (sentPair.srcSentence.TokensList.Count <= 2 || sentPair.tgtSentence.TokensList.Count <= 0)
                {
                    //No more record, it only contain <s> and </s>
                    break;
                }

                var seq = featurizer.BuildSequence(sentPair);
                if (seq.tgtSequence.SetLabel(sentPair.tgtSentence, featurizer.TagSet))
                {
                    dataSet.SequenceList.Add(seq);

                    //Show state at every 1000 record
                    RecordCount++;
                    if (RecordCount % 10000 == 0)
                    {
                        Logger.WriteLine("{0}...", RecordCount);
                    }
                }
            }

            sr.Close();
        }

        private static void LoadDataset(string strFileName, Config featurizer, DataSet<Sequence> dataSet)
        {
            Logger.WriteLine("Loading data set...");
            CheckCorpus(strFileName);

            var sr = new StreamReader(strFileName);
            var RecordCount = 0;

            while (true)
            {
                try
                {
                    //Extract features from it and convert it into sequence
                    var sent = new Sentence(ReadRecord(sr));
                    if (sent.TokensList.Count <= 2)
                    {
                        //No more record, it only contain <s> and </s>
                        break;
                    }

                    var seq = featurizer.BuildSequence(sent);

                    //Set label for the sequence
                    if (seq.SetLabel(sent, featurizer.TagSet))
                    {
                        //Add the sequence into data set
                        dataSet.SequenceList.Add(seq);

                        //Show state at every 1000 record
                        RecordCount++;
                        if (RecordCount % 10000 == 0)
                        {
                            Logger.WriteLine("{0}...", RecordCount);
                        }
                    }
                }
                catch (Exception err)
                {
                    Logger.WriteLine($"Fail to parse corpus: '{err.Message}'");
                    Logger.WriteLine($"Stack trace: '{err.StackTrace}'");

                    throw new InvalidDataException("Invalidated corpus.", err);
                }
            }

            sr.Close();
        }

        private static void Main(string[] args)
        {
            int i;

            //Initialize all given parameters
            InitParameters(args);

            if ((i = ArgPos("-mode", args)) >= 0)
            {
                var strMode = args[i + 1].ToLower();
                switch (strMode)
                {
                    case "train":
                        Train();
                        break;

                    case "test":
                        Test();
                        break;
                }
            }
            else
            {
                Logger.WriteLine(Logger.Level.err, "Running mode is required.");
                Usage();
            }
        }

        private static List<string[]> ReadRecord(StreamReader sr)
        {
            var record = new List<string[]>();
            string strLine;

            //Read each line from file
            while ((strLine = sr.ReadLine()) != null)
            {
                strLine = strLine.Trim();
                if (strLine.Length == 0)
                {
                    //If read a empty line, it should be the end of current record
                    return record;
                }

                record.Add(strLine.Split('\t'));
            }

            return record;
        }

        private static void Test()
        {
            if (string.IsNullOrEmpty(tagFilePath))
            {
                Logger.WriteLine(Logger.Level.err, "FAILED: The tag mapping file {0} isn't specified.", tagFilePath);
                UsageTest();
                return;
            }

            //Load tag name
            Logger.WriteLine($"Loading tag file '{tagFilePath}'");
            var tagSet = new TagSet(tagFilePath);

            if (string.IsNullOrEmpty(configFilePath))
            {
                Logger.WriteLine(Logger.Level.err, "FAILED: The configuration file {0} isn't specified.", configFilePath);
                UsageTest();
                return;
            }

            if (outputFilePath.Length == 0)
            {
                Logger.WriteLine(Logger.Level.err, "FAILED: The output file name should not be empty.");
                UsageTest();
                return;
            }

            //Create feature extractors and load word embedding data from file
            Logger.WriteLine($"Initializing config file = '{configFilePath}'");
            var config = new Config(configFilePath, tagSet);
            config.ShowFeatureSize();

            //Create instance for decoder
            Logger.WriteLine($"Loading model from {config.ModelFilePath} and creating decoder instance...");
            var decoder = new RNNDecoder(config);

            if (File.Exists(testFilePath) == false)
            {
                Logger.WriteLine(Logger.Level.err, $"FAILED: The test corpus {testFilePath} doesn't exist.");
                UsageTest();
                return;
            }

            var sr = new StreamReader(testFilePath);
            var sw = new StreamWriter(outputFilePath);

            while (true)
            {
                var sent = new Sentence(ReadRecord(sr));
                if (sent.TokensList.Count <= 2)
                {
                    //No more record, it only contains <s> and </s>
                    break;
                }

                if (nBest == 1)
                {
                    //Output decoded result
                    var output = decoder.Process(sent);
                    if (decoder.NetworkType != NETWORKTYPE.ForwardSeq2Seq)
                    {
                        //Append the decoded result into the end of feature set of each token
                        if (output != null)
                        {
                            var sb = new StringBuilder();
                            for (var i = 0; i < sent.TokensList.Count; i++)
                            {
                                var tokens = string.Join("\t", sent.TokensList[i]);
                                sb.Append(tokens);
                                sb.Append("\t");
                                sb.Append(tagSet.GetTagName(output[i]));
                                sb.AppendLine();
                            }
                            sw.WriteLine(sb.ToString());
                        }
                    }
                    else
                    {
                        //Print out source sentence at first, and then generated result sentence
                        var sb = new StringBuilder();
                        for (var i = 0; i < sent.TokensList.Count; i++)
                        {
                            var tokens = string.Join("\t", sent.TokensList[i]);
                            sb.AppendLine(tokens);
                        }
                        sw.WriteLine(sb.ToString());
                        sw.WriteLine();

                        sb.Clear();
                        for (var i = 0; i < output.Length; i++)
                        {
                            var token = tagSet.GetTagName(output[i]);
                            sb.AppendLine(token);
                        }
                        sw.WriteLine(sb.ToString());
                        sw.WriteLine();
                    }
                }
                else
                {
                    var output = decoder.ProcessNBest(sent, nBest);
                    var sb = new StringBuilder();
                    for (var i = 0; i < nBest; i++)
                    {
                        for (var j = 0; j < sent.TokensList.Count; j++)
                        {
                            var tokens = string.Join("\t", sent.TokensList[i]);
                            sb.Append(tokens);
                            sb.Append("\t");
                            sb.Append(tagSet.GetTagName(output[i][j]));
                            sb.AppendLine();
                        }
                        sb.AppendLine();
                    }

                    sw.WriteLine(sb.ToString());
                }
            }

            sr.Close();
            sw.Close();
        }

        public static LayerType ParseLayerType(string layerType)
        {
            foreach (
                var type in
                    Enum.GetValues(typeof(LayerType))
                        .Cast<LayerType>()
                        .Where(type => layerType.Equals(type.ToString(), StringComparison.InvariantCultureIgnoreCase)))
            {
                return type;
            }

            throw new ArgumentException($"Invalidated layer type: {layerType}");
        }

        private static void Train()
        {
            Logger.LogFile = "RNNSharpConsole.log";

            if (File.Exists(tagFilePath) == false)
            {
                Logger.WriteLine(Logger.Level.err, $"FAILED: The tag mapping file {tagFilePath} doesn't exist.");
                UsageTrain();
                return;
            }

            //Load tag id and its name from file
            var tagSet = new TagSet(tagFilePath);

            //Create configuration instance and set parameters
            var RNNConfig = new ModelSetting
            {
                VQ = iVQ,
                MaxIteration = maxIter,
                SaveStep = savestep,
                LearningRate = alpha,
                GradientCutoff = gradientCutoff,
                IsConstAlpha = constAlpha,
                IncrementalTrain = incrementalTrain,
                MiniBatchSize = miniBatchSize
            };

            //Dump RNN setting on console
            RNNConfig.DumpSetting();

            if (File.Exists(configFilePath) == false)
            {
                Logger.WriteLine(Logger.Level.err, "FAILED: The feature configuration file {0} doesn't exist.",
                    configFilePath);
                UsageTrain();
                return;
            }
            //Create feature extractors and load word embedding data from file
            var config = new Config(configFilePath, tagSet);
            config.ShowFeatureSize();

            if (File.Exists(trainFilePath) == false)
            {
                Logger.WriteLine(Logger.Level.err, "FAILED: The training corpus doesn't exist.");
                UsageTrain();
                return;
            }

            if (config.NetworkType != NETWORKTYPE.ForwardSeq2Seq)
            {
                //Create RNN encoder and save necessary parameters
                var encoder = new RNNEncoder<Sequence>(RNNConfig, config)
                {
                    TrainingSet = new DataSet<Sequence>(tagSet.GetSize())
                };

                //LoadFeatureConfig training corpus and extract feature set
                LoadDataset(trainFilePath, config, encoder.TrainingSet);

                if (string.IsNullOrEmpty(validFilePath) == false)
                {
                    //LoadFeatureConfig validated corpus and extract feature set
                    Logger.WriteLine("Loading validated corpus from {0}", validFilePath);
                    encoder.ValidationSet = new DataSet<Sequence>(tagSet.GetSize());
                    LoadDataset(validFilePath, config, encoder.ValidationSet);
                }
                else
                {
                    Logger.WriteLine("Validated corpus isn't specified.");
                    encoder.ValidationSet = null;
                }

                if (encoder.IsCRFTraining)
                {
                    Logger.WriteLine("Initialize output tag bigram transition probability...");
                    //Build tag bigram transition matrix
                    encoder.TrainingSet.BuildLabelBigramTransition();
                }

                //Start to train the model
                encoder.Train();
            }
            else
            {
                //Create RNN encoder and save necessary parameters
                var encoder = new RNNEncoder<SequencePair>(RNNConfig, config)
                {
                    TrainingSet = new DataSet<SequencePair>(tagSet.GetSize())
                };

                //LoadFeatureConfig training corpus and extract feature set

                LoadSeq2SeqDataSet(trainFilePath, config, encoder.TrainingSet);

                if (string.IsNullOrEmpty(validFilePath) == false)
                {
                    //LoadFeatureConfig validated corpus and extract feature set
                    Logger.WriteLine("Loading validated corpus from {0}", validFilePath);
                    encoder.ValidationSet = new DataSet<SequencePair>(tagSet.GetSize());
                    LoadSeq2SeqDataSet(validFilePath, config, encoder.ValidationSet);
                }
                else
                {
                    Logger.WriteLine("Validated corpus isn't specified.");
                    encoder.ValidationSet = null;
                }

                if (encoder.IsCRFTraining)
                {
                    Logger.WriteLine("Initialize output tag bigram transition probability...");
                    //Build tag bigram transition matrix
                    encoder.TrainingSet.BuildLabelBigramTransition();
                }

                //Start to train the model
                encoder.Train();
            }
        }
    }
}

<|start_filename|>RNNSharp/Layers/LSTMLayer.cs<|end_filename|>
﻿using AdvUtils;
using RNNSharp.Layers;
using System;
using System.Collections.Generic;
using System.IO;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;

namespace RNNSharp
{

    public class LSTMGateWeight
    {
        public float[][] weights;
        public float[][] weightsDelta;

        public float[][] deri;
        public float[][] learningRate;
        public int layerSize;
        public int denseFeatureSize;
        protected ParallelOptions parallelOption = new ParallelOptions();

        public LSTMGateWeight CloneSharedWeights()
        {
            LSTMGateWeight gateWeight = new LSTMGateWeight();
            gateWeight.InitWeights(layerSize, denseFeatureSize);
            gateWeight.weights = weights;
            gateWeight.weightsDelta = weightsDelta;
            gateWeight.learningRate = learningRate;

            return gateWeight;
        }

        /// <summary>
        /// Update weights
        /// </summary>
        public void UpdateWeights()
        {
            Vector<float> vecMiniBatchSize = new Vector<float>(RNNHelper.MiniBatchSize);
            for (var i = 0; i < layerSize; i++)
            {
                var j = 0;
                var weights_i = weights[i];
                var weightsDelta_i = weightsDelta[i];
                var learningrate_i = learningRate[i];
                var moreItems = (denseFeatureSize % Vector<float>.Count);
                while (j < denseFeatureSize - moreItems)
                {
                    //Vectorize weights delta
                    Vector<float> vecDelta = new Vector<float>(weightsDelta_i, j);
                    Vector<float>.Zero.CopyTo(weightsDelta_i, j);

                    //Normalize weights delta
                    vecDelta = vecDelta / vecMiniBatchSize;
                    vecDelta = RNNHelper.NormalizeGradient(vecDelta);

                    //Get learning rate dymanticly
                    var wlr_i = new Vector<float>(learningrate_i, j);
                    var vecLearningRate = RNNHelper.ComputeLearningRate(vecDelta, ref wlr_i);
                    wlr_i.CopyTo(learningrate_i, j);

                    //Update weights
                    Vector<float> vecWeights = new Vector<float>(weights_i, j);
                    vecWeights += vecLearningRate * vecDelta;
                    vecWeights.CopyTo(weights_i, j);

                    j += Vector<float>.Count;
                }

                while (j < denseFeatureSize)
                {
                    var delta = weightsDelta_i[j];
                    weightsDelta_i[j] = 0;

                    delta = delta / RNNHelper.MiniBatchSize;
                    delta = RNNHelper.NormalizeGradient(delta);

                    var wlr_i = learningrate_i[j];
                    var learningRate = ComputeLearningRate(delta, ref wlr_i);
                    learningrate_i[j] = wlr_i;

                    weights_i[j] += learningRate * delta;
                    j++;
                }
            }

        }

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        private float ComputeLearningRate(float delta, ref float weightLearningRate)
        {
            if (RNNHelper.IsConstAlpha)
            {
                return RNNHelper.LearningRate;
            }

            weightLearningRate += delta * delta;
            return (float)(RNNHelper.LearningRate / (Math.Sqrt(weightLearningRate) + 1.0));
        }

        public void CleanStatusForTraining()
        {
            if (denseFeatureSize > 0)
            {
                for (var i = 0; i < layerSize; i++)
                {
                    Array.Clear(learningRate[i], 0, learningRate[i].Length);
                    Array.Clear(weightsDelta[i], 0, weightsDelta[i].Length);
                }
            }
        }

        public void InitWeights(int LayerSize, int DenseFeatureSize)
        {
            layerSize = LayerSize;
            denseFeatureSize = DenseFeatureSize;

            weights = new float[layerSize][];
            for (var i = 0; i < layerSize; i++)
            {
                weights[i] = new float[denseFeatureSize];            
                for (var j = 0; j < denseFeatureSize; j++)
                {
                    weights[i][j] = RNNHelper.RandInitWeight();
                }
            }

        }

        public void InitInternaTrainingParameters(bool hasDeri = true)
        {
            if (learningRate == null)
            {
                learningRate = new float[layerSize][];
                weightsDelta = new float[layerSize][];
                for (var i = 0; i < layerSize; i++)
                {
                    learningRate[i] = new float[denseFeatureSize];
                    weightsDelta[i] = new float[denseFeatureSize];
                }
            }

            if (hasDeri)
            {
                deri = new float[layerSize][];
                for (var i = 0; i < layerSize; i++)
                {
                    deri[i] = new float[denseFeatureSize];
                }
            }
        }
    }

    public class LSTMNeuron : Neuron
    {
        public LSTMCell[] LSTMCells;

        public LSTMNeuron(int length) : base(length)
        {
            LSTMCells = new LSTMCell[length];
            LSTMCells = new LSTMCell[length];

            for (int k = 0; k < length; k++)
            {
                LSTMCells[k] = new LSTMCell();
                LSTMCells[k] = new LSTMCell();
            }
        }
    }

    public class LSTMLayer : ILayer
    {
        LSTMCell[] LSTMCells;
        LSTMCellWeight[] CellWeights;
        LSTMCellWeightDeri[] CellWeightsDeri;
        Vector4[] cellLearningRate;
        Vector3[] peepholeLearningRate;

        Vector4[] cellDelta;
        Vector3[] peepholeDelta;

        //Due to different data access patterns(random access for sparse features and continuous access for dense features), 
        //we use different data structure to keep features weights in order to improve performance 
        LSTMGateWeight wDenseInputGate;
        LSTMGateWeight wDenseForgetGate;
        LSTMGateWeight wDenseCellGate;
        LSTMGateWeight wDenseOutputGate;

        //X - wInputInputGate
        //Y - wInputForgetGate
        //Z - wInputCell
        //W - wInputOutputGate
        Vector4[][] sparseFeatureWeights;
        Vector4[][] sparseFeatureLearningRate;
        Dictionary<int, Vector3>[] sparseFeatureToHiddenDeri;

        Vector4[][] sparseFeatureWeightsDelta;

        Vector4 vecMaxGrad;
        Vector3 vecMaxGrad3;
        Vector4 vecMinGrad;
        Vector3 vecMinGrad3;

        Vector4 vecNormalLearningRate;
        Vector3 vecNormalLearningRate3;

        LSTMLayerConfig config;

        public float[] Cells { get; set; }
        public float[] Errs { get; set; }
        public int SparseFeatureSize { get; set; }
        public int DenseFeatureSize { get; set; }
        public int LayerSize
        {
            get { return config.LayerSize; }
            set { config.LayerSize = value; }
        }

        public LayerType LayerType
        {
            get { return config.LayerType; }
            set { config.LayerType = value; }
        }

        public LayerConfig LayerConfig { get; set; }

        public List<SparseVector> SparseFeatureGroups { get; set; }
        public List<float[]> DenseFeatureGroups { get; set; }

        public LSTMLayer() { }
        public LSTMLayer(LSTMLayerConfig config)
        {
            this.config = config;

            Cells = new float[LayerSize];
            Errs = new float[LayerSize];

            LSTMCells = new LSTMCell[LayerSize];
            for (var i = 0; i < LayerSize; i++)
            {
                LSTMCells[i] = new LSTMCell();
            }
        }

        public ILayer CreateLayerSharedWegiths()
        {
            LSTMLayer layer = new LSTMLayer(config);
            ShallowCopyWeightTo(layer);

            return layer;
        }

        public Neuron CopyNeuronTo(Neuron neuron)
        {
            LSTMNeuron lstmNeuron = neuron as LSTMNeuron;
            Cells.CopyTo(lstmNeuron.Cells, 0);
            for (int i = 0; i < LayerSize; i++)
            {
                lstmNeuron.LSTMCells[i].Set(LSTMCells[i]);
            }

            return lstmNeuron;
        }

        public void SetNeuron(Neuron neuron)
        {
            LSTMNeuron lstmNeuron = neuron as LSTMNeuron;
            for (int i = 0; i < LayerSize; i++)
            {
                LSTMCells[i].Set(lstmNeuron.LSTMCells[i]);
            }

            //  neuron.Errs.CopyTo(Errs, 0);

            Errs = neuron.Errs;
        }

        public void ShallowCopyWeightTo(ILayer destLayer)
        {
            LSTMLayer layer = destLayer as LSTMLayer;
            layer.SparseFeatureSize = SparseFeatureSize;
            layer.DenseFeatureSize = DenseFeatureSize;

            layer.sparseFeatureWeights = sparseFeatureWeights;
            layer.sparseFeatureWeightsDelta = sparseFeatureWeightsDelta;
            layer.sparseFeatureLearningRate = sparseFeatureLearningRate;

            layer.wDenseCellGate = wDenseCellGate.CloneSharedWeights();
            layer.wDenseForgetGate = wDenseForgetGate.CloneSharedWeights();
            layer.wDenseInputGate = wDenseInputGate.CloneSharedWeights();
            layer.wDenseOutputGate = wDenseOutputGate.CloneSharedWeights();

            layer.CellWeights = CellWeights;
            layer.cellDelta = cellDelta;
            layer.peepholeDelta = peepholeDelta;

            layer.InitializeInternalTrainingParameters();
        }

        public void InitializeInternalTrainingParameters()
        {
            if (SparseFeatureSize > 0)
            {
                sparseFeatureToHiddenDeri = new Dictionary<int, Vector3>[LayerSize];
                for (var i = 0; i < LayerSize; i++)
                {
                    sparseFeatureToHiddenDeri[i] = new Dictionary<int, Vector3>();
                }

                if (sparseFeatureLearningRate == null)
                {
                    sparseFeatureLearningRate = new Vector4[LayerSize][];
                    sparseFeatureWeightsDelta = new Vector4[LayerSize][];
                    for (var i = 0; i < LayerSize; i++)
                    {
                        sparseFeatureLearningRate[i] = new Vector4[SparseFeatureSize];
                        sparseFeatureWeightsDelta[i] = new Vector4[SparseFeatureSize];
                    }
                }
            }

            if (DenseFeatureSize > 0)
            {
                wDenseInputGate.InitInternaTrainingParameters();
                wDenseForgetGate.InitInternaTrainingParameters();
                wDenseCellGate.InitInternaTrainingParameters();
                wDenseOutputGate.InitInternaTrainingParameters(false);
            }

            cellDelta = new Vector4[LayerSize];
            peepholeDelta = new Vector3[LayerSize];
            CellWeightsDeri = new LSTMCellWeightDeri[LayerSize];
            for (var i = 0; i < LayerSize; i++)
            {
                CellWeightsDeri[i] = new LSTMCellWeightDeri();
            }

        }


        public void InitializeWeights(int sparseFeatureSize, int denseFeatureSize)
        {
            SparseFeatureSize = sparseFeatureSize;
            DenseFeatureSize = denseFeatureSize;

            InitializeCellWeights(null);

            if (SparseFeatureSize > 0)
            {
                sparseFeatureWeights = new Vector4[LayerSize][];
                for (var i = 0; i < LayerSize; i++)
                {
                    sparseFeatureWeights[i] = new Vector4[SparseFeatureSize];
                    for (var j = 0; j < SparseFeatureSize; j++)
                    {
                        sparseFeatureWeights[i][j] = InitializeLSTMWeight();
                    }
                }
            }

            if (DenseFeatureSize > 0)
            {
                wDenseInputGate = new LSTMGateWeight();
                wDenseForgetGate = new LSTMGateWeight();
                wDenseCellGate = new LSTMGateWeight();
                wDenseOutputGate = new LSTMGateWeight();
                wDenseInputGate.InitWeights(LayerSize, DenseFeatureSize);
                wDenseForgetGate.InitWeights(LayerSize, DenseFeatureSize);
                wDenseCellGate.InitWeights(LayerSize, DenseFeatureSize);
                wDenseOutputGate.InitWeights(LayerSize, DenseFeatureSize);
            }

            InitializeInternalTrainingParameters();

            Logger.WriteLine(
                "Initializing weights, sparse feature size: {0}, dense feature size: {1}, random value is {2}",
                SparseFeatureSize, DenseFeatureSize, RNNHelper.rand.NextDouble());
        }

        private void InitializeCellWeights(BinaryReader br)
        {
            CellWeights = new LSTMCellWeight[LayerSize];
            if (br != null)
            {
                //Load weight from input file
                for (var i = 0; i < LayerSize; i++)
                {
                    CellWeights[i] = new LSTMCellWeight();
                    CellWeights[i].wPeepholeIn = br.ReadDouble();
                    CellWeights[i].wPeepholeForget = br.ReadDouble();
                    CellWeights[i].wPeepholeOut = br.ReadDouble();

                    CellWeights[i].wCellIn = br.ReadDouble();
                    CellWeights[i].wCellForget = br.ReadDouble();
                    CellWeights[i].wCellState = br.ReadDouble();
                    CellWeights[i].wCellOut = br.ReadDouble();
                }
            }
            else
            {
                //Initialize weight by random number
                for (var i = 0; i < LayerSize; i++)
                {
                    CellWeights[i] = new LSTMCellWeight();
                    //internal weights, also important
                    CellWeights[i].wPeepholeIn = RNNHelper.RandInitWeight();
                    CellWeights[i].wPeepholeForget = RNNHelper.RandInitWeight();
                    CellWeights[i].wPeepholeOut = RNNHelper.RandInitWeight();

                    CellWeights[i].wCellIn = RNNHelper.RandInitWeight();
                    CellWeights[i].wCellForget = RNNHelper.RandInitWeight();
                    CellWeights[i].wCellState = RNNHelper.RandInitWeight();
                    CellWeights[i].wCellOut = RNNHelper.RandInitWeight();
                }
            }
        }

        private Vector4 InitializeLSTMWeight()
        {
            Vector4 w;

            //initialise each weight to random value
            w.X = RNNHelper.RandInitWeight();
            w.Y = RNNHelper.RandInitWeight();
            w.Z = RNNHelper.RandInitWeight();
            w.W = RNNHelper.RandInitWeight();

            return w;
        }

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        private double TanH(double x)
        {
            return Math.Tanh(x);
        }

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        private double TanHDerivative(double x)
        {
            var tmp = Math.Tanh(x);
            return 1 - tmp * tmp;
        }

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        private double Sigmoid(double x)
        {
            return 1.0 / (1.0 + Math.Exp(-x));
        }

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        private double SigmoidDerivative(double x)
        {
            var sigmoid = Sigmoid(x);
            return sigmoid * (1.0 - sigmoid);
        }

        private void SaveCellWeights(BinaryWriter fo)
        {
            for (var i = 0; i < LayerSize; i++)
            {
                fo.Write(CellWeights[i].wPeepholeIn);
                fo.Write(CellWeights[i].wPeepholeForget);
                fo.Write(CellWeights[i].wPeepholeOut);

                fo.Write(CellWeights[i].wCellIn);
                fo.Write(CellWeights[i].wCellForget);
                fo.Write(CellWeights[i].wCellState);
                fo.Write(CellWeights[i].wCellOut);
            }
        }

        private void SaveLSTMweights(LSTMGateWeight gateWeight, BinaryWriter fo, bool bVQ = false)
        {
            float[][] weights = gateWeight.weights;
            var w = weights.Length;
            var h = weights[0].Length;

            Logger.WriteLine($"Saving LSTM gate weight matrix. width: {w}, height: {h}");

            fo.Write(w);
            fo.Write(h);

            fo.Write(0);

            for (var i = 0; i < w; i++)
            {
                for (var j = 0; j < h; j++)
                {
                    fo.Write(weights[i][j]);
                }
            }
        }

        private void SaveLSTMWeights(Vector4[][] weight, BinaryWriter fo, bool bVQ = false)
        {
            var w = weight.Length;
            var h = weight[0].Length;
            var vqSize = 256;

            Logger.WriteLine("Saving LSTM weight matrix. width:{0}, height:{1}, vq:{2}", w, h, bVQ);

            fo.Write(weight.Length);
            fo.Write(weight[0].Length);

            if (bVQ == false)
            {
                fo.Write(0);

                for (var i = 0; i < w; i++)
                {
                    for (var j = 0; j < h; j++)
                    {
                        fo.Write(weight[i][j].X);
                        fo.Write(weight[i][j].Y);
                        fo.Write(weight[i][j].Z);
                        fo.Write(weight[i][j].W);
                    }
                }
            }
            else
            {
                //Build vector quantization model
                var vqInputCell = new VectorQuantization();
                var vqInputForgetGate = new VectorQuantization();
                var vqInputInputGate = new VectorQuantization();
                var vqInputOutputGate = new VectorQuantization();
                for (var i = 0; i < w; i++)
                {
                    for (var j = 0; j < h; j++)
                    {
                        vqInputInputGate.Add(weight[i][j].X);
                        vqInputForgetGate.Add(weight[i][j].Y);
                        vqInputCell.Add(weight[i][j].Z);
                        vqInputOutputGate.Add(weight[i][j].W);
                    }
                }

                var distortion = vqInputInputGate.BuildCodebook(vqSize);
                Logger.WriteLine("InputInputGate distortion: {0}", distortion);

                distortion = vqInputForgetGate.BuildCodebook(vqSize);
                Logger.WriteLine("InputForgetGate distortion: {0}", distortion);

                distortion = vqInputCell.BuildCodebook(vqSize);
                Logger.WriteLine("InputCell distortion: {0}", distortion);

                distortion = vqInputOutputGate.BuildCodebook(vqSize);
                Logger.WriteLine("InputOutputGate distortion: {0}", distortion);

                fo.Write(vqSize);

                //Save InputInputGate VQ codebook into file
                for (var j = 0; j < vqSize; j++)
                {
                    fo.Write(vqInputInputGate.CodeBook[j]);
                }

                //Save InputForgetGate VQ codebook into file
                for (var j = 0; j < vqSize; j++)
                {
                    fo.Write(vqInputForgetGate.CodeBook[j]);
                }

                //Save InputCell VQ codebook into file
                for (var j = 0; j < vqSize; j++)
                {
                    fo.Write(vqInputCell.CodeBook[j]);
                }

                //Save InputOutputGate VQ codebook into file
                for (var j = 0; j < vqSize; j++)
                {
                    fo.Write(vqInputOutputGate.CodeBook[j]);
                }

                for (var i = 0; i < w; i++)
                {
                    for (var j = 0; j < h; j++)
                    {
                        fo.Write((byte)vqInputInputGate.ComputeVQ(weight[i][j].X));
                        fo.Write((byte)vqInputForgetGate.ComputeVQ(weight[i][j].Y));
                        fo.Write((byte)vqInputCell.ComputeVQ(weight[i][j].Z));
                        fo.Write((byte)vqInputOutputGate.ComputeVQ(weight[i][j].W));
                    }
                }
            }
        }

        private static LSTMGateWeight LoadLSTMGateWeights(BinaryReader br)
        {
            var w = br.ReadInt32();
            var h = br.ReadInt32();
            var vqSize = br.ReadInt32();
            LSTMGateWeight gateWeight = new LSTMGateWeight();

            Logger.WriteLine("Loading LSTM-Weight: width:{0}, height:{1}, vqSize:{2}...", w, h, vqSize);

            var m = new float[w][];
            gateWeight.weights = m;
            gateWeight.layerSize = w;
            gateWeight.denseFeatureSize = h;

            for (var i = 0; i < w; i++)
            {
                m[i] = new float[h];
                for (var j = 0; j < h; j++)
                {
                    m[i][j] = br.ReadSingle();
                }
            }

            return gateWeight;
        }

        private static Vector4[][] LoadLSTMWeights(BinaryReader br)
        {
            var w = br.ReadInt32();
            var h = br.ReadInt32();
            var vqSize = br.ReadInt32();
            var m = new Vector4[w][];

            Logger.WriteLine("Loading LSTM-Weight: width:{0}, height:{1}, vqSize:{2}...", w, h, vqSize);
            if (vqSize == 0)
            {
                for (var i = 0; i < w; i++)
                {
                    m[i] = new Vector4[h];
                    for (var j = 0; j < h; j++)
                    {
                        m[i][j].X = br.ReadSingle();
                        m[i][j].Y = br.ReadSingle();
                        m[i][j].Z = br.ReadSingle();
                        m[i][j].W = br.ReadSingle();
                    }
                }
            }
            else
            {
                var codeBookInputCell = new List<float>();
                var codeBookInputForgetGate = new List<float>();
                var codeBookInputInputGate = new List<float>();
                var codeBookInputOutputGate = new List<float>();

                for (var i = 0; i < vqSize; i++)
                {
                    codeBookInputInputGate.Add(br.ReadSingle());
                }

                for (var i = 0; i < vqSize; i++)
                {
                    codeBookInputForgetGate.Add(br.ReadSingle());
                }

                for (var i = 0; i < vqSize; i++)
                {
                    codeBookInputCell.Add(br.ReadSingle());
                }

                for (var i = 0; i < vqSize; i++)
                {
                    codeBookInputOutputGate.Add(br.ReadSingle());
                }

                for (var i = 0; i < w; i++)
                {
                    m[i] = new Vector4[h];
                    for (var j = 0; j < h; j++)
                    {
                        int vqIdx = br.ReadByte();
                        m[i][j].X = codeBookInputInputGate[vqIdx];

                        vqIdx = br.ReadByte();
                        m[i][j].Y = codeBookInputForgetGate[vqIdx];

                        vqIdx = br.ReadByte();
                        m[i][j].Z = codeBookInputCell[vqIdx];

                        vqIdx = br.ReadByte();
                        m[i][j].W = codeBookInputOutputGate[vqIdx];
                    }
                }
            }

            return m;
        }

        public void Save(BinaryWriter fo)
        {
            fo.Write(LayerSize);
            fo.Write(SparseFeatureSize);
            fo.Write(DenseFeatureSize);

            //Save hidden layer weights
            Logger.WriteLine(
                $"Saving LSTM layer, size = '{LayerSize}', sparse feature size = '{SparseFeatureSize}', dense feature size = '{DenseFeatureSize}'");

            SaveCellWeights(fo);

            if (SparseFeatureSize > 0)
            {
                //weight input->hidden
                Logger.WriteLine("Saving sparse feature weights...");
                SaveLSTMWeights(sparseFeatureWeights, fo);
            }

            if (DenseFeatureSize > 0)
            {
                //weight fea->hidden
                Logger.WriteLine("Saving dense feature weights...");

                SaveLSTMweights(wDenseInputGate, fo);
                SaveLSTMweights(wDenseCellGate, fo);
                SaveLSTMweights(wDenseForgetGate, fo);
                SaveLSTMweights(wDenseOutputGate, fo);
            }
        }

        public void Load(BinaryReader br, LayerType layerType, bool forTraining = false)
        {
            config = new LSTMLayerConfig();
            config.LayerSize = br.ReadInt32();
            config.LayerType = layerType;

            Cells = new float[LayerSize];
            Errs = new float[LayerSize];

            LSTMCells = new LSTMCell[LayerSize];
            for (var i = 0; i < LayerSize; i++)
            {
                LSTMCells[i] = new LSTMCell();
            }

            SparseFeatureSize = br.ReadInt32();
            DenseFeatureSize = br.ReadInt32();

            //Create cells of each layer
            InitializeCellWeights(br);

            //Load weight matrix between each two layer pairs
            //weight input->hidden
            if (SparseFeatureSize > 0)
            {
                Logger.WriteLine("Loading sparse feature weights...");
                sparseFeatureWeights = LoadLSTMWeights(br);
            }

            if (DenseFeatureSize > 0)
            {
                //weight fea->hidden
                Logger.WriteLine("Loading dense feature weights...");
                wDenseInputGate = LoadLSTMGateWeights(br);
                wDenseCellGate = LoadLSTMGateWeights(br);
                wDenseForgetGate = LoadLSTMGateWeights(br);
                wDenseOutputGate = LoadLSTMGateWeights(br);
            }

            if (forTraining)
            {
                InitializeInternalTrainingParameters();
            }
        }

        public void CleanForTraining()
        {
            if (SparseFeatureSize > 0)
            {
                for (var i = 0; i < LayerSize; i++)
                {
                    Array.Clear(sparseFeatureLearningRate[i], 0, SparseFeatureSize);
                    Array.Clear(sparseFeatureWeightsDelta[i], 0, sparseFeatureWeightsDelta[i].Length);
                }
            }

            if (DenseFeatureSize > 0)
            {
                wDenseCellGate.CleanStatusForTraining();
                wDenseForgetGate.CleanStatusForTraining();
                wDenseInputGate.CleanStatusForTraining();
                wDenseOutputGate.CleanStatusForTraining();
            }

            peepholeLearningRate = new Vector3[LayerSize];
            peepholeDelta = new Vector3[LayerSize];
            cellLearningRate = new Vector4[LayerSize];
            cellDelta = new Vector4[LayerSize];


            for (var i = 0; i < LayerSize; i++)
            {
                peepholeDelta[i] = Vector3.Zero;
                peepholeLearningRate[i] = Vector3.Zero;

                cellDelta[i] = Vector4.Zero;
                cellLearningRate[i] = Vector4.Zero;
            }

            vecNormalLearningRate = new Vector4(RNNHelper.LearningRate, RNNHelper.LearningRate, RNNHelper.LearningRate,
                RNNHelper.LearningRate);
            vecNormalLearningRate3 = new Vector3(RNNHelper.LearningRate, RNNHelper.LearningRate, RNNHelper.LearningRate);

            vecMaxGrad = new Vector4(RNNHelper.GradientCutoff, RNNHelper.GradientCutoff, RNNHelper.GradientCutoff,
                RNNHelper.GradientCutoff);
            vecMinGrad = new Vector4(-RNNHelper.GradientCutoff, -RNNHelper.GradientCutoff, -RNNHelper.GradientCutoff,
                -RNNHelper.GradientCutoff);

            vecMaxGrad3 = new Vector3(RNNHelper.GradientCutoff, RNNHelper.GradientCutoff, RNNHelper.GradientCutoff);
            vecMinGrad3 = new Vector3(-RNNHelper.GradientCutoff, -RNNHelper.GradientCutoff, -RNNHelper.GradientCutoff);
        }

        public void ForwardPass(SparseVector sparseFeature, float[] denseFeature)
        {
            ForwardPass(new List<SparseVector> {sparseFeature }, new List <float[]>{ denseFeature});

        }

        // forward process. output layer consists of tag value
        public void ForwardPass(List<SparseVector> sparseFeatureGroups, List<float[]> denseFeatureGroup)
        {
            //inputs(t) -> hidden(t)
            //Get sparse feature and apply it into hidden layer
            SparseFeatureGroups = sparseFeatureGroups;
            DenseFeatureGroups = denseFeatureGroup;

            for (var j = 0; j < LayerSize; j++)
            {
                var cell_j = LSTMCells[j];
                var cellWeight_j = CellWeights[j];

                //hidden(t-1) -> hidden(t)
                cell_j.previousCellState = cell_j.cellState;
                cell_j.previousCellOutput = Cells[j];

                var vecCell_j = Vector4.Zero;

                if (SparseFeatureSize > 0)
                {
                    //Apply sparse weights
                    var weights = sparseFeatureWeights[j];
                    var deri = sparseFeatureToHiddenDeri[j];

                    foreach (var sparseFeature in SparseFeatureGroups)
                    {
                        foreach (var pair in sparseFeature)
                        {
                            vecCell_j += weights[pair.Key] * pair.Value;
                            if (deri.ContainsKey(pair.Key) == false)
                            {
                                deri.Add(pair.Key, Vector3.Zero);
                            }
                        }
                    }
                }

                if (DenseFeatureSize > 0)
                {
                    //Apply dense weights
                    var k = 0;
                    float[] denseInputGateWeight_j = wDenseInputGate.weights[j];
                    float[] denseForgetGateWeight_j = wDenseForgetGate.weights[j];
                    float[] denseCellGateWeight_j = wDenseCellGate.weights[j];
                    float[] denseOutputGateWeight_j = wDenseOutputGate.weights[j];

                    foreach (var denseFeature in DenseFeatureGroups)
                    {
                        int i = 0;
                        var denseFeatureSize = denseFeature.Length;
                        var moreItems = (denseFeatureSize % Vector<float>.Count);
                        while (i < denseFeatureSize - moreItems)
                        {
                            var vX = new Vector<float>(denseInputGateWeight_j, k);
                            var vY = new Vector<float>(denseForgetGateWeight_j, k);
                            var vZ = new Vector<float>(denseCellGateWeight_j, k);
                            var vW = new Vector<float>(denseOutputGateWeight_j, k);
                            var vFeature = new Vector<float>(denseFeature, i);

                            vecCell_j.X += Vector.Dot(vX, vFeature);
                            vecCell_j.Y += Vector.Dot(vY, vFeature);
                            vecCell_j.Z += Vector.Dot(vZ, vFeature);
                            vecCell_j.W += Vector.Dot(vW, vFeature);

                            k += Vector<float>.Count;
                            i += Vector<float>.Count;
                        }

                        while (i < denseFeatureSize)
                        {
                            vecCell_j.X += denseInputGateWeight_j[k] * denseFeature[i];
                            vecCell_j.Y += denseForgetGateWeight_j[k] * denseFeature[i];
                            vecCell_j.Z += denseCellGateWeight_j[k] * denseFeature[i];
                            vecCell_j.W += denseOutputGateWeight_j[k] * denseFeature[i];
                            k++;
                            i++;
                        }
                    }
                }

                //rest the value of the net input to zero
                cell_j.netIn = vecCell_j.X;
                cell_j.netForget = vecCell_j.Y;
                //reset each netCell state to zero
                cell_j.netCellState = vecCell_j.Z;
                //reset each netOut to zero
                cell_j.netOut = vecCell_j.W;

                //include internal connection multiplied by the previous cell state
                cell_j.netIn += cell_j.previousCellState * cellWeight_j.wPeepholeIn + cell_j.previousCellOutput * cellWeight_j.wCellIn;
                //squash input
                cell_j.yIn = Sigmoid(cell_j.netIn);

                //include internal connection multiplied by the previous cell state
                cell_j.netForget += cell_j.previousCellState * cellWeight_j.wPeepholeForget +
                                    cell_j.previousCellOutput * cellWeight_j.wCellForget;
                cell_j.yForget = Sigmoid(cell_j.netForget);

                cell_j.netCellState += cell_j.previousCellOutput * cellWeight_j.wCellState;
                cell_j.yCellState = TanH(cell_j.netCellState);

                //cell state is equal to the previous cell state multipled by the forget gate and the cell inputs multiplied by the input gate
                cell_j.cellState = cell_j.yForget * cell_j.previousCellState + cell_j.yIn * cell_j.yCellState;

                ////include the internal connection multiplied by the CURRENT cell state
                cell_j.netOut += cell_j.cellState * cellWeight_j.wPeepholeOut + cell_j.previousCellOutput * cellWeight_j.wCellOut;

                //squash output gate
                cell_j.yOut = Sigmoid(cell_j.netOut);

                Cells[j] = (float)(TanH(cell_j.cellState) * cell_j.yOut);

                LSTMCells[j] = cell_j;
            }
        }


        private void UpdateOutputGateWeights(LSTMGateWeight gateWeight, int i, float err)
        {
            var j = 0;
            float[] learningrate_i = gateWeight.learningRate[i];
            float[] weights_i = gateWeight.weightsDelta[i];

            foreach (var denseFeature in DenseFeatureGroups)
            {
                int k = 0;
                var denseFeatureSize = denseFeature.Length;
                var moreItems = (denseFeatureSize % Vector<float>.Count);
                while (k < denseFeatureSize - moreItems)
                {
                    Vector<float> vecDelta = new Vector<float>(denseFeature, k);
                    vecDelta = vecDelta * err;

                    var w_i = new Vector<float>(weights_i, j);
                    w_i += vecDelta;
                    w_i.CopyTo(weights_i, j);

                    j += Vector<float>.Count;
                    k += Vector<float>.Count;
                }

                while (k < denseFeatureSize)
                {
                    float delta = denseFeature[k] * err;
                    weights_i[j] += delta;

                    j++;
                    k++;
                }
            }
        }

        public void ComputeLayerErr(ILayer prevLayer)
        {
            ComputeLayerErr(prevLayer.Errs);
        }

        public void ComputeLayerErr(List<float[]> destErrsList, bool cleanDest = true)
        {
            if (cleanDest == true)
            {
                foreach (float[] destErr in destErrsList)
                {
                    Array.Clear(destErr, 0, destErr.Length);
                }
            }


            int height = wDenseOutputGate.weights.Length;
            for (var k = 0; k < height; k++)
            {
                int i = 0;
                float[] weights = wDenseOutputGate.weights[k];
                float err = Errs[k];

                foreach (float[] destErr in destErrsList)
                {
                    int j = 0;
                    var moreItems = (destErr.Length % Vector<float>.Count);
                    while (j < destErr.Length - moreItems)
                    {
                        Vector<float> vecWeights = new Vector<float>(weights, i);
                        Vector<float> vecErrs = new Vector<float>(destErr, j);
                        vecErrs += err * vecWeights;

                        vecErrs.CopyTo(destErr, j);
                        i += Vector<float>.Count;
                        j += Vector<float>.Count;
                    }

                    while (j < destErr.Length)
                    {
                        destErr[j] += err * weights[i];
                        i++;
                        j++;
                    }
                }
            }
        }


        public void ComputeLayerErr(float[] destErr, bool cleanDest = true)
        {
            if (cleanDest == true)
            {
                Array.Clear(destErr, 0, destErr.Length);
            }
            int height = wDenseOutputGate.weights.Length;
            int weight = wDenseOutputGate.weights[0].Length;
            for (var k = 0; k < height; k++)
            {
                int i = 0;
                float[] weights = wDenseOutputGate.weights[k];
                float err = Errs[k];

                var moreItems = (weight % Vector<float>.Count);
                while (i < weight - moreItems)
                {
                    Vector<float> vecWeights = new Vector<float>(weights, i);
                    Vector<float> vecErrs = new Vector<float>(destErr, i);
                    vecErrs += err * vecWeights;

                    vecErrs.CopyTo(destErr, i);
                    i += Vector<float>.Count;
                }

                while (i < weight)
                {
                    destErr[i] += err * weights[i];
                    i++;
                }
            }
        }

        private void UpdateGateWeights(LSTMGateWeight gateWeight, int i, float featureDerivate, float c_yForget, float err)
        {
            var j = 0;
            float[] deri_i = gateWeight.deri[i];
            float[] learningrate_i = gateWeight.learningRate[i];
            float[] weights_i = gateWeight.weightsDelta[i];

            foreach (var denseFeature in DenseFeatureGroups)
            {
                int k = 0;
                var denseFeatureSize = denseFeature.Length;
                var moreItems = (denseFeatureSize % Vector<float>.Count);
                while (k < denseFeatureSize - moreItems)
                {
                    var feature = new Vector<float>(denseFeature, k);
                    var wd = feature * featureDerivate;
                    var wd_i = new Vector<float>(deri_i, j);
                    wd += wd_i * c_yForget;
                    wd.CopyTo(deri_i, j);

                    Vector<float> vecDelta = wd * err;

                    var w_i = new Vector<float>(weights_i, j);
                    w_i += vecDelta;
                    w_i.CopyTo(weights_i, j);

                    j += Vector<float>.Count;
                    k += Vector<float>.Count;
                }

                while (k < denseFeatureSize)
                {
                    var wd = denseFeature[k] * featureDerivate;
                    wd += deri_i[j] * c_yForget;
                    deri_i[j] = wd;
                    weights_i[j] += wd * err;

                    j++;
                    k++;
                }
            }
        }

        public void BackwardPass()
        {
            //put variables for derivaties in weight class and cell class

            for (var i = 0; i < LayerSize; i++)
            {
                var c = LSTMCells[i];
                var cellWeight = CellWeights[i];
                var cellWeightDeri = CellWeightsDeri[i];

                //using the error find the gradient of the output gate
                var gradientOutputGate = (float)(SigmoidDerivative(c.netOut) * TanH(c.cellState) * Errs[i]);

                //internal cell state error
                var cellStateError =
                    (float)(c.yOut * Errs[i] * TanHDerivative(c.cellState) + gradientOutputGate * cellWeight.wPeepholeOut);

                var vecErr = new Vector4(cellStateError, cellStateError, cellStateError, gradientOutputGate);

                var Sigmoid2_ci_netCellState_mul_SigmoidDerivative_ci_netIn = TanH(c.netCellState) *
                                                                              SigmoidDerivative(c.netIn);
                var ci_previousCellState_mul_SigmoidDerivative_ci_netForget = c.previousCellState *
                                                                              SigmoidDerivative(c.netForget);
                var Sigmoid2Derivative_ci_netCellState_mul_ci_yIn = TanHDerivative(c.netCellState) * c.yIn;

                var vecDerivate = new Vector3(
                    (float)Sigmoid2_ci_netCellState_mul_SigmoidDerivative_ci_netIn,
                    (float)ci_previousCellState_mul_SigmoidDerivative_ci_netForget,
                    (float)Sigmoid2Derivative_ci_netCellState_mul_ci_yIn);
                var c_yForget = (float)c.yForget;

                if (SparseFeatureSize > 0)
                {
                    //Get sparse feature and apply it into hidden layer
                    var w_i = sparseFeatureWeightsDelta[i];
                    var wd_i = sparseFeatureToHiddenDeri[i];
                    var wlr_i = sparseFeatureLearningRate[i];

                    foreach (var sparseFeature in SparseFeatureGroups)
                    {
                        foreach (var entry in sparseFeature)
                        {
                            var wd = vecDerivate * entry.Value;
                            //Adding historical information
                            wd += wd_i[entry.Key] * c_yForget;
                            wd_i[entry.Key] = wd;

                            //Computing final err delta
                            var vecDelta = new Vector4(wd, entry.Value);
                            w_i[entry.Key] += vecErr * vecDelta;
                        }
                    }
                }

                if (DenseFeatureSize > 0)
                {
                    UpdateGateWeights(wDenseInputGate, i, vecDerivate.X, c_yForget, cellStateError);
                    UpdateGateWeights(wDenseForgetGate, i, vecDerivate.Y, c_yForget, cellStateError);
                    UpdateGateWeights(wDenseCellGate, i, vecDerivate.Z, c_yForget, cellStateError);
                    UpdateOutputGateWeights(wDenseOutputGate, i, gradientOutputGate);
                }

                //Update peephols weights
                //partial derivatives for internal connections
                cellWeightDeri.dSWPeepholeIn = cellWeightDeri.dSWPeepholeIn * c.yForget +
                                  Sigmoid2_ci_netCellState_mul_SigmoidDerivative_ci_netIn * c.previousCellState;

                //partial derivatives for internal connections, initially zero as dS is zero and previous cell state is zero
                cellWeightDeri.dSWPeepholeForget = cellWeightDeri.dSWPeepholeForget * c.yForget +
                                      ci_previousCellState_mul_SigmoidDerivative_ci_netForget * c.previousCellState;

                //update internal weights
                var vecCellDelta = new Vector3((float)cellWeightDeri.dSWPeepholeIn, (float)cellWeightDeri.dSWPeepholeForget, (float)c.cellState);
                var vecErr3 = new Vector3(cellStateError, cellStateError, gradientOutputGate);
                peepholeDelta[i] += vecErr3 * vecCellDelta;

                //Update cells weights
                //partial derivatives for internal connections
                cellWeightDeri.dSWCellIn = cellWeightDeri.dSWCellIn * c.yForget +
                              Sigmoid2_ci_netCellState_mul_SigmoidDerivative_ci_netIn * c.previousCellOutput;

                //partial derivatives for internal connections, initially zero as dS is zero and previous cell state is zero
                cellWeightDeri.dSWCellForget = cellWeightDeri.dSWCellForget * c.yForget +
                                  ci_previousCellState_mul_SigmoidDerivative_ci_netForget * c.previousCellOutput;

                cellWeightDeri.dSWCellState = cellWeightDeri.dSWCellState * c.yForget +
                                 Sigmoid2Derivative_ci_netCellState_mul_ci_yIn * c.previousCellOutput;

                var vecCellDelta4 = new Vector4((float)cellWeightDeri.dSWCellIn, (float)cellWeightDeri.dSWCellForget, (float)cellWeightDeri.dSWCellState, (float)c.previousCellOutput);
                cellDelta[i] += vecErr * vecCellDelta4;

                LSTMCells[i] = c;
            }
        }
        public void UpdateWeights()
        {
            wDenseInputGate.UpdateWeights();
            wDenseForgetGate.UpdateWeights();
            wDenseCellGate.UpdateWeights();
            wDenseOutputGate.UpdateWeights();

            for (var i = 0; i < LayerSize; i++)
            {
                LSTMCellWeight cellWeights_i = CellWeights[i];

                //Normalize cell peephole weights delta
                var vecPeepholeDelta = peepholeDelta[i];
                peepholeDelta[i] = Vector3.Zero;

                vecPeepholeDelta = vecPeepholeDelta / RNNHelper.MiniBatchSize;
                vecPeepholeDelta = Vector3.Clamp(vecPeepholeDelta, vecMinGrad3, vecMaxGrad3);

                //Normalize cell weights delta
                var vecCellDelta = cellDelta[i];
                cellDelta[i] = Vector4.Zero;

                vecCellDelta = vecCellDelta / RNNHelper.MiniBatchSize;
                vecCellDelta = Vector4.Clamp(vecCellDelta, vecMinGrad, vecMaxGrad);

                //Computing actual learning rate
                var vecPeepholeLearningRate = ComputeLearningRate(vecPeepholeDelta, ref peepholeLearningRate[i]);
                vecPeepholeDelta = vecPeepholeLearningRate * vecPeepholeDelta;

                var vecCellLearningRate = ComputeLearningRate(vecCellDelta, ref cellLearningRate[i]);
                vecCellDelta = vecCellLearningRate * vecCellDelta;

                cellWeights_i.wPeepholeIn += vecPeepholeDelta.X;
                cellWeights_i.wPeepholeForget += vecPeepholeDelta.Y;
                cellWeights_i.wPeepholeOut += vecPeepholeDelta.Z;
                cellWeights_i.wCellIn += vecCellDelta.X;
                cellWeights_i.wCellForget += vecCellDelta.Y;
                cellWeights_i.wCellState += vecCellDelta.Z;
                cellWeights_i.wCellOut += vecCellDelta.W;

                //Update weights for sparse features
                if (SparseFeatureSize > 0)
                {
                    var wlr_i = sparseFeatureLearningRate[i];
                    var sparseFeatureWeightsDelta_i = sparseFeatureWeightsDelta[i];
                    var sparseFeatureWeights_i = sparseFeatureWeights[i];
                    for (var j = 0; j < SparseFeatureSize; j++)
                    {
                        if (sparseFeatureWeightsDelta_i[j] != Vector4.Zero)
                        {
                            Vector4 vecDelta = sparseFeatureWeightsDelta_i[j];
                            sparseFeatureWeightsDelta_i[j] = Vector4.Zero;

                            vecDelta = vecDelta / RNNHelper.MiniBatchSize;

                            vecDelta = Vector4.Clamp(vecDelta, vecMinGrad, vecMaxGrad);

                            var vecLearningRate = ComputeLearningRate(vecDelta, ref wlr_i[j]);
                            sparseFeatureWeights_i[j] += vecDelta * vecLearningRate;
                        }
                    }
                }
            }
        }

        public void Reset()
        {
            for (var i = 0; i < LayerSize; i++)
            {
                Cells[i] = 0;
                InitializeLSTMCell(LSTMCells[i], CellWeights[i], CellWeightsDeri[i]);

                if (SparseFeatureSize > 0)
                {
                    sparseFeatureToHiddenDeri[i].Clear();
                }
            }
        }

        protected RunningMode runningMode;
        public void SetRunningMode(RunningMode mode)
        {
            runningMode = mode;
        }

        private void InitializeLSTMCell(LSTMCell c, LSTMCellWeight cw, LSTMCellWeightDeri deri)
        {
            c.cellState = 0;

            //partial derivatives
            deri.dSWPeepholeIn = 0;
            deri.dSWPeepholeForget = 0;

            deri.dSWCellIn = 0;
            deri.dSWCellForget = 0;
            deri.dSWCellState = 0;
        }

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        private Vector3 ComputeLearningRate(Vector3 vecDelta, ref Vector3 vecWeightLearningRate)
        {
            if (RNNHelper.IsConstAlpha)
            {
                return vecNormalLearningRate3;
            }

            vecWeightLearningRate += vecDelta * vecDelta;
            return vecNormalLearningRate3 / (Vector3.SquareRoot(vecWeightLearningRate) + Vector3.One);
        }

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        private Vector4 ComputeLearningRate(Vector4 vecDelta, ref Vector4 vecWeightLearningRate)
        {
            if (RNNHelper.IsConstAlpha)
            {
                return vecNormalLearningRate;
            }

            vecWeightLearningRate += vecDelta * vecDelta;
            return vecNormalLearningRate / (Vector4.SquareRoot(vecWeightLearningRate) + Vector4.One);
        }
    }

    public class LSTMCellWeightDeri
    {
        public double dSWCellForget;
        public double dSWCellIn;
        public double dSWCellState;
        public double dSWPeepholeForget;
        public double dSWPeepholeIn;
    }


    public class LSTMCellWeight
    {
        //Weights for each gate
        public double wCellForget;
        public double wCellIn;
        public double wCellOut;
        public double wCellState;
        public double wPeepholeForget;
        public double wPeepholeIn;
        public double wPeepholeOut;
    }

    public class LSTMCell
    {
        //The following fields are only for forward
        public double previousCellState;
        public double previousCellOutput;
        public double cellState;

        public double netCellState;
        public double netForget;
        public double netIn;
        public double netOut;

        public double yCellState;
        public double yForget;
        public double yIn;
        public double yOut;

        public LSTMCell()
        {

        }

        public LSTMCell(LSTMCell cell)
        {
            Set(cell);
        }

        public void Set(LSTMCell cell)
        {
            previousCellState = cell.previousCellState;
            previousCellOutput = cell.previousCellOutput;
            cellState = cell.cellState;
            netCellState = cell.netCellState;
            netForget = cell.netForget;
            netIn = cell.netIn;
            netOut = cell.netOut;
            yCellState = cell.yCellState;
            yForget = cell.yForget;
            yIn = cell.yIn;
            yOut = cell.yOut;
        }
    }
}

<|start_filename|>RNNSharp/Networks/RNN.cs<|end_filename|>
﻿using AdvUtils;
using RNNSharp.Layers;
using System;
using System.Collections.Generic;
using System.IO;
using System.Numerics;
using System.Threading.Tasks;

namespace RNNSharp.Networks
{
    public abstract class RNN<T> where T : ISequence
    {
        public virtual bool IsCRFTraining { get; set; }
        public virtual bool bVQ { get; set; }

        protected Matrix<float> CRFSeqOutput;
        protected Matrix<float> CRFWeights { get; set; }
        protected IOutputLayer OutputLayer { get; set; }

        private Matrix<float> crfWeightsDelta;

        private double[][] alphaSet;
        private double[][] betaSet;

        public int MaxSeqLength = 1024;

        public abstract void CreateNetwork(List<LayerConfig> hiddenLayersConfig, LayerConfig outputLayerConfig, DataSet<T> TrainingSet, Config featurizer);

        public abstract int[] ProcessSequenceCRF(Sequence pSequence, RunningMode runningMode);

        public abstract int[] ProcessSequence(ISequence sequence, RunningMode runningMode, bool outputRawScore, out Matrix<float> m);

        public abstract int[] ProcessSequence(ISentence sentence, Config featurizer, RunningMode runningMode, bool outputRawScore, out Matrix<float> m);

        public abstract void CleanStatusForTraining();

        public abstract void SaveModel(string filename);

        public abstract void LoadModel(string filename, bool bTrain = false);

        public abstract float[][] ComputeTopHiddenLayerOutput(Sequence pSequence);

        public abstract int GetTopHiddenLayerSize();

        public abstract RNN<T> Clone();


        public virtual void UpdateWeights()
        {

        }

        public static RNN<T> CreateRNN(NETWORKTYPE networkType)
        {
            RNN<T> rnn = null;
            switch (networkType)
            {
                case NETWORKTYPE.Forward:
                    rnn = new ForwardRNN<T>();
                    break;
                case NETWORKTYPE.ForwardSeq2Seq:
                    rnn = new ForwardRNNSeq2Seq<T>();
                    break;
                case NETWORKTYPE.BiDirectional:
                    rnn = new BiRNN<T>();
                    break;
                case NETWORKTYPE.BiDirectionalAverage:
                    rnn = new BiRNNAvg<T>();
                    break;

            }
            return rnn;
        }

        protected IOutputLayer CreateOutputLayer(LayerConfig outputLayerConfig, int sparseFeatureSize, int denseFeatureSize)
        {
            IOutputLayer outputLayer = null;
            switch (outputLayerConfig.LayerType)
            {
                case LayerType.SampledSoftmax:
                    Logger.WriteLine("Create sampled softmax layer as output layer");
                    outputLayer = new SampledSoftmaxLayer(outputLayerConfig as SampledSoftmaxLayerConfig);
                    outputLayer.InitializeWeights(0, denseFeatureSize);
                    break;

                case LayerType.Softmax:
                    Logger.WriteLine("Create softmax layer as output layer.");
                    outputLayer = new SoftmaxLayer(outputLayerConfig as SoftmaxLayerConfig);
                    outputLayer.InitializeWeights(sparseFeatureSize, denseFeatureSize);
                    break;

                case LayerType.Simple:
                    Logger.WriteLine("Create simple layer as output layer.");
                    outputLayer = new SimpleLayer(outputLayerConfig as SimpleLayerConfig);
                    outputLayer.InitializeWeights(sparseFeatureSize, denseFeatureSize);
                    break;
            }
            outputLayer.LabelShortList = new List<int>();

            return outputLayer;
        }

        protected virtual List<ILayer> CreateLayers(List<LayerConfig> hiddenLayersConfig)
        {
            var hiddenLayers = new List<ILayer>();
            for (var i = 0; i < hiddenLayersConfig.Count; i++)
            {
                ILayer layer = null;
                switch (hiddenLayersConfig[i].LayerType)
                {
                    case LayerType.LSTM:
                        layer = new LSTMLayer(hiddenLayersConfig[i] as LSTMLayerConfig);
                        Logger.WriteLine("Create LSTM layer.");
                        break;

                    case LayerType.DropOut:
                        layer = new DropoutLayer(hiddenLayersConfig[i] as DropoutLayerConfig);
                        Logger.WriteLine("Create Dropout layer.");
                        break;
                }

                hiddenLayers.Add(layer);
            }

            return hiddenLayers;
        }

        public void SetRuntimeFeatures(State state, int curState, int numStates, int[] predicted, bool forward = true)
        {
            if (predicted != null && state.RuntimeFeatures != null)
            {
                // set runtime feature
                for (var i = 0; i < state.RuntimeFeatures.Length; i++)
                {
                    for (var j = 0; j < OutputLayer.LayerSize; j++)
                    {
                        //Clean up run time feature value and then set a new one
                        state.SetRuntimeFeature(i, j, 0);
                    }

                    var pos = curState + (forward ? 1 : -1) * state.RuntimeFeatures[i].OffsetToCurrentState;
                    if (pos >= 0 && pos < numStates)
                    {
                        state.SetRuntimeFeature(i, predicted[pos], 1);
                    }
                }
            }
        }

        public void ForwardBackward(int numStates, Matrix<float> m_RawOutput)
        {
            //forward
            var OutputLayerSize = OutputLayer.LayerSize;

            for (var i = 0; i < numStates; i++)
            {
                for (var j = 0; j < OutputLayerSize; j++)
                {
                    double dscore0 = 0;
                    if (i > 0)
                    {
                        for (var k = 0; k < OutputLayerSize; k++)
                        {
                            var fbgm = CRFWeights[j][k];
                            var finit = alphaSet[i - 1][k];
                            var ftmp = fbgm + finit;

                            dscore0 = MathUtil.logsumexp(dscore0, ftmp, k == 0);
                        }
                    }
                    alphaSet[i][j] = dscore0 + m_RawOutput[i][j];
                }
            }

            //backward
            for (var i = numStates - 1; i >= 0; i--)
            {
                for (var j = 0; j < OutputLayerSize; j++)
                {
                    double dscore0 = 0;
                    if (i < numStates - 1)
                    {
                        for (var k = 0; k < OutputLayerSize; k++)
                        {
                            var fbgm = CRFWeights[k][j];
                            var finit = betaSet[i + 1][k];
                            var ftmp = fbgm + finit;

                            dscore0 = MathUtil.logsumexp(dscore0, ftmp, k == 0);
                        }
                    }
                    betaSet[i][j] = dscore0 + m_RawOutput[i][j];
                }
            }


            //Z_
            double Z_ = 0.0f;
            var betaSet_0 = betaSet[0];
            for (var i = 0; i < OutputLayerSize; i++)
            {
                Z_ = MathUtil.logsumexp(Z_, betaSet_0[i], i == 0);
            }

            //Calculate the output probability of each node
            for (var i = 0; i < numStates; i++)
            {
                var CRFSeqOutput_i = CRFSeqOutput[i];
                var alphaSet_i = alphaSet[i];
                var betaSet_i = betaSet[i];
                var m_RawOutput_i = m_RawOutput[i];
                for (var j = 0; j < OutputLayerSize; j++)
                {
                    CRFSeqOutput_i[j] = (float)Math.Exp(alphaSet_i[j] + betaSet_i[j] - m_RawOutput_i[j] - Z_);
                }
            }
        }

        public int[] Viterbi(Matrix<float> ys, int seqLen)
        {
            var OutputLayerSize = OutputLayer.LayerSize;

            var vPath = new int[seqLen, OutputLayerSize];

            var vPreAlpha = new float[OutputLayerSize];
            var vAlpha = new float[OutputLayerSize];

            var nStartTagIndex = 0;
            //viterbi algorithm
            for (var i = 0; i < OutputLayerSize; i++)
            {
                vPreAlpha[i] = ys[0][i];
                if (i != nStartTagIndex)
                    vPreAlpha[i] += float.MinValue;
                vPath[0, i] = nStartTagIndex;
            }
            for (var t = 0; t < seqLen; t++)
            {
                for (var j = 0; j < OutputLayerSize; j++)
                {
                    vPath[t, j] = 0;
                    var CRFTagTransWeights_j = CRFWeights[j];
                    var ys_t = ys[t];
                    var maxScore = float.MinValue;
                    for (var i = 0; i < OutputLayerSize; i++)
                    {
                        var score = vPreAlpha[i] + CRFTagTransWeights_j[i] + ys_t[j];
                        if (score > maxScore)
                        {
                            maxScore = score;
                            vPath[t, j] = i;
                        }
                    }

                    vAlpha[j] = maxScore;
                }
                vPreAlpha = vAlpha;
                vAlpha = new float[OutputLayerSize];
            }

            //backtrace to get the best result path
            var tagOutputs = new int[seqLen];
            tagOutputs[seqLen - 1] = nStartTagIndex;
            var nNextTag = tagOutputs[seqLen - 1];
            for (var t = seqLen - 2; t >= 0; t--)
            {
                tagOutputs[t] = vPath[t + 1, nNextTag];
                nNextTag = tagOutputs[t];
            }

            return tagOutputs;
        }

        public void InitializeCRFWeights(List<List<float>> m)
        {
            var OutputLayerSize = OutputLayer.LayerSize;
            CRFWeights = new Matrix<float>(OutputLayerSize, OutputLayerSize);
            for (var i = 0; i < OutputLayerSize; i++)
            {
                for (var j = 0; j < OutputLayerSize; j++)
                {
                    CRFWeights[i][j] = m[i][j];
                }
            }
        }

        public void InitializeCRFVariablesForTraining()
        {
            var OutputLayerSize = OutputLayer.LayerSize;
            CRFSeqOutput = new Matrix<float>(MaxSeqLength, OutputLayerSize);

            alphaSet = new double[MaxSeqLength][];
            betaSet = new double[MaxSeqLength][];
            for (var i = 0; i < MaxSeqLength; i++)
            {
                alphaSet[i] = new double[OutputLayerSize];
                betaSet[i] = new double[OutputLayerSize];
            }

            crfWeightsDelta = new Matrix<float>(OutputLayerSize, OutputLayerSize);
        }

        //  private static object lockerCRF = new object();
        public void UpdateBigramTransition(Sequence seq)
        {
            var OutputLayerSize = OutputLayer.LayerSize;
            var numStates = seq.States.Length;

            crfWeightsDelta.Clean();

            for (var timeat = 1; timeat < numStates; timeat++)
            {
                var CRFSeqOutput_timeat = CRFSeqOutput[timeat];
                var CRFSeqOutput_pre_timeat = CRFSeqOutput[timeat - 1];
                for (var i = 0; i < OutputLayerSize; i++)
                {
                    var CRFSeqOutput_timeat_i = CRFSeqOutput_timeat[i];
                    var CRFTagTransWeights_i = CRFWeights[i];
                    var crfWeightsDelta_i = crfWeightsDelta[i];
                    var j = 0;

                    var vecCRFSeqOutput_timeat_i = new Vector<float>(CRFSeqOutput_timeat_i);

                    var moreItems = (OutputLayerSize % Vector<float>.Count);
                    while (j < OutputLayerSize - moreItems)
                    {
                        var v1 = new Vector<float>(CRFTagTransWeights_i, j);
                        var v2 = new Vector<float>(CRFSeqOutput_pre_timeat, j);
                        var v = new Vector<float>(crfWeightsDelta_i, j);

                        v -= v1 * vecCRFSeqOutput_timeat_i * v2;
                        v.CopyTo(crfWeightsDelta_i, j);

                        j += Vector<float>.Count;
                    }

                    while (j < OutputLayerSize)
                    {
                        crfWeightsDelta_i[j] -= CRFTagTransWeights_i[j] * CRFSeqOutput_timeat_i * CRFSeqOutput_pre_timeat[j];
                        j++;
                    }
                }

                var iTagId = seq.States[timeat].Label;
                var iLastTagId = seq.States[timeat - 1].Label;
                crfWeightsDelta[iTagId][iLastTagId] += 1;
            }

            //Update tag Bigram LM

            for (var i = 0; i < OutputLayerSize; i++)
            {
                var CRFWeights_i = CRFWeights[i];
                var CRFWeightsDelta_i = crfWeightsDelta[i];
                var j = 0;

                var moreItems = (OutputLayerSize % Vector<float>.Count);
                while (j < OutputLayerSize - moreItems)
                {
                    var vecDelta = new Vector<float>(CRFWeightsDelta_i, j);
                    //Normalize delta
                    vecDelta = RNNHelper.NormalizeGradient(vecDelta);

                    //Update weights
                    var vecWeights = new Vector<float>(CRFWeights_i, j);
                    vecWeights += RNNHelper.vecNormalLearningRate * vecDelta;
                    vecWeights.CopyTo(CRFWeights_i, j);

                    j += Vector<float>.Count;
                }

                while (j < OutputLayerSize)
                {
                    var delta = CRFWeightsDelta_i[j];
                    delta = RNNHelper.NormalizeGradient(delta);
                    CRFWeights_i[j] += RNNHelper.LearningRate * delta;
                    j++;
                }
            }
        }

        public int[][] DecodeNBestCRF(Sentence sent, Config config, int N)
        {
            //ys contains the output of RNN for each word
            Matrix<float> ys;
            ProcessSequence(sent, config, RunningMode.Test, true, out ys);

            var n = sent.TokensList.Count;
            var K = OutputLayer.LayerSize;
            var STP = CRFWeights;
            var vPath = new PAIR<int, int>[n, K, N];
            var DUMP_LABEL = -1;
            var vPreAlpha = new float[K, N];
            var vAlpha = new float[K, N];

            var nStartTagIndex = 0;
            var nEndTagIndex = 0;
            var MIN_VALUE = float.MinValue;

            //viterbi algorithm
            for (var i = 0; i < K; i++)
            {
                for (var j = 0; j < N; j++)
                {
                    vPreAlpha[i, j] = MIN_VALUE;
                    vPath[0, i, j] = new PAIR<int, int>(DUMP_LABEL, 0);
                }
            }
            vPreAlpha[nStartTagIndex, 0] = ys[0][nStartTagIndex];
            vPath[0, nStartTagIndex, 0].first = nStartTagIndex;

            var q = new PriorityQueue<float, PAIR<int, int>>();

            for (var t = 1; t < n; t++)
            {
                for (var j = 0; j < K; j++)
                {
                    while (q.Count() > 0)
                        q.Dequeue();
                    var _stp = STP[j][0];
                    var _y = ys[t][j];
                    for (var k = 0; k < N; k++)
                    {
                        var score = vPreAlpha[0, k] + _stp + _y;
                        q.Enqueue(score, new PAIR<int, int>(0, k));
                    }
                    for (var i = 1; i < K; i++)
                    {
                        _stp = STP[j][i];
                        for (var k = 0; k < N; k++)
                        {
                            var score = vPreAlpha[i, k] + _stp + _y;
                            if (score <= q.Peek().Key)
                                break;
                            q.Dequeue();
                            q.Enqueue(score, new PAIR<int, int>(i, k));
                        }
                    }
                    var idx = N - 1;
                    while (q.Count() > 0)
                    {
                        vAlpha[j, idx] = q.Peek().Key;
                        vPath[t, j, idx] = q.Peek().Value;
                        idx--;
                        q.Dequeue();
                    }
                }
                vPreAlpha = vAlpha;
                vAlpha = new float[K, N];
            }

            //backtrace to get the n-best result path
            var vTagOutput = new int[N][];
            for (var i = 0; i < N; i++)
            {
                vTagOutput[i] = new int[n];
            }

            for (var k = 0; k < N; k++)
            {
                vTagOutput[k][n - 1] = nEndTagIndex;
                var decision = new PAIR<int, int>(nEndTagIndex, k);
                for (var t = n - 2; t >= 0; t--)
                {
                    vTagOutput[k][t] = vPath[t + 1, decision.first, decision.second].first;
                    decision = vPath[t + 1, decision.first, decision.second];
                }
            }

            return vTagOutput;
        }

        public int[] DecodeNN(Sentence sent, Config config)
        {
            Matrix<float> ys;
            return ProcessSequence(sent, config, RunningMode.Test, false, out ys);
        }

        public int[] DecodeCRF(Sentence sent, Config config)
        {
            //ys contains the output of RNN for each word
            Matrix<float> ys;
            ProcessSequence(sent, config, RunningMode.Test, true, out ys);
            return Viterbi(ys, sent.TokensList.Count);
        }

        public static ILayer Load(LayerType layerType, BinaryReader br, bool forTraining = false)
        {
            ILayer layer = null;
            switch (layerType)
            {
                case LayerType.LSTM:
                    layer = new LSTMLayer();
                    break;
                case LayerType.DropOut:
                    layer = new DropoutLayer();
                    break;
                case LayerType.Softmax:
                    layer = new SoftmaxLayer();
                    break;
                case LayerType.SampledSoftmax:
                    layer = new SampledSoftmaxLayer();
                    break;
                case LayerType.Simple:
                    layer = new SimpleLayer();
                    break;
            }

            layer.Load(br, layerType, forTraining);

            return layer;
        }
    }
}

<|start_filename|>RNNSharp/ModelSetting.cs<|end_filename|>
﻿using AdvUtils;
using System.Numerics;

/// <summary>
/// RNNSharp written by <NAME> (<EMAIL>)
/// </summary>

namespace RNNSharp
{
    public class ModelSetting
    {
        public ModelSetting()
        {
            MaxIteration = 20;
            LearningRate = 0.1f;
            GradientCutoff = 15.0f;
            IncrementalTrain = false;
        }

        public int MiniBatchSize { get; set; }
        public bool IncrementalTrain { get; set; }
        public float LearningRate { get; set; }
        public int MaxIteration { get; set; }
        public bool IsConstAlpha { get; set; }
        public long SaveStep { get; set; }
        public int VQ { get; set; }
        public float GradientCutoff { get; set; }

        public void DumpSetting()
        {
            Logger.WriteLine($"Use const learning rate: {IsConstAlpha}");
            Logger.WriteLine($"Starting learning rate: {LearningRate}");
            Logger.WriteLine($"Max Iteration: {MaxIteration}");
            Logger.WriteLine($"SIMD: {Vector.IsHardwareAccelerated}, Size: {Vector<float>.Count * sizeof(float) * 8}bits");
            Logger.WriteLine($"Gradient cut-off: {GradientCutoff}");

            if (SaveStep > 0)
            {
                Logger.WriteLine($"Save temporary model after every {SaveStep} sentences");
            }

            Logger.WriteLine($"Incremental training: {IncrementalTrain}");
            Logger.WriteLine($"Mini batch size: {MiniBatchSize}");
        }
    }
}

<|start_filename|>RNNSharp/Networks/ForwardRNNSeq2SeqLabeling.cs<|end_filename|>
﻿using AdvUtils;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Text;
using System.Threading.Tasks;

namespace RNNSharp.Networks
{
    class ForwardRNNSeq2SeqLabeling<T> : ForwardRNNSeq2Seq<T> where T : ISequence
    {
        public ForwardRNNSeq2SeqLabeling()
            :base()
        {

        }

        public override RNN<T> Clone()
        {
            List<SimpleLayer> hiddenLayers = new List<SimpleLayer>();

            foreach (SimpleLayer layer in HiddenLayerList)
            {
                hiddenLayers.Add(layer.CreateLayerSharedWegiths());
            }

            ForwardRNNSeq2SeqLabeling<T> rnn = new ForwardRNNSeq2SeqLabeling<T>();
            rnn.HiddenLayerList = hiddenLayers;
            rnn.OutputLayer = OutputLayer.CreateLayerSharedWegiths();
            rnn.CRFTagTransWeights = CRFTagTransWeights;
            rnn.MaxSeqLength = MaxSeqLength;
            rnn.crfLocker = crfLocker;

            return rnn;
        }

        public override int[] ProcessSequence(ISentence sentence, Config featurizer, RunningMode runningMode, bool outputRawScore, out Matrix<float> m)
        {
            var sequencePair = featurizer.BuildSequence(sentence as SentencePair);
            return TrainSequencePair(sequencePair, runningMode, outputRawScore, out m);
        }
    }
}


<|start_filename|>RNNSharp/Networks/FowardRNN.cs<|end_filename|>
﻿using AdvUtils;
using RNNSharp.Layers;
using System;
using System.Collections.Generic;
using System.IO;
using System.Numerics;
using System.Threading.Tasks;

/// <summary>
/// RNNSharp written by <NAME> (<EMAIL>)
/// </summary>

namespace RNNSharp.Networks
{
    public class PAIR<T, K>
    {
        public T first;
        public K second;

        public PAIR(T f, K s)
        {
            first = f;
            second = s;
        }
    }

    public class ForwardRNN<T> : RNN<T> where T : ISequence
    {
        public ForwardRNN()
        {
            
        }

        public override void CreateNetwork(List<LayerConfig> hiddenLayersConfig, LayerConfig outputLayerConfig, DataSet<T> TrainingSet, Config featurizer)
        {
            HiddenLayerList = CreateLayers(hiddenLayersConfig);
            for (var i = 0; i < HiddenLayerList.Count; i++)
            {
                ILayer layer = HiddenLayerList[i];
                layer.InitializeWeights(TrainingSet.SparseFeatureSize, i == 0 ? TrainingSet.DenseFeatureSize : HiddenLayerList[i - 1].LayerSize);
                layer.SetRunningMode(RunningMode.Training);

                Logger.WriteLine($"Create hidden layer {i}: size = {layer.LayerSize}, sparse feature size = {layer.SparseFeatureSize}, dense feature size = {layer.DenseFeatureSize}");
            }

            outputLayerConfig.LayerSize = TrainingSet.TagSize;
            OutputLayer = CreateOutputLayer(outputLayerConfig, TrainingSet.SparseFeatureSize, HiddenLayerList[HiddenLayerList.Count - 1].LayerSize);
            OutputLayer.SetRunningMode(RunningMode.Training);

            Logger.WriteLine($"Create a Forward recurrent neural network with {HiddenLayerList.Count} hidden layers");
        }

        public List<ILayer> HiddenLayerList { get; set; }

        public override void UpdateWeights()
        {
            foreach (var layer in HiddenLayerList)
            {
                layer.UpdateWeights();
            }

            OutputLayer.UpdateWeights();
        }

        public override RNN<T> Clone()
        {
            List<ILayer> hiddenLayers = new List<ILayer>();

            foreach (ILayer layer in HiddenLayerList)
            {
                hiddenLayers.Add(layer.CreateLayerSharedWegiths());
            }

            ForwardRNN<T> rnn = new ForwardRNN<T>();
            rnn.HiddenLayerList = hiddenLayers;
            rnn.OutputLayer = (IOutputLayer)OutputLayer.CreateLayerSharedWegiths();
            rnn.CRFWeights = CRFWeights;
            rnn.MaxSeqLength = MaxSeqLength;
            rnn.bVQ = bVQ;
            rnn.IsCRFTraining = IsCRFTraining;
            if (rnn.IsCRFTraining)
            {
                rnn.InitializeCRFVariablesForTraining();
            }

            return rnn;
        }

        public override int GetTopHiddenLayerSize()
        {
            return HiddenLayerList[HiddenLayerList.Count - 1].LayerSize;
        }

        public override float[][] ComputeTopHiddenLayerOutput(Sequence pSequence)
        {
            var numStates = pSequence.States.Length;
            var numLayers = HiddenLayerList.Count;

            //reset all layers
            foreach (var layer in HiddenLayerList)
            {
                layer.Reset();
            }

            var outputs = new float[numStates][];
            for (var curState = 0; curState < numStates; curState++)
            {
                //Compute first layer
                var state = pSequence.States[curState];
                SetRuntimeFeatures(state, curState, numStates, null);
                HiddenLayerList[0].ForwardPass(state.SparseFeature, state.DenseFeature.CopyTo());

                //Compute each layer
                for (var i = 1; i < numLayers; i++)
                {
                    //We use previous layer's output as dense feature for current layer
                    HiddenLayerList[i].ForwardPass(state.SparseFeature, HiddenLayerList[i - 1].Cells);
                }

                var tmpOutput = new float[HiddenLayerList[numLayers - 1].Cells.Length];
                for (var i = 0; i < HiddenLayerList[numLayers - 1].Cells.Length; i++)
                {
                    tmpOutput[i] = HiddenLayerList[numLayers - 1].Cells[i];
                }
                outputs[curState] = tmpOutput;
            }

            return outputs;
        }

        public override int[] ProcessSequence(ISentence sentence, Config featurizer, RunningMode runningMode, bool outputRawScore, out Matrix<float> m)
        {
            var seq = featurizer.BuildSequence(sentence as Sentence);

            return ProcessSequence(seq, runningMode, outputRawScore, out m);
        }


        public override int[] ProcessSequence(ISequence sequence, RunningMode runningMode, bool outputRawScore, out Matrix<float> m)
        {
            Sequence pSequence = sequence as Sequence;

            var numStates = pSequence.States.Length;
            var numLayers = HiddenLayerList.Count;

            m = outputRawScore ? new Matrix<float>(numStates, OutputLayer.LayerSize) : null;

            var predicted = new int[numStates];
            var isTraining = runningMode == RunningMode.Training;

            //reset all layers
            foreach (var layer in HiddenLayerList)
            {
                layer.Reset();
            }

            //Set current sentence labels into short list in output layer
            OutputLayer.LabelShortList.Clear();
            foreach (var state in pSequence.States)
            {
                OutputLayer.LabelShortList.Add(state.Label);
            }

            for (var curState = 0; curState < numStates; curState++)
            {
                //Compute first layer
                var state = pSequence.States[curState];
                SetRuntimeFeatures(state, curState, numStates, predicted);
                HiddenLayerList[0].ForwardPass(state.SparseFeature, state.DenseFeature.CopyTo());

                //Compute each layer
                for (var i = 1; i < numLayers; i++)
                {
                    //We use previous layer's output as dense feature for current layer
                    HiddenLayerList[i].ForwardPass(state.SparseFeature, HiddenLayerList[i - 1].Cells);
                }

                //Compute output layer
                OutputLayer.ForwardPass(state.SparseFeature, HiddenLayerList[numLayers - 1].Cells);

                if (m != null)
                {
                    OutputLayer.Cells.CopyTo(m[curState], 0);
                }

                predicted[curState] = OutputLayer.GetBestOutputIndex();

                if (runningMode == RunningMode.Training)
                {
                    // error propogation
                    OutputLayer.ComputeOutputLoss(CRFSeqOutput, state, curState);

                    //propogate errors to each layer from output layer to input layer
                    OutputLayer.ComputeLayerErr(HiddenLayerList[numLayers - 1]);

                    for (var i = numLayers - 1; i > 0; i--)
                    {
                        HiddenLayerList[i].ComputeLayerErr(HiddenLayerList[i - 1]);
                    }

                    //Update net weights
                    OutputLayer.BackwardPass();

                    for (var i = 0; i < numLayers; i++)
                    { HiddenLayerList[i].BackwardPass(); }
                }
            }

            return predicted;
        }

        public override int[] ProcessSequenceCRF(Sequence pSequence, RunningMode runningMode)
        {
            var numStates = pSequence.States.Length;
            var numLayers = HiddenLayerList.Count;

            //Get network output without CRF
            Matrix<float> nnOutput;
            ProcessSequence(pSequence, RunningMode.Test, true, out nnOutput);

            //Compute CRF result
            ForwardBackward(numStates, nnOutput);

            //Compute best path in CRF result
            var predicted = Viterbi(nnOutput, numStates);

            if (runningMode == RunningMode.Training)
            {
                //Update tag bigram transition for CRF model
                UpdateBigramTransition(pSequence);

                //Reset all layer states
                foreach (var layer in HiddenLayerList)
                {
                    layer.Reset();
                }

                for (var curState = 0; curState < numStates; curState++)
                {
                    // error propogation
                    var state = pSequence.States[curState];
                    SetRuntimeFeatures(state, curState, numStates, null);
                    HiddenLayerList[0].SetRunningMode(runningMode);
                    HiddenLayerList[0].ForwardPass(state.SparseFeature, state.DenseFeature.CopyTo());

                    for (var i = 1; i < numLayers; i++)
                    {
                        HiddenLayerList[i].SetRunningMode(runningMode);
                        HiddenLayerList[i].ForwardPass(state.SparseFeature, HiddenLayerList[i - 1].Cells);
                    }

                    OutputLayer.ComputeOutputLoss(CRFSeqOutput, state, curState);

                    //propogate errors to each layer from output layer to input layer
                    OutputLayer.ComputeLayerErr(HiddenLayerList[numLayers - 1]);

                    for (var i = numLayers - 1; i > 0; i--)
                    {
                        HiddenLayerList[i].ComputeLayerErr(HiddenLayerList[i - 1]);
                    }

                    //Update net weights
                    OutputLayer.BackwardPass();

                    for (var i = 0; i < numLayers; i++)
                    { HiddenLayerList[i].BackwardPass(); }
 
                }
            }

            return predicted;
        }

        // save model as binary format
        public override void SaveModel(string filename)
        {
            var sw = new StreamWriter(filename);
            var fo = new BinaryWriter(sw.BaseStream);

            fo.Write(IsCRFTraining);
            fo.Write(HiddenLayerList.Count);
            foreach (var layer in HiddenLayerList)
            {
                fo.Write((int)layer.LayerType);
                layer.Save(fo);
            }

            fo.Write((int)OutputLayer.LayerType);
            OutputLayer.Save(fo);

            if (IsCRFTraining)
            {
                //Save CRF feature weights
                RNNHelper.SaveMatrix(CRFWeights, fo);
            }

            fo.Close();
        }

        public override void LoadModel(string filename, bool bTrain = false)
        {
            Logger.WriteLine("Loading SimpleRNN model: {0}", filename);

            var sr = new StreamReader(filename);
            var br = new BinaryReader(sr.BaseStream);

            IsCRFTraining = br.ReadBoolean();

            //Create cells of each layer
            var layerSize = br.ReadInt32();
            LayerType layerType = LayerType.None;
            HiddenLayerList = new List<ILayer>();
            for (var i = 0; i < layerSize; i++)
            {
                layerType = (LayerType)br.ReadInt32();
                HiddenLayerList.Add(Load(layerType, br, bTrain));

                ILayer layer = HiddenLayerList[HiddenLayerList.Count - 1];
                if (bTrain)
                {
                    layer.SetRunningMode(RunningMode.Training);
                }
                else
                {
                    layer.SetRunningMode(RunningMode.Test);
                }
            }

            Logger.WriteLine("Create output layer");
            layerType = (LayerType)br.ReadInt32();
            OutputLayer = (IOutputLayer)Load(layerType, br, bTrain);

            if (bTrain)
            {
                OutputLayer.SetRunningMode(RunningMode.Training);
            }
            else
            {
                OutputLayer.SetRunningMode(RunningMode.Test);
            }

            if (IsCRFTraining)
            {
                Logger.WriteLine("Loading CRF tag trans weights...");
                CRFWeights = RNNHelper.LoadMatrix(br);
            }

            sr.Close();
        }

        public override void CleanStatusForTraining()
        {
            foreach (var layer in HiddenLayerList)
            {
                layer.CleanForTraining();
            }

            OutputLayer.CleanForTraining();
        }
    }
}

<|start_filename|>RNNSharp/Networks/BiRNNAvg.cs<|end_filename|>
﻿using AdvUtils;
using RNNSharp.Layers;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Text;
using System.Threading.Tasks;

namespace RNNSharp.Networks
{
    class BiRNNAvg<T> : BiRNN<T> where T : ISequence
    {
        public BiRNNAvg() : base()
        {

        }

        protected override void InitLayersOutputCache()
        {
            layersOutput = new List<float[][]>();
            for (int i = 0; i < numOfLayers; i++)
            {
                var layerOutputs = new float[MaxSeqLength][];
                for (int j = 0; j < MaxSeqLength; j++)
                {
                    layerOutputs[j] = new float[forwardHiddenLayers[i].LayerSize];
                }

                layersOutput.Add(layerOutputs);
            }

        }

        public override void CreateNetwork(List<LayerConfig> hiddenLayersConfig, LayerConfig outputLayerConfig, DataSet<T> TrainingSet, Config featurizer)
        {
            var forwardHiddenLayers = CreateLayers(hiddenLayersConfig);
            var backwardHiddenLayers = CreateLayers(hiddenLayersConfig);

            for (var i = 0; i < hiddenLayersConfig.Count; i++)
            {
                ILayer forwardLayer = forwardHiddenLayers[i];
                ILayer backwardLayer = backwardHiddenLayers[i];

                var denseFeatureSize = TrainingSet.DenseFeatureSize;
                var sparseFeatureSize = TrainingSet.SparseFeatureSize;
                if (i > 0)
                {
                    denseFeatureSize = forwardHiddenLayers[i - 1].LayerSize;
                    sparseFeatureSize = 0;
                }

                forwardLayer.InitializeWeights(sparseFeatureSize, denseFeatureSize);
                backwardLayer.InitializeWeights(sparseFeatureSize, denseFeatureSize);

                forwardLayer.SetRunningMode(RunningMode.Training);
                backwardLayer.SetRunningMode(RunningMode.Training);

                Logger.WriteLine($"Create hidden layer {i}: size = {forwardLayer.LayerSize}, sparse feature size = {forwardLayer.SparseFeatureSize}, dense feature size = {forwardLayer.DenseFeatureSize}");
            }

            outputLayerConfig.LayerSize = TrainingSet.TagSize;
            IOutputLayer outputLayer = CreateOutputLayer(outputLayerConfig, 0, forwardHiddenLayers[forwardHiddenLayers.Count - 1].LayerSize);
            outputLayer.SetRunningMode(RunningMode.Training);

            Logger.WriteLine($"Create a bi-directional recurrent neural network with {forwardHiddenLayers.Count} hidden layers. Forward and backward layers are concatnated.");
            InitCache(forwardHiddenLayers, backwardHiddenLayers, outputLayer);
        }

        public override RNN<T> Clone()
        {
            List<ILayer> forwardLayers = new List<ILayer>();
            List<ILayer> backwardLayers = new List<ILayer>();

            foreach (ILayer layer in forwardHiddenLayers)
            {
                forwardLayers.Add(layer.CreateLayerSharedWegiths());
            }

            foreach (ILayer layer in backwardHiddenLayers)
            {
                backwardLayers.Add(layer.CreateLayerSharedWegiths());
            }

            BiRNNAvg<T> rnn = new BiRNNAvg<T>();
            rnn.InitCache(forwardLayers, backwardLayers, (IOutputLayer)OutputLayer.CreateLayerSharedWegiths());
            rnn.CRFWeights = CRFWeights;
            rnn.MaxSeqLength = MaxSeqLength;
            rnn.bVQ = bVQ;
            rnn.IsCRFTraining = IsCRFTraining;
            if (rnn.IsCRFTraining)
            {
                rnn.InitializeCRFVariablesForTraining();
            }

            return rnn;
        }

        protected override void MergeForwardBackwardLayers(int numStates, int layerSize, int layerIdx)
        {
            //Merge forward and backward
            float[][] stateOutputs = layersOutput[layerIdx];


            for (var curState = 0; curState < numStates; curState++)
            {
                var forwardCells = forwardCellList[layerIdx][curState].Cells;
                var backwardCells = backwardCellList[layerIdx][curState].Cells;
                var mergedLayer = stateOutputs[curState];

                var i = 0;
                Vector<float> vecDiv2 = new Vector<float>(2.0f);

                var moreItems = (layerSize % Vector<float>.Count);
                while (i < layerSize - moreItems)
                {
                    var v1 = new Vector<float>(forwardCells, i);
                    var v2 = new Vector<float>(backwardCells, i);
                    var vec = (v1 + v2) / vecDiv2;

                    vec.CopyTo(mergedLayer, i);

                    i += Vector<float>.Count;
                }

                while (i < layerSize)
                {
                    mergedLayer[i] = (forwardCells[i] + backwardCells[i]) / 2.0f;
                    i++;
                }
            }
        }

        public override int GetTopHiddenLayerSize()
        {
                return forwardHiddenLayers[forwardHiddenLayers.Count - 1].LayerSize;
        }


        /// <summary>
        ///     Pass error from the last layer to the first layer
        /// </summary>
        /// <param name="pSequence"></param>
        /// <param name="seqFinalOutput"></param>
        /// <returns></returns>
        protected override void ComputeDeepErr(Sequence pSequence)
        {
            var numStates = pSequence.States.Length;
            var numLayers = forwardHiddenLayers.Count;

            //Calculate output layer error
            for (var curState = 0; curState < numStates; curState++)
            {
                OutputLayer.Cells = OutputCells[curState].Cells;
                OutputLayer.Errs = OutputCells[curState].Errs;
                OutputLayer.ComputeOutputLoss(CRFSeqOutput, pSequence.States[curState], curState);
            }


            ////Now we already have err in output layer, let's pass them back to other layers
            ////Pass error from i+1 to i layer
            var errLayer1 = forwardCellList[numLayers - 1];
            var errLayer2 = backwardCellList[numLayers - 1];

            for (var curState = 0; curState < numStates; curState++)
            {
                OutputLayer.Errs = OutputCells[curState].Errs;
                OutputLayer.ComputeLayerErr(errLayer1[curState].Errs);

                errLayer1[curState].Errs.CopyTo(errLayer2[curState].Errs, 0);
            }

            for (var i = numLayers - 2; i >= 0; i--)
            {
                var lastForwardLayer = forwardHiddenLayers[i + 1];
                var errLayerF = forwardCellList[i];
                var srcErrLayerF = forwardCellList[i + 1];

                var lastBackwardLayer = backwardHiddenLayers[i + 1];
                var errLayerB = backwardCellList[i];
                var srcErrLayerB = backwardCellList[i + 1];

                for (var curState = 0; curState < numStates; curState++)
                {
                    var errLayerFCur = errLayerF[curState];
                    var errLayerBCur = errLayerB[curState];

                    lastForwardLayer.Errs = srcErrLayerF[curState].Errs;
                    lastForwardLayer.ComputeLayerErr(errLayerFCur.Errs);

                    lastBackwardLayer.Errs = srcErrLayerB[curState].Errs;
                    lastBackwardLayer.ComputeLayerErr(errLayerBCur.Errs);
                }

            }

        }
    }
}


<|start_filename|>RNNSharp/Layers/ILayer.cs<|end_filename|>
﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace RNNSharp.Layers
{
    public interface ILayer
    {
        float[] Cells { get; set; }
        float[] Errs { get; set; }
        int LayerSize { get; }
        LayerType LayerType { get; }
        LayerConfig LayerConfig { get; set; }
        int SparseFeatureSize { get; set; }
        int DenseFeatureSize { get; set; }
        List<SparseVector> SparseFeatureGroups { get; set; }
        List<float[]> DenseFeatureGroups { get; set; }
        ILayer CreateLayerSharedWegiths();
        Neuron CopyNeuronTo(Neuron neuron);
        void SetNeuron(Neuron neuron);
        void InitializeWeights(int sparseFeatureSize, int denseFeatureSize);

        void Save(BinaryWriter fo);
        void Load(BinaryReader br, LayerType layerType, bool forTraining = false);

        void CleanForTraining();
        void ForwardPass(List<SparseVector> sparseFeatureGroups, List<float[]> denseFeatureGroup);
        void ForwardPass(SparseVector sparseFeature, float[] denseFeature);
        void ComputeLayerErr(ILayer prevLayer);
        void ComputeLayerErr(List<float[]> destErrsList, bool cleanDest = true);
        void ComputeLayerErr(float[] destErr, bool cleanDest = true);
        void BackwardPass();
        void UpdateWeights();
        void Reset();
        void SetRunningMode(RunningMode mode);
    }
}


<|start_filename|>RNNSharp/Layers/IOutputLayer.cs<|end_filename|>
﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace RNNSharp.Layers
{
    public interface IOutputLayer : ILayer
    {
        List<int> LabelShortList { get; set; }
        int GetBestOutputIndex();
        void ComputeOutputLoss(Matrix<float> CRFSeqOutput, State state, int timeat);
    }
}


<|start_filename|>RNNSharp/Networks/BiRNN.cs<|end_filename|>
﻿using AdvUtils;
using RNNSharp.Layers;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Numerics;
using System.Threading.Tasks;

/// <summary>
/// RNNSharp written by <NAME> (<EMAIL>)
/// </summary>

namespace RNNSharp.Networks
{
    internal class BiRNN<T> : RNN<T> where T : ISequence
    {
        protected List<ILayer> backwardHiddenLayers = new List<ILayer>();
        protected List<ILayer> forwardHiddenLayers = new List<ILayer>();
        protected int numOfLayers => forwardHiddenLayers.Count;
        protected List<float[][]> layersOutput;
        protected List<Neuron[]> forwardCellList;
        protected List<Neuron[]> backwardCellList;
 //IOutputLayer[] seqFinalOutput;
        protected Neuron[] OutputCells;

        public override void UpdateWeights()
        {
            foreach (var layer in backwardHiddenLayers)
            {
                layer.UpdateWeights();
            }

            foreach (var layer in forwardHiddenLayers)
            {
                layer.UpdateWeights();
            }

            OutputLayer.UpdateWeights();
        }

        protected void InitCache(List<ILayer> s_forwardRNN, List<ILayer> s_backwardRNN, IOutputLayer outputLayer)
        {
            forwardHiddenLayers = s_forwardRNN;
            backwardHiddenLayers = s_backwardRNN;

            //Initialize output layer
            OutputLayer = outputLayer;

            forwardCellList = new List<Neuron[]>();
            backwardCellList = new List<Neuron[]>();

            for (int i = 0; i < numOfLayers; i++)
            {
                var forwardCells = new Neuron[MaxSeqLength];
                var backwardCells = new Neuron[MaxSeqLength];

                for (int j = 0; j < MaxSeqLength; j++)
                {
                    if (forwardHiddenLayers[i] is DropoutLayer)
                    {
                        forwardCells[j] = new DropoutNeuron(forwardHiddenLayers[i].LayerSize);
                        backwardCells[j] = new DropoutNeuron(forwardHiddenLayers[i].LayerSize);
                    }
                    else if (forwardHiddenLayers[i] is LSTMLayer)
                    {
                        forwardCells[j] = new LSTMNeuron(forwardHiddenLayers[i].LayerSize);
                        backwardCells[j] = new LSTMNeuron(forwardHiddenLayers[i].LayerSize);
                    }
                    else
                    {
                        forwardCells[j] = new Neuron(forwardHiddenLayers[i].LayerSize);
                        backwardCells[j] = new Neuron(forwardHiddenLayers[i].LayerSize);
                    }
                }

                forwardCellList.Add(forwardCells);
                backwardCellList.Add(backwardCells);
            }

            OutputCells = new Neuron[MaxSeqLength];
            for (var i = 0; i < MaxSeqLength; i++)
            {
                OutputCells[i] = new Neuron(OutputLayer.LayerSize);
            }

            InitLayersOutputCache();
        }

        public BiRNN()
        {
        }

     
        protected virtual void InitLayersOutputCache()
        {
            layersOutput = new List<float[][]>();
            for (int i = 0; i < numOfLayers; i++)
            {
                var layerOutputs = new float[MaxSeqLength][];
                for (int j = 0; j < MaxSeqLength; j++)
                {
                    layerOutputs[j] = new float[forwardHiddenLayers[i].LayerSize * 2];
                }

                layersOutput.Add(layerOutputs);
            }
        }

        public override void CreateNetwork(List<LayerConfig> hiddenLayersConfig, LayerConfig outputLayerConfig, DataSet<T> TrainingSet, Config featurizer)
        {
            var forwardHiddenLayers = CreateLayers(hiddenLayersConfig);
            var backwardHiddenLayers = CreateLayers(hiddenLayersConfig);

            for (var i = 0; i < hiddenLayersConfig.Count; i++)
            {
                ILayer forwardLayer = forwardHiddenLayers[i];
                ILayer backwardLayer = backwardHiddenLayers[i];

                var denseFeatureSize = TrainingSet.DenseFeatureSize;
                if (i > 0)
                {
                    denseFeatureSize = forwardHiddenLayers[i - 1].LayerSize * 2;
                }

                forwardLayer.InitializeWeights(TrainingSet.SparseFeatureSize, denseFeatureSize);
                backwardLayer.InitializeWeights(TrainingSet.SparseFeatureSize, denseFeatureSize);

                forwardLayer.SetRunningMode(RunningMode.Training);
                backwardLayer.SetRunningMode(RunningMode.Training);

                Logger.WriteLine($"Create hidden layer {i}: size = {forwardLayer.LayerSize}, sparse feature size = {forwardLayer.SparseFeatureSize}, dense feature size = {forwardLayer.DenseFeatureSize}");
            }

            outputLayerConfig.LayerSize = TrainingSet.TagSize;
            IOutputLayer outputLayer = CreateOutputLayer(outputLayerConfig, TrainingSet.SparseFeatureSize, forwardHiddenLayers[forwardHiddenLayers.Count - 1].LayerSize * 2);
            outputLayer.SetRunningMode(RunningMode.Training);

            Logger.WriteLine($"Create a bi-directional recurrent neural network with {forwardHiddenLayers.Count} hidden layers. Forward and backward layers are concatnated.");
            InitCache(forwardHiddenLayers, backwardHiddenLayers, outputLayer);
        }


        public override RNN<T> Clone()
        {
            List<ILayer> forwardLayers = new List<ILayer>();
            List<ILayer> backwardLayers = new List<ILayer>();

            foreach (ILayer layer in forwardHiddenLayers)
            {
                forwardLayers.Add(layer.CreateLayerSharedWegiths());
            }

            foreach (ILayer layer in backwardHiddenLayers)
            {
                backwardLayers.Add(layer.CreateLayerSharedWegiths());
            }

            BiRNN<T> rnn = new BiRNN<T>();
            rnn.InitCache(forwardLayers, backwardLayers, (IOutputLayer)OutputLayer.CreateLayerSharedWegiths());
            rnn.CRFWeights = CRFWeights;
            rnn.MaxSeqLength = MaxSeqLength;
            rnn.bVQ = bVQ;
            rnn.IsCRFTraining = IsCRFTraining;
            if (rnn.IsCRFTraining)
            {
                rnn.InitializeCRFVariablesForTraining();
            }

            return rnn;
        }

        public override void CleanStatusForTraining()
        {
            foreach (var layer in forwardHiddenLayers)
            {
                layer.CleanForTraining();
            }

            foreach (var layer in backwardHiddenLayers)
            {
                layer.CleanForTraining();
            }

            OutputLayer.CleanForTraining();
        }

        protected virtual void MergeForwardBackwardLayers(int numStates, int layerSize, int layerIdx)
        {
            //Merge forward and backward
            float[][] stateOutputs = layersOutput[layerIdx];
            for (var curState = 0; curState < numStates; curState++)
            {
                var forwardCells = forwardCellList[layerIdx][curState].Cells;
                var backwardCells = backwardCellList[layerIdx][curState].Cells;
                var mergedLayer = stateOutputs[curState];

                var i = 0;
                var moreItems = (layerSize % Vector<float>.Count);
                while (i < layerSize - moreItems)
                {
                    var v1 = new Vector<float>(forwardCells, i);
                    var v2 = new Vector<float>(backwardCells, i);

                    v1.CopyTo(mergedLayer, i);
                    v2.CopyTo(mergedLayer, layerSize + i);

                    i += Vector<float>.Count;
                }

                while (i < layerSize)
                {
                    mergedLayer[i] = forwardCells[i];
                    mergedLayer[layerSize + i] = backwardCells[i];
                    i++;
                }
            }
        }



        private void ComputeMiddleLayers(Sequence pSequence, ILayer forwardLayer, ILayer backwardLayer, RunningMode runningMode, int layerIdx)
        {
            var numStates = pSequence.States.Length;
            float[][] lastLayerOutputs = layersOutput[layerIdx - 1];


            //Computing forward RNN
            forwardLayer.Reset();
            for (var curState = 0; curState < numStates; curState++)
            {
                var state = pSequence.States[curState];
                forwardLayer.ForwardPass(state.SparseFeature, lastLayerOutputs[curState]);
                forwardLayer.CopyNeuronTo(forwardCellList[layerIdx][curState]);
            }

            //Computing backward RNN
            backwardLayer.Reset();
            for (var curState = numStates - 1; curState >= 0; curState--)
            {
                var state = pSequence.States[curState];
                backwardLayer.ForwardPass(state.SparseFeature, lastLayerOutputs[curState]);
                backwardLayer.CopyNeuronTo(backwardCellList[layerIdx][curState]);
            }


            //Merge forward and backward
            MergeForwardBackwardLayers(numStates, forwardLayer.LayerSize, layerIdx);
        }

        /// <summary>
        ///     Compute the output of bottom layer
        /// </summary>
        /// <param name="sequence"></param>
        /// <param name="forwardLayer"></param>
        /// <param name="backwardLayer"></param>
        /// <returns></returns>
        private void ComputeBottomLayer(Sequence sequence, ILayer forwardLayer, ILayer backwardLayer, RunningMode runningMode)
        {
            var numStates = sequence.States.Length;

            //Computing forward RNN
            forwardLayer.Reset();
            for (var curState = 0; curState < numStates; curState++)
            {
                var state = sequence.States[curState];
                forwardLayer.ForwardPass(state.SparseFeature, state.DenseFeature.CopyTo());
                forwardLayer.CopyNeuronTo(forwardCellList[0][curState]);
            }


            //Computing backward RNN
            backwardLayer.Reset();
            for (var curState = numStates - 1; curState >= 0; curState--)
            {
                var state = sequence.States[curState];
                backwardLayer.ForwardPass(state.SparseFeature, state.DenseFeature.CopyTo());
                backwardLayer.CopyNeuronTo(backwardCellList[0][curState]);
            }

            //Merge forward and backward
            MergeForwardBackwardLayers(numStates, forwardLayer.LayerSize, 0);
        }

        private void ComputeTopLayer(Sequence pSequence, out Matrix<float> rawOutputLayer, RunningMode runningMode, bool outputRawScore)
        {
            var numStates = pSequence.States.Length;
            var lastLayerOutputs = layersOutput[forwardHiddenLayers.Count - 1];

            //Calculate output layer
            Matrix<float> tmpOutputResult = null;
            if (outputRawScore)
            {
                tmpOutputResult = new Matrix<float>(numStates, OutputLayer.LayerSize);
            }

            OutputLayer.LabelShortList = pSequence.States.Select(state => state.Label).ToList();
            for (var curState = 0; curState < numStates; curState++)
            {
                var state = pSequence.States[curState];
                OutputLayer.ForwardPass(state.SparseFeature, lastLayerOutputs[curState]);
                OutputLayer.CopyNeuronTo(OutputCells[curState]);

                if (outputRawScore)
                {
                    OutputLayer.Cells.CopyTo(tmpOutputResult[curState], 0);
                }
            }

            rawOutputLayer = tmpOutputResult;
        }

        public override int GetTopHiddenLayerSize()
        {
            return forwardHiddenLayers[forwardHiddenLayers.Count - 1].LayerSize * 2;
        }

        public override float[][] ComputeTopHiddenLayerOutput(Sequence pSequence)
        {
            ComputeBottomLayer(pSequence, forwardHiddenLayers[0], backwardHiddenLayers[0], RunningMode.Test);
            for (var i = 1; i < forwardHiddenLayers.Count; i++)
            {
                ComputeMiddleLayers(pSequence, forwardHiddenLayers[i], backwardHiddenLayers[i], RunningMode.Test, i);
            }

            return layersOutput[forwardHiddenLayers.Count - 1];
        }

        /// <summary>
        ///     Computing the output of each layer in the neural network
        /// </summary>
        /// <param name="pSequence"></param>
        /// <param name="isTraining"></param>
        /// <param name="layerList"></param>
        /// <param name="rawOutputLayer"></param>
        /// <returns></returns>
        private void ComputeLayers(Sequence pSequence, RunningMode runningMode, out Matrix<float> rawOutputLayer, bool outputRawScore)
        {
            ComputeBottomLayer(pSequence, forwardHiddenLayers[0], backwardHiddenLayers[0], runningMode);
            for (var i = 1; i < forwardHiddenLayers.Count; i++)
            {
                ComputeMiddleLayers(pSequence, forwardHiddenLayers[i], backwardHiddenLayers[i], runningMode, i);
            }

            ComputeTopLayer(pSequence, out rawOutputLayer, runningMode, outputRawScore);
        }

        /// <summary>
        ///     Pass error from the last layer to the first layer
        /// </summary>
        /// <param name="pSequence"></param>
        /// <param name="seqFinalOutput"></param>
        /// <returns></returns>
        protected virtual void ComputeDeepErr(Sequence pSequence)
        {
            var numStates = pSequence.States.Length;
            var numLayers = forwardHiddenLayers.Count;

            //Calculate output layer error
            for (var curState = 0; curState < numStates; curState++)
            {
                OutputLayer.Cells = OutputCells[curState].Cells;
                OutputLayer.Errs = OutputCells[curState].Errs;
                OutputLayer.ComputeOutputLoss(CRFSeqOutput, pSequence.States[curState], curState);
            }


            ////Now we already have err in output layer, let's pass them back to other layers
            ////Pass error from i+1 to i layer
            var errLayer1 = forwardCellList[numLayers - 1];
            var errLayer2 = backwardCellList[numLayers - 1];

            for (var curState = 0; curState < numStates; curState++)
            {
                List<float[]> destErrsList = new List<float[]>();
                destErrsList.Add(errLayer1[curState].Errs);
                destErrsList.Add(errLayer2[curState].Errs);

                OutputLayer.Errs = OutputCells[curState].Errs;
                OutputLayer.ComputeLayerErr(destErrsList);
            }

            Vector<float> vecTwo = new Vector<float>(2.0f);
            for (var i = numLayers - 2; i >= 0; i--)
            {
                var lastForwardLayer = forwardHiddenLayers[i + 1];
                var errLayerF = forwardCellList[i];
                var srcErrLayerF = forwardCellList[i + 1];

                var lastBackwardLayer = backwardHiddenLayers[i + 1];
                var errLayerB = backwardCellList[i]; 
                var srcErrLayerB = backwardCellList[i + 1];

                for (var curState = 0; curState < numStates; curState++)
                {
                    var errLayerFCur = errLayerF[curState];
                    var errLayerBCur = errLayerB[curState];

                    List<float[]> destErrList = new List<float[]>();
                    destErrList.Add(errLayerFCur.Errs);
                    destErrList.Add(errLayerBCur.Errs);

                    lastForwardLayer.Errs = srcErrLayerF[curState].Errs;
                    lastForwardLayer.ComputeLayerErr(destErrList);

                    lastBackwardLayer.Errs = srcErrLayerB[curState].Errs;
                    lastBackwardLayer.ComputeLayerErr(destErrList, false);


                    int j = 0;
                    int errLength = errLayerFCur.Errs.Length;
                    var moreItems = (errLength % Vector<float>.Count);
                    while (j < errLength - moreItems)
                    {
                        Vector<float> vecErrLayerF = new Vector<float>(errLayerFCur.Errs, j);
                        Vector<float> vecErrLayerB = new Vector<float>(errLayerBCur.Errs, j);

                        vecErrLayerF /= vecTwo;
                        vecErrLayerB /= vecTwo;

                        vecErrLayerF.CopyTo(errLayerFCur.Errs, j);
                        vecErrLayerB.CopyTo(errLayerBCur.Errs, j);

                        j += Vector<float>.Count;
                    }

                    while (j < errLength)
                    {
                        errLayerFCur.Errs[j] /= 2.0f;
                        errLayerBCur.Errs[j] /= 2.0f;

                        j++;
                    }

                }

            }

        }
        protected virtual void DeepLearningNet(Sequence pSequence)
        {
            var numStates = pSequence.States.Length;
            var numLayers = forwardHiddenLayers.Count;

            //Learning output layer
            for (var curState = 0; curState < numStates; curState++)
            {
                State state = pSequence.States[curState];
                UpdateWeightsAt(OutputLayer, OutputCells[curState], state.SparseFeature, layersOutput[numLayers - 1][curState]);
            }

            for (var i = 0; i < numLayers; i++)
            {
                float[][] layerOutputs_i = (i > 0) ? layersOutput[i - 1] : null;
                Neuron[] forwardNeurons = forwardCellList[i];
                var forwardLayer = forwardHiddenLayers[i];
                for (var curState = 0; curState < numStates; curState++)
                {
                    State state = pSequence.States[curState];
                    UpdateWeightsAt(forwardLayer, forwardNeurons[curState], state.SparseFeature, (i == 0) ? state.DenseFeature.CopyTo() : layerOutputs_i[curState]);
                }

                Neuron[] backwardNeurons = backwardCellList[i];
                var backwardLayer = backwardHiddenLayers[i];
                for (var curState2 = numStates - 1;curState2 >= 0;curState2--)
                {
                    State state = pSequence.States[curState2];
                    UpdateWeightsAt(backwardLayer, backwardNeurons[curState2], state.SparseFeature, (i == 0) ? state.DenseFeature.CopyTo() : layerOutputs_i[curState2]);
                }
            }
        }

        protected void UpdateWeightsAt(ILayer layer, Neuron neuron, SparseVector sparseFeature, float[] denseFeature)
        {
            layer.SparseFeatureGroups = new List<SparseVector>();
            layer.SparseFeatureGroups.Add(sparseFeature);
            layer.DenseFeatureGroups = new List<float[]>();
            layer.DenseFeatureGroups.Add(denseFeature);
            layer.SetNeuron(neuron);
            layer.BackwardPass();
        }

        public override int[] ProcessSequence(ISentence sentence, Config featurizer, RunningMode runningMode, bool outputRawScore, out Matrix<float> m)
        {
            var seq = featurizer.BuildSequence(sentence as Sentence);
            return ProcessSequence(seq, runningMode, outputRawScore, out m);
        }

        /// <summary>
        ///     Process a given sequence by bi-directional recurrent neural network
        /// </summary>
        /// <param name="pSequence"></param>
        /// <param name="runningMode"></param>
        /// <returns></returns>
        public override int[] ProcessSequence(ISequence sequence, RunningMode runningMode, bool outputRawScore, out Matrix<float> rawOutputLayer)
        {
            Sequence pSequence = sequence as Sequence;
            //Forward process from bottom layer to top layer
            int[] seqBestOutput;
            ComputeLayers(pSequence, runningMode, out rawOutputLayer, outputRawScore);

            //Get best output result of each state
            var numStates = pSequence.States.Length;
            seqBestOutput = new int[numStates];

            for (var curState = 0; curState < numStates; curState++)
            {
                seqBestOutput[curState] = OutputCells[curState].GetMaxCellIndex();
            }

            if (runningMode == RunningMode.Training)
            {
                //In training mode, we calculate each layer's error and update their net weights
                ComputeDeepErr(pSequence);
                DeepLearningNet(pSequence);
            }

            return seqBestOutput;
        }

        /// <summary>
        ///     Process a given sequence by bi-directional recurrent neural network and CRF
        /// </summary>
        /// <param name="pSequence"></param>
        /// <param name="runningMode"></param>
        /// <returns></returns>
        public override int[] ProcessSequenceCRF(Sequence pSequence, RunningMode runningMode)
        {
            //Reset the network
            var numStates = pSequence.States.Length;
            Matrix<float> rawOutputLayer;

            ComputeLayers(pSequence, runningMode, out rawOutputLayer, true);

            ForwardBackward(numStates, rawOutputLayer);

            var predict = Viterbi(rawOutputLayer, numStates);

            if (runningMode == RunningMode.Training)
            {
                UpdateBigramTransition(pSequence);
                ComputeDeepErr(pSequence);
                DeepLearningNet(pSequence);
            }

            return predict;
        }

        public override void SaveModel(string filename)
        {
            //Save meta data
            using (var sw = new StreamWriter(filename))
            {
                var fo = new BinaryWriter(sw.BaseStream);
                fo.Write(IsCRFTraining);
                fo.Write(forwardHiddenLayers.Count);
                //Save forward layers
                foreach (var layer in forwardHiddenLayers)
                {
                    fo.Write((int)layer.LayerType);
                    layer.Save(fo);
                }
                //Save backward layers
                foreach (var layer in backwardHiddenLayers)
                {
                    fo.Write((int)layer.LayerType);
                    layer.Save(fo);
                }
                //Save output layer
                fo.Write((int)OutputLayer.LayerType);
                OutputLayer.Save(fo);

                if (IsCRFTraining)
                {
                    // Save CRF features weights
                    RNNHelper.SaveMatrix(CRFWeights, fo);
                }
            }
        }

        public override void LoadModel(string filename, bool bTrain = false)
        {
            Logger.WriteLine(Logger.Level.info, "Loading bi-directional model: {0}", filename);

            using (var sr = new StreamReader(filename))
            {
                var br = new BinaryReader(sr.BaseStream);

                IsCRFTraining = br.ReadBoolean();
                var layerSize = br.ReadInt32();
                LayerType layerType = LayerType.None;

                //Load forward layers from file
                forwardHiddenLayers = new List<ILayer>();
                for (var i = 0; i < layerSize; i++)
                {
                    layerType = (LayerType)br.ReadInt32();
                    forwardHiddenLayers.Add(Load(layerType, br, bTrain));

                    ILayer layer = forwardHiddenLayers[forwardHiddenLayers.Count - 1];
                    if (bTrain)
                    {
                        layer.SetRunningMode(RunningMode.Training);
                    }
                    else
                    {
                        layer.SetRunningMode(RunningMode.Test);
                    }
                }

                //Load backward layers from file
                backwardHiddenLayers = new List<ILayer>();
                for (var i = 0; i < layerSize; i++)
                {
                    layerType = (LayerType)br.ReadInt32();
                    backwardHiddenLayers.Add(Load(layerType, br, bTrain));

                    ILayer layer = backwardHiddenLayers[backwardHiddenLayers.Count - 1];
                    if (bTrain)
                    {
                        layer.SetRunningMode(RunningMode.Training);
                    }
                    else
                    {
                        layer.SetRunningMode(RunningMode.Test);
                    }
                }

                Logger.WriteLine("Create output layer");
                layerType = (LayerType)br.ReadInt32();
                OutputLayer = (IOutputLayer)Load(layerType, br, bTrain);

                if (bTrain)
                {
                    OutputLayer.SetRunningMode(RunningMode.Training);
                }
                else
                {
                    OutputLayer.SetRunningMode(RunningMode.Test);
                }

                if (IsCRFTraining)
                {
                    Logger.WriteLine("Loading CRF tag trans weights...");
                    CRFWeights = RNNHelper.LoadMatrix(br);
                }

                if (bTrain)
                {
                    InitCache(forwardHiddenLayers, backwardHiddenLayers, (IOutputLayer)OutputLayer.CreateLayerSharedWegiths());
                }
            }
        }
    }
}

<|start_filename|>RNNSharp/Networks/ForwardRNNSeq2Seq.cs<|end_filename|>
﻿using AdvUtils;
using RNNSharp.Layers;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Text;
using System.Threading.Tasks;

namespace RNNSharp.Networks
{
    class ForwardRNNSeq2Seq<T> : ForwardRNN<T> where T : ISequence
    {
        List<float[]>[] denseFeatureGroupsList;
        List<float[]> denseFeatureGroupsOutputLayer;
        List<SparseVector> sparseFeatureGorups;
        int numSrcDenseFeatureGroups = 2;

        public ForwardRNNSeq2Seq()
            : base()
        {

        }

        public override void CreateNetwork(List<LayerConfig> hiddenLayersConfig, LayerConfig outputLayerConfig, DataSet<T> TrainingSet, Config featurizer)
        {
            var sparseFeatureSize = TrainingSet.SparseFeatureSize;
            sparseFeatureSize += featurizer.Seq2SeqAutoEncoder.Config.SparseFeatureSize;
            sparseFeatureGorups = new List<SparseVector>();

            Logger.WriteLine("Sparse Feature Format: [{0}][{1}] = {2}", TrainingSet.SparseFeatureSize, featurizer.Seq2SeqAutoEncoder.Config.SparseFeatureSize, sparseFeatureSize);

            //Create all hidden layers
            HiddenLayerList = CreateLayers(hiddenLayersConfig);

            var srcDenseFeatureSize = featurizer.Seq2SeqAutoEncoder.GetTopHiddenLayerSize() * numSrcDenseFeatureGroups;
            denseFeatureGroupsList = new List<float[]>[HiddenLayerList.Count];
            for (var i = 0; i < HiddenLayerList.Count; i++)
            {
                ILayer layer = HiddenLayerList[i];
                layer.InitializeWeights(sparseFeatureSize, i == 0 ? (srcDenseFeatureSize + TrainingSet.DenseFeatureSize) : (srcDenseFeatureSize + HiddenLayerList[i - 1].LayerSize));
                layer.SetRunningMode(RunningMode.Training);

                Logger.WriteLine($"Create hidden layer {i}: size = {layer.LayerSize}, sparse feature size = {layer.SparseFeatureSize}, dense feature size = {layer.DenseFeatureSize}");

                denseFeatureGroupsList[i] = new List<float[]>();

            }

            //Create output layer
            outputLayerConfig.LayerSize = TrainingSet.TagSize;
            OutputLayer = CreateOutputLayer(outputLayerConfig, sparseFeatureSize, (srcDenseFeatureSize + HiddenLayerList[HiddenLayerList.Count - 1].LayerSize));
            OutputLayer.SetRunningMode(RunningMode.Training);

            Logger.WriteLine($"Create a Forward recurrent neural sequence-to-sequence network with {HiddenLayerList.Count} hidden layers");

            denseFeatureGroupsOutputLayer = new List<float[]>();
        }

        public override RNN<T> Clone()
        {
            List<ILayer> hiddenLayers = new List<ILayer>();
            List<float[]>[] denseFeatureGroupsList = new List<float[]>[HiddenLayerList.Count];

            int i = 0;
            foreach (ILayer layer in HiddenLayerList)
            {
                hiddenLayers.Add(layer.CreateLayerSharedWegiths());
                denseFeatureGroupsList[i] = new List<float[]>();
                i++;
            }

            List<float[]> denseFeatureGroupsOutputLayer = new List<float[]>();

            ForwardRNNSeq2Seq<T> rnn = new ForwardRNNSeq2Seq<T>();
            rnn.HiddenLayerList = hiddenLayers;
            rnn.OutputLayer = (IOutputLayer)OutputLayer.CreateLayerSharedWegiths();
            rnn.CRFWeights = CRFWeights;
            rnn.denseFeatureGroupsList = denseFeatureGroupsList;
            rnn.denseFeatureGroupsOutputLayer = denseFeatureGroupsOutputLayer;
            rnn.sparseFeatureGorups = new List<SparseVector>();
            rnn.MaxSeqLength = MaxSeqLength;
            rnn.bVQ = bVQ;
            rnn.IsCRFTraining = IsCRFTraining;
            if (rnn.IsCRFTraining)
            {
                rnn.InitializeCRFVariablesForTraining();
            }

            return rnn;
        }

        /// <summary>
        /// Extract features from source sequence
        /// </summary>
        /// <param name="decoder"></param>
        /// <param name="srcSequence"></param>
        /// <param name="targetSparseFeatureSize"></param>
        /// <param name="srcHiddenAvgOutput"></param>
        /// <param name="srcSparseFeatures"></param>
        private void ExtractSourceSentenceFeature(RNNDecoder decoder, Sequence srcSequence, int targetSparseFeatureSize, List<float[]> srcDenseFeatureGroups, SparseVector srcSparseFeatures)
        {
            //Extract dense features from source sequence
            var srcOutputs = decoder.ComputeTopHiddenLayerOutput(srcSequence);
            int srcSequenceLength = srcOutputs.Length - 1;
            srcDenseFeatureGroups.Add(srcOutputs[srcSequenceLength]);

            if (numSrcDenseFeatureGroups > 1)
            {
                srcDenseFeatureGroups.Add(srcOutputs[0]);
            }

            if (numSrcDenseFeatureGroups > 2)
            {
                float srcOffsetPerBlock = (float)srcSequenceLength / (float)(numSrcDenseFeatureGroups - 1);
                if (srcOffsetPerBlock < 1.0)
                {
                    srcOffsetPerBlock = 1.0f;
                }

                float idx = srcOffsetPerBlock;
                while (srcDenseFeatureGroups.Count < numSrcDenseFeatureGroups && idx < srcSequenceLength)
                {
                    srcDenseFeatureGroups.Add(srcOutputs[(int)idx]);
                    idx += srcOffsetPerBlock;
                }
            }

            //Extract sparse features from source sequence
            Dictionary<int, float> srcSparseFeaturesDict = new Dictionary<int, float>();
            for (var i = 0; i < srcSequence.States.Length; i++)
            {
                foreach (var kv in srcSequence.States[i].SparseFeature)
                {
                    var srcSparseFeatureIndex = kv.Key + targetSparseFeatureSize;

                    if (srcSparseFeaturesDict.ContainsKey(srcSparseFeatureIndex) == false)
                    {
                        srcSparseFeaturesDict.Add(srcSparseFeatureIndex, kv.Value);
                    }
                    else
                    {
                        srcSparseFeaturesDict[srcSparseFeatureIndex] += kv.Value;
                    }
                }
            }

            srcSparseFeatures.SetLength(srcSequence.SparseFeatureSize + targetSparseFeatureSize);
            srcSparseFeatures.AddKeyValuePairData(srcSparseFeaturesDict);
        }

        public override int[] ProcessSequence(ISentence sentence, Config featurizer, RunningMode runningMode, bool outputRawScore, out Matrix<float> m)
        {
            if (runningMode == RunningMode.Training)
            {
                var sequencePair = featurizer.BuildSequence(sentence as SentencePair);
                return TrainSequencePair(sequencePair, runningMode, outputRawScore, out m);
            }
            else
            {
                return PredictTargetSentence(sentence as Sentence, featurizer, out m);
            }
        }

        private int[] PredictTargetSentence(Sentence sentence, Config featurizer, out Matrix<float> m)
        {
            m = null;

            var curState = featurizer.BuildState(new[] { "<s>" });
            curState.Label = featurizer.TagSet.GetIndex("<s>");

            //Reset all layers
            foreach (var layer in HiddenLayerList)
            {
                layer.Reset();
            }

            //Extract features from source sentence
            var srcSequence = featurizer.Seq2SeqAutoEncoder.Config.BuildSequence(sentence);
            List<float[]> srcDenseFeatureGorups = new List<float[]>();
            SparseVector srcSparseFeatures = new SparseVector();
            ExtractSourceSentenceFeature(featurizer.Seq2SeqAutoEncoder, srcSequence, curState.SparseFeature.Length, srcDenseFeatureGorups, srcSparseFeatures);

            var numLayers = HiddenLayerList.Count;
            var predicted = new List<int> { curState.Label };

            //Set sparse feature group from source sequence
            sparseFeatureGorups.Clear();
            sparseFeatureGorups.Add(srcSparseFeatures);
            sparseFeatureGorups.Add(null);
            int targetSparseFeatureIndex = sparseFeatureGorups.Count - 1;

            //Set dense feature groups from source sequence
            for (var i = 0; i < numLayers; i++)
            {
                denseFeatureGroupsList[i].Clear();
                denseFeatureGroupsList[i].AddRange(srcDenseFeatureGorups);
                denseFeatureGroupsList[i].Add(null);
            }
            denseFeatureGroupsOutputLayer.Clear();
            denseFeatureGroupsOutputLayer.AddRange(srcDenseFeatureGorups);
            denseFeatureGroupsOutputLayer.Add(null);
            int targetDenseFeatureIndex = denseFeatureGroupsOutputLayer.Count - 1;

            while (true)
            {
                //Set sparse feature groups
                sparseFeatureGorups[targetSparseFeatureIndex] = curState.SparseFeature;

                //Compute first layer
                denseFeatureGroupsList[0][targetDenseFeatureIndex] = curState.DenseFeature.CopyTo();
                HiddenLayerList[0].ForwardPass(sparseFeatureGorups, denseFeatureGroupsList[0]);

                //Compute middle layers
                for (var i = 1; i < numLayers; i++)
                {
                    //We use previous layer's output as dense feature for current layer
                    denseFeatureGroupsList[i][targetDenseFeatureIndex] = HiddenLayerList[i - 1].Cells;
                    HiddenLayerList[i].ForwardPass(sparseFeatureGorups, denseFeatureGroupsList[i]);
                }

                //Compute output layer
                denseFeatureGroupsOutputLayer[targetDenseFeatureIndex] = HiddenLayerList[numLayers - 1].Cells;
                OutputLayer.ForwardPass(sparseFeatureGorups, denseFeatureGroupsOutputLayer);


                var nextTagId = OutputLayer.GetBestOutputIndex();
                var nextWord = featurizer.TagSet.GetTagName(nextTagId);

                curState = featurizer.BuildState(new[] { nextWord });
                curState.Label = nextTagId;

                predicted.Add(nextTagId);

                if (nextWord == "</s>" || predicted.Count >= 100)
                {
                    break;
                }
            }

            return predicted.ToArray();
        }

        public override int[] ProcessSequence(ISequence sequence, RunningMode runningMode, bool outputRawScore, out Matrix<float> m)
        {
            return TrainSequencePair(sequence, runningMode, outputRawScore, out m);
        }

        private int[] TrainSequencePair(ISequence sequence, RunningMode runningMode, bool outputRawScore, out Matrix<float> m)
        {
            SequencePair pSequence = sequence as SequencePair;
            var tgtSequence = pSequence.tgtSequence;

            //Reset all layers
            foreach (var layer in HiddenLayerList)
            {
                layer.Reset();
            }

            Sequence srcSequence;

            //Extract features from source sentences
            srcSequence = pSequence.autoEncoder.Config.BuildSequence(pSequence.srcSentence);
            List<float[]> srcDenseFeatureGorups = new List<float[]>();
            SparseVector srcSparseFeatures = new SparseVector();
            ExtractSourceSentenceFeature(pSequence.autoEncoder, srcSequence, tgtSequence.SparseFeatureSize, srcDenseFeatureGorups, srcSparseFeatures);

            var numStates = pSequence.tgtSequence.States.Length;
            var numLayers = HiddenLayerList.Count;
            var predicted = new int[numStates];

            m = outputRawScore ? new Matrix<float>(numStates, OutputLayer.LayerSize) : null;

            //Set target sentence labels into short list in output layer
            OutputLayer.LabelShortList.Clear();
            foreach (var state in tgtSequence.States)
            {
                OutputLayer.LabelShortList.Add(state.Label);
            }

            //Set sparse feature group from source sequence
            sparseFeatureGorups.Clear();
            sparseFeatureGorups.Add(srcSparseFeatures);
            sparseFeatureGorups.Add(null);
            int targetSparseFeatureIndex = sparseFeatureGorups.Count - 1;

            //Set dense feature groups from source sequence
            for (var i = 0; i < numLayers; i++)
            {
                denseFeatureGroupsList[i].Clear();
                denseFeatureGroupsList[i].AddRange(srcDenseFeatureGorups);
                denseFeatureGroupsList[i].Add(null);
            }
            denseFeatureGroupsOutputLayer.Clear();
            denseFeatureGroupsOutputLayer.AddRange(srcDenseFeatureGorups);
            denseFeatureGroupsOutputLayer.Add(null);
            int targetDenseFeatureIndex = denseFeatureGroupsOutputLayer.Count - 1;

            for (var curState = 0; curState < numStates; curState++)
            {
                var state = tgtSequence.States[curState];

                //Set sparse feature groups
                sparseFeatureGorups[targetSparseFeatureIndex] = state.SparseFeature;

                //Compute first layer
                denseFeatureGroupsList[0][targetDenseFeatureIndex] = state.DenseFeature.CopyTo();
                HiddenLayerList[0].ForwardPass(sparseFeatureGorups, denseFeatureGroupsList[0]);

                //Compute middle layers
                for (var i = 1; i < numLayers; i++)
                {
                    //We use previous layer's output as dense feature for current layer
                    denseFeatureGroupsList[i][targetDenseFeatureIndex] = HiddenLayerList[i - 1].Cells;
                    HiddenLayerList[i].ForwardPass(sparseFeatureGorups, denseFeatureGroupsList[i]);
                }

                //Compute output layer
                denseFeatureGroupsOutputLayer[targetDenseFeatureIndex] = HiddenLayerList[numLayers - 1].Cells;
                OutputLayer.ForwardPass(sparseFeatureGorups, denseFeatureGroupsOutputLayer);

                if (m != null)
                {
                    OutputLayer.Cells.CopyTo(m[curState], 0);
                }

                predicted[curState] = OutputLayer.GetBestOutputIndex();

                if (runningMode == RunningMode.Training)
                {
                    // error propogation
                    OutputLayer.ComputeOutputLoss(CRFSeqOutput, state, curState);

                    //propogate errors to each layer from output layer to input layer
                    OutputLayer.ComputeLayerErr(HiddenLayerList[numLayers - 1]);

                    for (var i = numLayers - 1; i > 0; i--)
                    {
                        HiddenLayerList[i].ComputeLayerErr(HiddenLayerList[i - 1]);
                    }

                    //Update net weights
                    OutputLayer.BackwardPass();
                    for (var i = 0; i < numLayers; i++)
                    {
                        HiddenLayerList[i].BackwardPass();
                    }

                }
            }

            return predicted;
        }
    }
}
