text:
// ==========================================================================
//                              generalProcessing.h
// ==========================================================================
// Copyright (c) 2006-2012, Knut Reinert, FU Berlin
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//     * Redistributions of source code must retain the above copyright
//       notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//     * Neither the name of Knut Reinert or the FU Berlin nor the names of
//       its contributors may be used to endorse or promote products derived
//       from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL KNUT REINERT OR THE FU BERLIN BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
// DAMAGE.
//
// ==========================================================================
// Author: Sebastian Roskosch <serosko@zedat.fu-berlin.de>
// ==========================================================================
// This file provides functions used by different parts of seqan-flexbar
// which is based in the implementation of the original flexbar program
// in [1].
// [1] Dodt, M.; Roehr, J.T.; Ahmed, R.; Dieterich, C.  FLEXBAR—Flexible
// Barcode and Adapter Processing for Next-Generation Sequencing Platforms.
// Biology 2012, 1, 895-905.
// ==========================================================================

#ifndef SANDBOX_GROUP3_APPS_SEQDPT_GENERALPROCESSING_H_
#define SANDBOX_GROUP3_APPS_SEQDPT_GENERALPROCESSING_H_

#ifdef _OPENMP
#include <omp.h>
#endif
#include <seqan/basic.h>
#include <seqan/sequence.h>
#include <seqan/parallel.h>

#include <seqan/stream.h>

using namespace seqan;

// ============================================================================
// Tags, Classes, Enums
// ============================================================================

/**
.Class.GeneralStats:
..summary:Struct holding information on general statistics.
..include:seqan/generalProcessing.h
.Memvar.GeneralStats#removedSeqs
..class:Class.GeneralStats
..summary:Number of deleted sequences due to N's
..type:nolink:unsigned
.Memvar.GeneralStats#uncalledBases
..class:Class.GeneralStats
..summary:Number of uncalled bases (possibly masked) in surviving sequences.
..type:nolink:unsigned long
.Memvar.GeneralStats#removedSeqsShort
..class:Class.GeneralStats
..summary:Number of deleted sequences due to shortness.
..type:nolink:unsigned

.Memfunc.GeneralStats#GeneralStats:
..class:Class.GeneralStats
..summary:Default Constructor
..signature:GeneralStats(void)
*/
struct GeneralStats
{
	unsigned removedSeqs;       //Number of deleted sequences due to N's
    unsigned long uncalledBases;//Number of uncalled bases (evtl. Masked) in surviving sequences
    unsigned removedSeqsShort;  //Number of deleted sequences due to shortness.

    GeneralStats() : removedSeqs(0), uncalledBases(0), removedSeqsShort(0) {};
};

// ============================================================================
// Functions
// ============================================================================

/**
.Function.findN
..summary:Function for finding N's in a single sequence, substituting them and/or deleting it, if
 the number of found N's is too high.
..signature:findN(seq, allowed, substitute)
..param.seq:The sequence.
...type:nolink:String
..param.allowed:Number of allowed N's per sequence.
...type:nolink:unsigned
..param.substitute:The (optional) substitute base for N's.
...type:nolink:Dna5
...type:nolink:Dna5Q
...type:nolink:char
..remarks:Alphabet of the given sequence must be Dna5 or Dna5Q, otherwise results will be wrong.
..returns:The Number of found N's or substitutions. If too many N's were found, -1 is returned.
...type:nolink:int
..see:Function.processN
*/
template<typename TAlpha, typename TSub>
int findN(String<TAlpha>& seq, unsigned allowed, TSub substitute)
{
    unsigned limit = length(seq);
    TAlpha wanted = 'N';
    unsigned c = 0;
    for (unsigned i = 0; i < limit; ++i)
    {
        if (seq[i] == wanted)
        {
            seq[i] = substitute;
            ++c;
            if (c > allowed)
            {
                return -1;       //sequence will be removed
            }
        }
    }
    return c;                   //sequence not deleted, number of substitutions returned
}

//Overload if no substitution shall be performed
template<typename TAlpha>
int findN(String<TAlpha>& seq, unsigned allowed)
{
    unsigned limit = length(seq);
    TAlpha wanted = 'N';
    unsigned c = 0;
    for (unsigned i = 0; i < limit; ++i)
    {
        if (seq[i] == wanted)
        {
            ++c;
            if (c > allowed)
            {
                return -1;       //sequence will be removed
            }
        }
    }
    return c;                   //sequence not deleted, number of N's returned
}

/**
.Function.processN
..summary:Function for finding N's in all given sequences, substituting them and/or deleting them, if
 the number of found N's is too high.
..signature:processN(seqs, ids, seqsRev, idsRev, multiplex, allowed, substitute, stats)
..signature:processN(seqs, ids, seqsRev, idsRev, multiplex, allowed, stats)
..signature:processN(seqs, ids, seqsRev, idsRev, allowed, substitute, stats)
..signature:processN(seqs, ids, seqsRev, idsRev, allowed, stats)
..signature:processN(seqs, ids, multiplex, allowed, substitute, stats)
..signature:processN(seqs, ids, multiplex, allowed, stats)
..signature:processN(seqs, ids, allowed, substitute, stats)
..signature:processN(seqs, ids, allowed, stats)
..param.seqs:The forward reads.
...type:Class.StringSet
..param.ids:The ids associated with the forward reads.
...type:Class.StringSet
..param.seqsRev:The backward reads.
...type:Class.StringSet
..param.idsRev:The ids associated with the backward reads.
...type:Class.StringSet
..param.multiplex:The multiplex barcodes of the reads.
...type:Class.StringSet
..param.allowed:Number of allowed N's per sequence.
...type:nolink:unsigned
..param.substitute:The (optional) substitute base for N's.
...type:nolink:Dna5
...type:nolink:Dna5Q
...type:nolink:char
..param.stats:The GeneralStats object holding general statistic data.
...type:Class.GeneralStats
..remarks:This function simply calls findN and performs the deletion of ids/sequences/barcodes if necessary
..remarks:If the number of N's in one part of the sequence (e.g. the backward reads) is too hight, also the forward read
 will be deleted.
..returns:void
..see:Function.findN
*/
//single-end data with substitutions
template<typename TSeqs, typename TIds, typename TSub>
void processN(TSeqs& seqs, TIds& ids, unsigned allowed, TSub substitute, GeneralStats& stats)
{
    int limit = length(seqs);
    StringSet<int> res;
    resize(res, limit);
    SEQAN_OMP_PRAGMA(parallel for default(shared) schedule(static))
    for (int i = 0; i < limit; ++i)
    {
        res[i] = findN(seqs[i], allowed, substitute);
    }
    //Simply erasing a sequence would lead to unnecessary reallocations, therefore we use another method
    unsigned ex = 0;
    for (int i = length(res) - 1; i >= 0 ; --i)
    {                                   //integer necessary because unsigned would cause error after last iteration
        if (res[i] == -1)                //Placing all sequences/ids which shall be erased at the end of their container
        {
            SEQAN_OMP_PRAGMA(parallel default(shared))
            {
                SEQAN_OMP_PRAGMA(sections nowait)
                {
                    SEQAN_OMP_PRAGMA(section)     //distributing the swapping actions
                    swap(seqs[i], seqs[limit - ex - 1]);
                    SEQAN_OMP_PRAGMA(section)
                    swap(ids[i], ids[limit - ex - 1]);
                    SEQAN_OMP_PRAGMA(section)
                    ++ex;
                }
            }
        }
        else
        {
            stats.uncalledBases += res[i];
        }
    }
    if (ex != 0)
    {
        SEQAN_OMP_PRAGMA(parallel default(shared))
        {
            SEQAN_OMP_PRAGMA(sections nowait)
            {   //Resizing the containers to erase all unwanted sequences/ids at once
                SEQAN_OMP_PRAGMA(section)
                resize(seqs, limit - ex);
                SEQAN_OMP_PRAGMA(section)
                resize(ids, limit - ex);
                SEQAN_OMP_PRAGMA(section)
                stats.removedSeqs += ex;
            }
        }
    }
}

//Overload for single-end data and no substitutions
template<typename TSeqs, typename TIds>
void processN(TSeqs& seqs, TIds& ids, unsigned allowed, GeneralStats& stats)
{
    int limit = length(seqs);
    StringSet<int> res;
    resize(res, limit);
    SEQAN_OMP_PRAGMA(parallel for default(shared)schedule(static))
    for (int i = 0; i < limit; ++i)
    {
        res[i] = findN(seqs[i], allowed);
    }
    unsigned ex = 0;
    for (int  i = length(res) -1; i >= 0; --i)
    {
        if (res[i] == -1)
        {
            swap(seqs[i], seqs[limit - ex - 1]);
            swap(ids[i], ids[limit - ex - 1]);
            ++ex;
        }
        else
        {
            stats.uncalledBases += res[i];
        }
    }
    if (ex != 0)
    {
        resize(seqs, limit - ex);
        resize(ids, limit - ex);
        stats.removedSeqs += ex;
    }
}

//paired-end data with substitutions
template<typename TSeqs, typename TIds, typename TSub>
void processN(TSeqs& seqs, TIds& ids, TSeqs& seqsRev, TIds& idsRev, unsigned allowed, TSub substitute, GeneralStats& stats)
{
    int limit = length(seqs);
    StringSet<int> res;
    resize(res, limit);
    unsigned uncalled = 0;
    SEQAN_OMP_PRAGMA(parallel for default(shared) schedule(static) reduction(+:uncalled))
    for (int i = 0; i < limit; ++i)
    {
        res[i] = findN(seqs[i], allowed, substitute);
        if (res[i] != -1)
        {
            int resTemp = res[i];
            res[i] = findN(seqsRev[i], allowed, substitute);
            if (res[i] != -1)
            {
                uncalled += (res[i] + resTemp);
            }
        }
    }
    stats.uncalledBases += uncalled;
    unsigned ex = 0;
    for (int i = length(res) - 1; i >= 0 ; --i)
    {
        if (res[i] == -1)
        {
            swap(seqs[i], seqs[limit - ex - 1]);
            swap(ids[i], ids[limit - ex - 1]);
            swap(seqsRev[i], seqsRev[limit - ex - 1]);
            swap(idsRev[i], idsRev[limit - ex - 1]);
            ++ex;
        }
    }
    if (ex != 0)
    {
        resize(seqs, limit - ex);
        resize(ids, limit - ex);
        resize(seqsRev, limit - ex);
        resize(idsRev, limit - ex);
        stats.removedSeqs += (2 * ex);
    }
}

//paired-end data without substitutions
template<typename TSeqs, typename TIds>
void processN(TSeqs& seqs, TIds& ids, TSeqs& seqsRev, TIds& idsRev, unsigned allowed, GeneralStats& stats)
{
    int limit = length(seqs);
    StringSet<int> res;
    resize(res, limit);
    unsigned uncalled = 0;
    SEQAN_OMP_PRAGMA(parallel for default(shared)schedule(static) reduction(+:uncalled))
    for (int i = 0; i < limit; ++i)
    {
        res[i] = findN(seqs[i], allowed);
        if (res[i] != -1)
        {
            int resTemp = res[i];
            res[i] = findN(seqsRev[i], allowed);
            if (res[i] != -1)
            {
                uncalled += (res[i] + resTemp);
            }
        }
    }
    stats.uncalledBases += uncalled;
    unsigned ex = 0;
    for (int i = length(res) - 1; i >= 0 ; --i)
    {
        if (res[i] == -1)
        {
            swap(seqs[i], seqs[limit - ex - 1]);
            swap(ids[i], ids[limit - ex - 1]);
            swap(seqsRev[i], seqsRev[limit - ex - 1]);
            swap(idsRev[i], idsRev[limit - ex - 1]);
            ++ex;
        }
    }
    if (ex != 0)
    {
        resize(seqs, limit - ex);
        resize(ids, limit - ex);
        resize(seqsRev, limit - ex);
        resize(idsRev, limit - ex);
        stats.removedSeqs += (2 * ex);
    }
}

//Overload single-end data with substitutions and multiplex barcodes
template<typename TSeqs, typename TIds, typename TMulti, typename TSub>
void processN(TSeqs& seqs, TIds& ids, TMulti& multiplex, unsigned allowed, TSub substitute, GeneralStats& stats)
{
    int limit = length(seqs);
    StringSet<int> res;
    resize(res, limit);
    unsigned uncalled = 0;
    SEQAN_OMP_PRAGMA(parallel for default(shared) schedule(static) reduction(+:uncalled))
    for (int i = 0; i < limit; ++i)
    {
        res[i] = findN(seqs[i], allowed, substitute);
        if (res[i] != -1)
        {
            int resTemp = res[i];
            res[i] = findN(multiplex[i], allowed, substitute);
            if (res[i] != -1)
            {
                uncalled += (res[i] + resTemp);
            }
        }
    }
    stats.uncalledBases += uncalled;
    unsigned ex = 0;
    for (int i = length(res) - 1; i >= 0 ; --i)
    {
        if (res[i] == -1)
        {
            swap(seqs[i], seqs[limit - ex - 1]);
            swap(ids[i], ids[limit - ex - 1]);
            swap(multiplex[i], multiplex[limit - ex - 1]);
            ++ex;
        }
    }
    if (ex != 0)
    {
        resize(seqs, limit-ex);
        resize(ids, limit-ex);
        resize(multiplex, limit-ex);
        stats.removedSeqs += ex;
    }
}

//Overload for single-end data, no substitutions and multiplex barcodes
template<typename TSeqs, typename TIds, typename TMulti>
void processN(TSeqs& seqs, TIds& ids, TMulti& multiplex, unsigned allowed, GeneralStats& stats)
{
    int limit = length(seqs);
    StringSet<int> res;
    resize(res, limit);
    unsigned uncalled = 0;
    SEQAN_OMP_PRAGMA(parallel for default(shared) schedule(static) reduction(+:uncalled))
    for (int i = 0; i < limit; ++i)
    {
        res[i] = findN(seqs[i], allowed);
        if (res[i] != -1)
        {
            int resTemp = res[i];
            res[i] = findN(multiplex[i], allowed);
            if (res[i] != -1)
            {
                uncalled += (res[i] + resTemp);
            }
        }
    }
    stats.uncalledBases += uncalled;
    unsigned ex = 0;
    for (int i = length(res) - 1; i >= 0 ; --i)
    {
        if (res[i] == -1)
        {
            swap(seqs[i], seqs[limit - ex - 1]);
            swap(ids[i], ids[limit - ex - 1]);
            swap(multiplex[i], multiplex[limit - ex - 1]);
            ++ex;
        }
    }
    if (ex != 0)
    {
        resize(seqs, limit-ex);
        resize(ids, limit-ex);
        resize(multiplex, limit-ex);
        stats.removedSeqs += ex;
    }
}

//paired-end data with substitutions and multiplex barcodes
template<typename TSeqs, typename TIds, typename TMulti, typename TSub>
void processN(TSeqs& seqs, TIds& ids, TSeqs& seqsRev, TIds& idsRev, TMulti& multiplex, unsigned allowed,
    TSub substitute, GeneralStats& stats)
{
    int limit = length(seqs);
    StringSet<int> res;
    resize(res, limit);
    unsigned uncalled = 0;
    SEQAN_OMP_PRAGMA(parallel for default(shared)schedule(static) reduction(+:uncalled))
    for (int i = 0; i < limit; ++i)
    {
        res[i] = findN(seqs[i], allowed, substitute);
        if (res[i] != -1)
        {
            int resTemp = res[i];
            res[i] = findN(seqsRev[i], allowed, substitute);
            if (res[i] != -1)
            {
                resTemp += res[i];
                res[i] = findN(multiplex[i], allowed, substitute);
                if (res[i] != -1)
                {
                    uncalled += (res[i] + resTemp);
                }
            }
        }
    }
    stats.uncalledBases += uncalled;
    unsigned ex = 0;
    for (int i = length(res) - 1; i >= 0 ; --i)
    {
        if (res[i] == -1)
        {
            swap(seqs[i], seqs[limit - ex - 1]);
            swap(seqsRev[i], seqsRev[limit - ex - 1]);
            swap(ids[i], ids[limit - ex - 1]);
            swap(idsRev[i], idsRev[limit - ex - 1]);
            swap(multiplex[i], multiplex[limit - ex - 1]);
            ++ex;
        }
    }
    if (ex != 0)
    {
        resize(seqs, limit-ex);
        resize(seqsRev, limit-ex);
        resize(ids, limit-ex);
        resize(idsRev, limit-ex);
        resize(multiplex, limit-ex);
        stats.removedSeqs += (2 * ex);
    }
}

//paired-end data without substitutions and multiplex barcodes
template<typename TSeqs, typename TIds, typename TMulti >
void processN(TSeqs& seqs, TIds& ids, TSeqs& seqsRev, TIds& idsRev, TMulti& multiplex, unsigned allowed, GeneralStats& stats)
{
    int limit = length(seqs);
    StringSet<int> res;
    resize(res, limit);
    unsigned uncalled = 0;
    SEQAN_OMP_PRAGMA(parallel for default(shared) schedule(static) reduction(+:uncalled))
    for (int i = 0; i < limit; ++i)
    {
        res[i] = findN(seqs[i], allowed);
        if (res[i] != -1)
        {
            int resTemp = res[i];
            res[i] = findN(seqsRev[i], allowed);
            if (res[i] != -1)
            {
                resTemp += res[i];
                res[i] = findN(multiplex[i], allowed);
                if (res[i] != -1)
                {
                    uncalled += (res[i] + resTemp);
                }
            }
        }
    }
    stats.uncalledBases += uncalled;
    unsigned ex = 0;
    for (int i = length(res) - 1; i >= 0 ; --i)
    {
        if (res[i] == -1)
        {
            swap(seqs[i], seqs[limit - ex - 1]);
            swap(seqsRev[i], seqsRev[limit - ex - 1]);
            swap(ids[i], ids[limit - ex - 1]);
            swap(idsRev[i], idsRev[limit - ex - 1]);
            swap(multiplex[i], multiplex[limit - ex - 1]);
            ++ex;
        }
    }
    if (ex != 0)
    {
        resize(seqs, limit-ex);
        resize(seqsRev, limit-ex);
        resize(ids, limit-ex);
        resize(idsRev, limit-ex);
        resize(multiplex, limit-ex);
        stats.removedSeqs += (2 * ex);
    }
}

/**
.Function.preTrim
..summary:Function for trimming the sequences and deleting too short ones before further processing.
..signature:preTrim(seqs, ids, seqsRev, idsRev, multiplex, head, tail, min, stats)
..signature:preTrim(seqs, ids, seqsRev, idsRev, head, tail, min, stats)
..signature:preTrim(seqs, ids, multiplex, head, tail, min, stats)
..signature:preTrim(seqs, ids, head, tail, min, stats)
..param.seqs:The forward reads.
...type:Class.StringSet
..param.ids:The ids associated with the forward reads.
...type:Class.StringSet
..param.seqsRev:The backward reads.
...type:Class.StringSet
..param.idsRev:The ids associated with the backward reads.
...type:Class.StringSet
..param.multiplex:The multiplex barcodes of the reads.
...type:Class.StringSet
..param.head:Number of bases to be cut at the 5' end(s).
...type:nolink:unsigned
..param.tail:Number of bases to be cut at the 3' end(s).
...type:nolink:unsigned
..param.min:Minimum sequence length AFTER trimming.
...type:nolink:unsigned
..param.stats:The GeneralStats object holding general statistic data.
...type:Class.GeneralStats
..returns:void
*/
template<typename TSeqs, typename TIds> //Version for single end data
void preTrim(TSeqs& seqs, TIds& ids, unsigned head, unsigned tail, unsigned min, GeneralStats& stats)
{
    int i = 0;
    int limit = length(seqs);
    StringSet<bool> rem;

    resize(rem, limit);
    if (head > 0 && tail > 0)
    {
        SEQAN_OMP_PRAGMA(parallel for default(shared) private(i) schedule(static))
        for (i = 0; i < limit; ++i)
        {
            if (length(seqs[i]) > (head + tail))
            {
                erase(seqs[i], 0, head);
                erase(seqs[i], length(seqs[i]) - tail, length(seqs[i]));
                if (length(seqs[i]) >= min)
                {
                    rem[i] = false;
                }
                else
                {
                    rem[i] = true;
                }
            }
            else
            {
                rem[i] = true;
            }
        }
    }
    else if (head > 0)
    {
        SEQAN_OMP_PRAGMA(parallel for default(shared) private(i) schedule(static))
        for (i = 0; i < limit; ++i)
        {
           if (length(seqs[i]) > head)
            {
                erase(seqs[i], 0, head);
                if (length(seqs[i]) >= min)
                {
                    rem[i] = false;
                }
                else
                {
                    rem[i] = true;
                }
            }
            else
            {
                rem[i] = true;
            }
        }
    }
    else if (tail > 0)
    {
        SEQAN_OMP_PRAGMA(parallel for default(shared) private(i) schedule(static))
        for (i = 0; i < limit; ++i)
        {
            if (length(seqs[i]) > tail)
            {
                erase(seqs[i], length(seqs[i]) - tail, length(seqs[i]));
                if (length(seqs[i]) >= min)
                {
                    rem[i] = false;
                }
                else
                {
                    rem[i] = true;
                }
            }
            else
            {
                rem[i] = true;
            }
        }
    }
    else
    {
        SEQAN_OMP_PRAGMA(parallel for default(shared) private(i) schedule(static))
        for (i = 0; i < limit; ++i)
        {
            if (length(seqs[i]) >= min)
            {
                rem[i] = false;
            }
            else
            {
                rem[i] = true;
            }
        }
    }
    unsigned  ex = 0;
    for (int j = length(rem) - 1; j >= 0; --j)
    {
        if (rem[j])
        {
            swap(seqs[j], seqs[limit - ex - 1]);
            swap(ids[j], ids[limit - ex - 1]);
            ++ex;
        }
    }
    if (ex != 0)
    {
        resize(seqs, limit - ex);
        resize(ids, limit - ex);
        stats.removedSeqsShort += ex;
    }
}

//Overload for single end data with multiplex barcodes
template<typename TSeqs, typename TIds, typename TMulti>
void preTrim(TSeqs& seqs, TIds& ids, TMulti& multiplex, unsigned head, unsigned tail, unsigned min, GeneralStats& stats)
{
    int i = 0;
    int limit = length(seqs);
    StringSet<bool> rem;

    resize(rem, limit);
    if (head > 0 && tail > 0)
    {
        SEQAN_OMP_PRAGMA(parallel for default(shared) private(i) schedule(static))
        for (i = 0; i < limit; ++i)
        {
            if (length(seqs[i]) > (head + tail))
            {
                erase(seqs[i], 0, head);
                erase(seqs[i], length(seqs[i]) - tail, length(seqs[i]));
                if (length(seqs[i]) >= min)
                {
                    rem[i] = false;
                }
                else
                {
                    rem[i] = true;
                }
            }
            else
            {
                rem[i] = true;
            }
        }
    }
    else if (head > 0)
    {
       SEQAN_OMP_PRAGMA(parallel for default(shared) private(i) schedule(static))
        for (i = 0; i < limit; ++i)
        {
           if (length(seqs[i]) > head)
            {
                erase(seqs[i], 0, head);
                if (length(seqs[i]) >= min)
                {
                    rem[i] = false;
                }
                else
                {
                    rem[i] = true;
                }
            }
            else
            {
                rem[i] = true;
            }
        }
    }
    else if (tail > 0)
    {
        SEQAN_OMP_PRAGMA(parallel for default(shared) private(i) schedule(static))
        for (i = 0; i < limit; ++i)
        {
            if (length(seqs[i]) > tail)
            {
                erase(seqs[i], length(seqs[i]) - tail, length(seqs[i]));
                if (length(seqs[i]) >= min)
                {
                    rem[i] = false;
                }
                else
                {
                    rem[i] = true;
                }
            }
            else
            {
                rem[i] = true;
            }
        }
    }
    else
    {
        SEQAN_OMP_PRAGMA(parallel for default(shared) private(i) schedule(static))
        for (i = 0; i < limit; ++i)
        {
            if (length(seqs[i]) >= min)
            {
                rem[i] = false;
            }
            else
            {
                rem[i] = true;
            }
        }
    }
    unsigned ex = 0;
    for (int j = length(rem) - 1; j >= 0; --j)
    {
        if (rem[j])
        {
            swap(seqs[j], seqs[limit - ex -1]);
            swap(ids[j], ids[limit - ex -1]);
            swap(multiplex[j], multiplex[limit - ex -1]);
            ++ex;
        }
    }
    if (ex != 0)
    {
        resize(seqs, limit - ex);
        resize(ids, limit - ex);
        resize(multiplex, limit - ex);
        stats.removedSeqsShort += ex;
    }
}

//Overload for paired end data
template<typename TSeqs, typename TIds>
void preTrim(TSeqs& seqs, TIds& ids, TSeqs& seqsRev, TIds& idsRev, unsigned head, unsigned tail, unsigned min,
    GeneralStats& stats)
{
    int i = 0;
    int limit = length(seqs);
    StringSet<bool> rem;
    resize(rem, limit);
    if (head > 0 && tail > 0)
    {
        SEQAN_OMP_PRAGMA(parallel for default(shared) private(i) schedule(static))
        for (i = 0; i < limit; ++i)
        {
            if ((length(seqs[i]) > (head + tail)) && (length(seqsRev[i]) > (head + tail)))
            {
                erase(seqs[i], 0, head);
                erase(seqs[i], length(seqs[i]) - tail, length(seqs[i]));
                erase(seqsRev[i], 0, head);
                erase(seqsRev[i], length(seqsRev[i]) - tail, length(seqsRev[i]));
                if ((length(seqs[i]) >= min) && (length(seqsRev[i]) >= min))
                {
                    rem[i] = false;
                }
                else
                {
                    rem[i] = true;
                }
            }
            else
            {
                rem[i] = true;
            }
        }
    }
    else if (head > 0)
    {
        SEQAN_OMP_PRAGMA(parallel for default(shared) private(i) schedule(static))
        for (i = 0; i < limit; ++i)
        {
            if ((length(seqs[i]) > head) && (length(seqsRev[i]) > head))
            {
                erase(seqs[i], 0, head);
                erase(seqsRev[i], 0, head);
                if ((length(seqs[i]) >= min) && (length(seqsRev[i]) >= min))
                {
                    rem[i] = false;
                }
                else
                {
                    rem[i] = true;
                }
            }
            else
            {
                rem[i] = true;
            }
        }
    }
    else if (tail > 0)
    {
        SEQAN_OMP_PRAGMA(parallel for default(shared) private(i) schedule(static))
        for (i = 0; i < limit; ++i)
        {
            if ((length(seqs[i]) > tail) && (length(seqsRev[i]) > tail))
            {
                erase(seqs[i], length(seqs[i]) - tail, length(seqs[i]));
                erase(seqsRev[i], length(seqsRev[i]) - tail, length(seqsRev[i]));
                if ((length(seqs[i]) >= min) && (length(seqsRev[i]) >= min))
                {
                    rem[i] = false;
                }
                else
                {
                    rem[i] = true;
                }
            }
            else
            {
                rem[i] = true;
            }
        }
    }
    else
    {
        SEQAN_OMP_PRAGMA(parallel for default(shared) private(i) schedule(static))
        for (i = 0; i < limit; ++i)
        {
            if ((length(seqs[i]) >= min) && (length(seqsRev[i]) >= min))
            {
                rem[i] = false;
            }
            else
            {
                rem[i] = true;
            }
        }
    }
    unsigned ex = 0;
    for (int j = length(rem) - 1; j >= 0; --j)
    {
        if (rem[j])
        {
            swap(seqs[j], seqs[limit - ex - 1]);
            swap(seqsRev[j], seqsRev[limit - ex - 1]);
            swap(ids[j], ids[limit - ex - 1]);
            swap(idsRev[j], idsRev[limit - ex - 1]);
            ++ex;
        }
    }
    if (ex != 0)
    {
        resize(seqs, limit - ex);
        resize(seqsRev, limit - ex);
        resize(ids, limit - ex);
        resize(idsRev, limit - ex);
        stats.removedSeqsShort += (2 * ex);
    }

}

//Overload for paired end data with multiplex barcodes
template<typename TSeqs, typename TIds,  typename TMulti>
void preTrim(TSeqs& seqs, TIds& ids, TSeqs& seqsRev, TIds& idsRev, TMulti& multiplex, unsigned head, unsigned tail, unsigned min,
    GeneralStats& stats)
{
    int i = 0;
    int limit = length(seqs);
    StringSet<bool> rem;
    resize(rem, limit);
    if (head > 0 && tail > 0)
    {
        SEQAN_OMP_PRAGMA(parallel for default(shared) private(i) schedule(static))
        for (i = 0; i < limit; ++i)
        {
            if ((length(seqs[i]) > (head + tail)) && (length(seqsRev[i]) > (head + tail)))
            {
                erase(seqs[i], 0, head);
                erase(seqs[i], length(seqs[i]) - tail, length(seqs[i]));
                erase(seqsRev[i], 0, head);
                erase(seqsRev[i], length(seqsRev[i]) - tail, length(seqsRev[i]));
                if ((length(seqs[i]) >= min) && (length(seqsRev[i]) >= min))
                {
                    rem[i] = false;
                }
                else
                {
                    rem[i] = true;
                }
            }
            else
            {
                rem[i] = true;
            }
        }
    }
    else if (head > 0)
    {
        SEQAN_OMP_PRAGMA(parallel for default(shared) private(i) schedule(static))
        for (i = 0; i < limit; ++i)
        {
            if ((length(seqs[i]) > head) && (length(seqsRev[i]) > head))
            {
                erase(seqs[i], 0, head);
                erase(seqsRev[i], 0, head);
                if ((length(seqs[i]) >= min) && (length(seqsRev[i]) >= min))
                {
                    rem[i] = false;
                }
                else
                {
                    rem[i] = true;
                }
            }
            else
            {
                rem[i] = true;
            }
        }
    }
    else if (tail > 0)
    {
        SEQAN_OMP_PRAGMA(parallel for default(shared) private(i) schedule(static))
        for (i = 0; i < limit; ++i)
        {
            if ((length(seqs[i]) > tail) && (length(seqsRev[i]) > tail))
            {
                erase(seqs[i], length(seqs[i]) - tail, length(seqs[i]));
                erase(seqsRev[i], length(seqsRev[i]) - tail, length(seqsRev[i]));
                if ((length(seqs[i]) >= min) && (length(seqsRev[i]) >= min))
                {
                    rem[i] = false;
                }
                else
                {
                    rem[i] = true;
                }
            }
            else
            {
                rem[i] = true;
            }
        }
    }
    else
    {
        SEQAN_OMP_PRAGMA(parallel for default(shared) private(i) schedule(static))
        for (i = 0; i < limit; ++i)
        {
            if ((length(seqs[i]) >= min) && (length(seqsRev[i]) >= min))
            {
                rem[i] = false;
            }
            else
            {
                rem[i] = true;
            }
        }
    }
    unsigned ex = 0;
    for (int j = length(rem) - 1; j >= 0; --j)
    {
        if (rem[j])
        {
            swap(seqs[j], seqs[j - ex - 1]);
            swap(seqsRev[j], seqsRev[j - ex - 1]);
            swap(ids[j], ids[j - ex - 1]);
            swap(idsRev[j], idsRev[j - ex - 1]);
            swap(multiplex[j], multiplex[j - ex - 1]);
            ++ex;
        }
    }
    if (ex != 0)
    {
        resize(seqs, limit - ex);
        resize(seqsRev, limit - ex);
        resize(ids, limit - ex);
        resize(idsRev, limit - ex);
        resize(multiplex, limit - ex);
        stats.removedSeqsShort += (2 * ex);
    }
}

//Trims sequences to specific length and deletes to short ones together with their IDs
template<typename TSeqs,typename TIds>
void trimTo(TSeqs& seqs, TIds& ids, const unsigned len, GeneralStats& stats)
{
    StringSet<bool> rem;
    int limit = length(seqs);
    resize(rem, limit);
    SEQAN_OMP_PRAGMA(parallel for default(shared) schedule(static))
    for (int i = 0; i < limit; ++i)
    {
        if (length(seqs[i]) < len)
        {
            rem[i] = true;
        }
        else
        {
            rem[i] = false;
            if (length(seqs[i]) > len)
            {
                erase(seqs[i], len, length(seqs[i]));
            }
        }
    }
    unsigned ex = 0;
    for (int j = limit - 1; j >= 0; --j)
    {
        if (rem[j])
        {
            swap(seqs[j], seqs[limit  - ex - 1]);
            swap(ids[j], ids[limit  - ex - 1]);
            ++ex;
        }
    }
    if (ex != 0)
    {
        resize(seqs, limit - ex);
        resize(ids, limit - ex);
        stats.removedSeqsShort += ex;
    }
}

//Overload for paired end-data
template<typename TSeqs,typename TIds>
void trimTo(TSeqs& seqs, TIds& ids, TSeqs& seqsRev, TIds& idsRev, const unsigned len, GeneralStats& stats)
{
    int i = 0;
    StringSet<bool> rem;
    int limit = length(seqs);
    resize(rem, limit);
    SEQAN_OMP_PRAGMA(parallel for default(shared) private(i) schedule(static))
    for (i = 0; i < limit; ++i)
    {
        if ((length(seqs[i]) < len) || (length(seqsRev[i]) < len))
        {
            rem[i] = true;
        }
        else
        {
            rem[i] = false;
            if (length(seqs[i]) > len)
            {
                erase(seqs[i], len, length(seqs[i]));
            }
            if (length(seqsRev[i]) > len)
            {
                erase(seqsRev[i], len, length(seqsRev[i]));
            }
        }
    }
    unsigned ex = 0;
    for (int j = length(rem) - 1; j >= 0; --j)
    {
        if (rem[j])
        {
            swap(seqs[j], seqs[limit - ex - 1]);
            swap(seqsRev[j], seqsRev[limit - ex - 1]);
            swap(ids[j], ids[limit - ex - 1]);
            swap(idsRev[j], idsRev[limit - ex - 1]);
            ++ex;
        }
    }
    if (ex != 0)
    {
        resize(seqs, limit - ex);
        resize(seqsRev, limit - ex);
        resize(ids, limit - ex);
        resize(idsRev, limit - ex);
        stats.removedSeqsShort += (2 * ex);
    }
}

#endif  // #ifndef SANDBOX_GROUP3_APPS_SEQDPT_GENERALPROCESSING_H_


text:
from django.template import RequestContext
from django.shortcuts import render_to_response, get_object_or_404
from django.db.models import Q
from django.template.loader import select_template
from viewpoint.settings import DEFAULT_BLOG, MONTH_FORMAT
from django.views.generic.date_based import archive_day, archive_month, \
                                            archive_year, archive_today, \
                                            archive_week, object_detail

from models import Blog, Entry

def get_template(blog_slug, template_name):
    """
    Given a blog slug and template name (without the .html), this will return
    the appropriate template, looking in various directories.
    """
    return select_template((
            'viewpoint/%s/%s.html' % (blog_slug, template_name),
            'viewpoint/%s.html' % template_name
        )).name

def generic_blog_entry_view(request, *args,  **kwargs):
    """
    A generic override for the default Django generic views
    """
    blog_slug = kwargs.pop('blog_slug', DEFAULT_BLOG)
    if request.user.is_staff:
        queryset = Entry.objects.filter(blog__slug=blog_slug)
    else:
        queryset = Entry.objects.published(blog__slug=blog_slug)
    params = {
        'queryset': queryset,
        'date_field': 'pub_date',
        'allow_empty': True,
    }
    params.update(kwargs)

    if 'slug' in kwargs.keys():
        if 'template_name' not in params.keys():
            params['template_name'] = get_template(blog_slug, 'entry_detail')
            params['month_format'] = MONTH_FORMAT
        params.pop('allow_empty')
        return object_detail(request, **params)
    elif 'day' in kwargs.keys():
        if 'template_name' not in params.keys():
            params['template_name'] = get_template(blog_slug, 'entry_archive_day')
            params['month_format'] = MONTH_FORMAT
        return archive_day(request, **params)
    elif 'month' in kwargs.keys():
        if 'template_name' not in params.keys():
            params['template_name'] = get_template(blog_slug, 'entry_archive_month')
            params['month_format'] = MONTH_FORMAT
        return archive_month(request, **params)
    elif 'week' in kwargs.keys():
        if 'template_name' not in params.keys():
            params['template_name'] = get_template(blog_slug, 'entry_archive_week')
        return archive_week(request, **params)
    elif 'year' in kwargs.keys():
        if 'template_name' not in params.keys():
            params['template_name'] = get_template(blog_slug, 'entry_archive_year')
        return archive_year(request, **params)
    else:
        if 'template_name' not in params.keys():
            params['template_name'] = get_template(blog_slug, 'entry_archive_today')
        return archive_today(request, **params)

def blog_detail(request, blog_slug=DEFAULT_BLOG):
    """
    Return the blog_detail page for the specified blog.
    """
    blog = get_object_or_404(Blog, slug=blog_slug, public=True)
    return render_to_response(get_template(blog_slug, 'blog_detail'), {
            'object':blog,
        }, context_instance=RequestContext(request))

def feeds_index(request, template_name='viewpoint/feeds_index.html'):
    blogs = Blog.objects.filter(public=True).order_by('title')
    return render_to_response(template_name,
                              {'blogs': blogs},
                              context_instance=RequestContext(request))

redpajama_set_name: RedPajamaGithub


text:
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">

        <title>SFML - Simple and Fast Multimedia Library</title>
        <meta http-equiv="Content-Type" content="text/html;"/>
        <meta charset="utf-8"/>

        <link rel="stylesheet" type="text/css" href="doxygen.css" title="default" media="screen,print" />

                SFML 2.3.0

Main Page

Related Pages

Modules

Classes

Files

Class List

Class Index

Class Hierarchy

Class Members

sf

Thread

Public Member Functions |
List of all members

sf::Thread Class Reference
System module

Utility class to manipulate threads.
 More...

#include <Thread.hpp>

Inheritance diagram for sf::Thread:

  <map id="sf::Thread_map" name="sf::Thread_map">
<area href="classsf_1_1NonCopyable.html" title="Utility class that makes any derived class non-copyable. " alt="sf::NonCopyable" shape="rect" coords="0,0,105,24"/>
</map>

Public Member Functions

template<typename F >

 	Thread (F function)

 	Construct the thread from a functor with no argument.  More...

template<typename F , typename A >

 	Thread (F function, A argument)

 	Construct the thread from a functor with an argument.  More...

template<typename C >

 	Thread (void(C::*function)(), C *object)

 	Construct the thread from a member function and an object.  More...

 	~Thread ()

 	Destructor.  More...

void 	launch ()

 	Run the thread.  More...

void 	wait ()

 	Wait until the thread finishes.  More...

void 	terminate ()

 	Terminate the thread.  More...

Detailed Description

Utility class to manipulate threads.

Threads provide a way to run multiple parts of the code in parallel.

When you launch a new thread, the execution is split and both the new thread and the caller run in parallel.

To use a sf::Thread, you construct it directly with the function to execute as the entry point of the thread. sf::Thread has multiple template constructors, which means that you can use several types of entry points:

non-member functions with no argument

non-member functions with one argument of any type

functors with no argument (this one is particularly useful for compatibility with boost/std::bind)

functors with one argument of any type

member functions from any class with no argument

The function argument, if any, is copied in the sf::Thread instance, as well as the functor (if the corresponding constructor is used). Class instances, however, are passed by pointer so you must make sure that the object won't be destroyed while the thread is still using it.

The thread ends when its function is terminated. If the owner sf::Thread instance is destroyed before the thread is finished, the destructor will wait (see wait())

Usage examples:

// example 1: non member function with one argument

void threadFunc(int argument)

{

    ...

}

sf::Thread thread(&threadFunc, 5);

thread.launch(); // start the thread (internally calls threadFunc(5))

// example 2: member function

class Task

{

public:

    void run()

    {

        ...

    }

};

Task task;

sf::Thread thread(&Task::run, &task);

thread.launch(); // start the thread (internally calls task.run())

// example 3: functor

struct Task

{

    void operator()()

    {

        ...

    }

};

sf::Thread thread(Task());

thread.launch(); // start the thread (internally calls operator() on the Task instance)

Creating parallel threads of execution can be dangerous: all threads inside the same process share the same memory space, which means that you may end up accessing the same variable from multiple threads at the same time. To prevent this kind of situations, you can use mutexes (see sf::Mutex).

See also

sf::Mutex

Definition at line 48 of file Thread.hpp.

Constructor & Destructor Documentation

template<typename F >

          sf::Thread::Thread
          (
          F
          function	)

Construct the thread from a functor with no argument.

This constructor works for function objects, as well as free functions.

Use this constructor for this kind of function:

void function();

// --- or ----

struct Functor

{

    void operator()();

};

 Note: this does *not* run the thread, use launch().

Parameters

function	Functor or free function to use as the entry point of the thread

template<typename F , typename A >

          sf::Thread::Thread
          (
          F
          function,

          A
          argument

          )

Construct the thread from a functor with an argument.

This constructor works for function objects, as well as free functions. It is a template, which means that the argument can have any type (int, std::string, void*, Toto, ...).

Use this constructor for this kind of function:

void function(int arg);

// --- or ----

struct Functor

{

    void operator()(std::string arg);

};

 Note: this does *not* run the thread, use launch().

Parameters

function	Functor or free function to use as the entry point of the thread

argument	argument to forward to the function

template<typename C >

          sf::Thread::Thread
          (
          void(C::*)()
          function,

          C *
          object

          )

Construct the thread from a member function and an object.

This constructor is a template, which means that you can use it with any class. Use this constructor for this kind of function:

class MyClass

{

public:

    void function();

};

 Note: this does *not* run the thread, use launch().

Parameters

function	Entry point of the thread

object	Pointer to the object to use

          sf::Thread::~Thread
          (
          	)

Destructor.

This destructor calls wait(), so that the internal thread cannot survive after its sf::Thread instance is destroyed.

Member Function Documentation

          void sf::Thread::launch
          (
          	)

Run the thread.

This function starts the entry point passed to the thread's constructor, and returns immediately. After this function returns, the thread's function is running in parallel to the calling code.

          void sf::Thread::terminate
          (
          	)

Terminate the thread.

This function immediately stops the thread, without waiting for its function to finish. Terminating a thread with this function is not safe, and can lead to local variables not being destroyed on some operating systems. You should rather try to make the thread function terminate by itself.

          void sf::Thread::wait
          (
          	)

Wait until the thread finishes.

This function will block the execution until the thread's function ends. Warning: if the thread function never ends, the calling thread will block forever. If this function is called from its owner thread, it returns without doing anything.

The documentation for this class was generated from the following file:

Thread.hpp

                SFML is licensed under the terms and conditions of the zlib/png license.

                Copyright © Laurent Gomila  ::
                Documentation generated by doxygen  ::


text:
package org.apache.flink.streaming.api.operators.collect.utils;

import org.apache.flink.api.common.accumulators.SerializedListAccumulator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer;
import org.apache.flink.runtime.operators.coordination.CoordinationRequest;
import org.apache.flink.runtime.operators.coordination.CoordinationRequestHandler;
import org.apache.flink.runtime.operators.coordination.CoordinationResponse;
import org.apache.flink.streaming.api.operators.collect.CollectCoordinationRequest;
import org.apache.flink.streaming.api.operators.collect.CollectCoordinationResponse;
import org.apache.flink.streaming.api.operators.collect.CollectSinkFunction;
import org.apache.flink.util.OptionalFailure;

import org.junit.Assert;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;

/** A {@link CoordinationRequestHandler} to test fetching SELECT query results. */
public abstract class AbstractTestCoordinationRequestHandler<T>
        implements CoordinationRequestHandler {

    protected static final int BATCH_SIZE = 3;

    protected final TypeSerializer<T> serializer;
    protected final String accumulatorName;

    protected LinkedList<T> buffered;
    protected String version;
    protected long offset;
    protected long checkpointedOffset;

    private final Map<String, OptionalFailure<Object>> accumulatorResults;

    protected final Random random;
    protected boolean closed;

    public AbstractTestCoordinationRequestHandler(
            TypeSerializer<T> serializer, String accumulatorName) {
        this.serializer = serializer;
        this.accumulatorName = accumulatorName;

        this.buffered = new LinkedList<>();
        this.version = UUID.randomUUID().toString();
        this.offset = 0;
        this.checkpointedOffset = 0;

        this.accumulatorResults = new HashMap<>();

        this.random = new Random();
        this.closed = false;
    }

    @Override
    public CompletableFuture<CoordinationResponse> handleCoordinationRequest(
            CoordinationRequest request) {
        if (closed) {
            throw new RuntimeException("Handler closed");
        }

        Assert.assertTrue(request instanceof CollectCoordinationRequest);
        CollectCoordinationRequest collectRequest = (CollectCoordinationRequest) request;

        updateBufferedResults();
        Assert.assertTrue(offset <= collectRequest.getOffset());

        List<T> subList = Collections.emptyList();
        if (collectRequest.getVersion().equals(version)) {
            while (buffered.size() > 0 && collectRequest.getOffset() > offset) {
                buffered.removeFirst();
                offset++;
            }
            subList = new ArrayList<>();
            Iterator<T> iterator = buffered.iterator();
            for (int i = 0; i < BATCH_SIZE && iterator.hasNext(); i++) {
                subList.add(iterator.next());
            }
        }
        List<byte[]> nextBatch = CollectTestUtils.toBytesList(subList, serializer);

        CoordinationResponse response;
        if (random.nextBoolean()) {
            // with 50% chance we return valid result
            response = new CollectCoordinationResponse(version, checkpointedOffset, nextBatch);
        } else {
            // with 50% chance we return invalid result
            response =
                    new CollectCoordinationResponse(
                            collectRequest.getVersion(), -1, Collections.emptyList());
        }
        return CompletableFuture.completedFuture(response);
    }

    protected abstract void updateBufferedResults();

    public boolean isClosed() {
        return closed;
    }

    public Map<String, OptionalFailure<Object>> getAccumulatorResults() {
        return accumulatorResults;
    }

    protected void buildAccumulatorResults() {
        List<byte[]> finalResults = CollectTestUtils.toBytesList(buffered, serializer);
        SerializedListAccumulator<byte[]> listAccumulator = new SerializedListAccumulator<>();
        try {
            byte[] serializedResult =
                    CollectSinkFunction.serializeAccumulatorResult(
                            offset, version, checkpointedOffset, finalResults);
            listAccumulator.add(serializedResult, BytePrimitiveArraySerializer.INSTANCE);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }

        accumulatorResults.put(
                accumulatorName, OptionalFailure.of(listAccumulator.getLocalValue()));
    }
}


text:
namespace base {
template <typename Type>
struct StaticMemorySingletonTraits;
}  // namespace base

namespace logging {

// Event ID for the log messages we generate.
EXTERN_C BASE_EXPORT const GUID kLogEventId;

// Feature enable mask for LogEventProvider.
enum LogEnableMask {
  // If this bit is set in our provider enable mask, we will include
  // a stack trace with every log message.
  ENABLE_STACK_TRACE_CAPTURE = 0x0001,
  // If this bit is set in our provider enable mask, the provider will log
  // a LOG message with only the textual content of the message, and no
  // stack trace.
  ENABLE_LOG_MESSAGE_ONLY = 0x0002,
};

// The message types our log event provider generates.
// ETW likes user message types to start at 10.
enum LogMessageTypes {
  // A textual only log message, contains a zero-terminated string.
  LOG_MESSAGE = 10,
  // A message with a stack trace, followed by the zero-terminated
  // message text.
  LOG_MESSAGE_WITH_STACKTRACE = 11,
  // A message with:
  //  a stack trace,
  //  the line number as a four byte integer,
  //  the file as a zero terminated UTF8 string,
  //  the zero-terminated UTF8 message text.
  LOG_MESSAGE_FULL = 12,
};

// Trace provider class to drive log control and transport
// with Event Tracing for Windows.
class BASE_EXPORT LogEventProvider : public base::win::EtwTraceProvider {
 public:
  LogEventProvider(const LogEventProvider&) = delete;
  LogEventProvider& operator=(const LogEventProvider&) = delete;
  static LogEventProvider* GetInstance();

  static bool LogMessage(logging::LogSeverity severity, const char* file,
      int line, size_t message_start, const std::string& str);

  static void Initialize(const GUID& provider_name);
  static void Uninitialize();

 protected:
  // Overridden to manipulate the log level on ETW control callbacks.
  void OnEventsEnabled() override;
  void OnEventsDisabled() override;

 private:
  LogEventProvider();

  // The log severity prior to OnEventsEnabled,
  // restored in OnEventsDisabled.
  logging::LogSeverity old_log_level_;

  friend struct base::StaticMemorySingletonTraits<LogEventProvider>;
};

}  // namespace logging

#endif  // BASE_LOGGING_WIN_H_


text:
module OpenAM
  module Auth
    class API
      include Singleton

      def initialize
        @config       = OpenAM::Auth.config
        @cookie_name  = nil

        raise OpenAM::Auth::Error.new('configured OpenAM URL is invalid') if @config.url.nil?
        uri = URI(@config.url)

        raise OpenAM::Auth::Error.new('configured OpenAM URL is invalid') if uri.scheme.nil?

        if uri.scheme != 'https' && @config.scheme_http.nil?
          raise OpenAM::Auth::Error.new('OpenAM URL scheme is not HTTPS')
        end
        @config.url = uri.to_s
      end

      def cookie_name
        @cookie_name ||= get_cookie_name
      end

      # method: logout
      #
      # JSON Result Examples:
      # { :result => "Successfully logged out" }
      # { "code": 403, "reason": "Forbidden", "message": "Access denied" }
      def logout(token=nil)
        raise OpenAM::Auth::Error.new('token value is invalid') if token.nil?
        c_name = cookie_name

        res = OpenAM::Auth::HTTP.post(
          build_api(@config.logout_api),
          headers: {
            c_name => token,
            'Content-Type' => 'application/json'
          }
        )

        if res.body.nil? || res.contentType.nil?
          raise OpenAM::Auth::Error.new('logout request results invalid')
        else
          if res.contentType =~ /\Aapplication\/json/
            JsonResult.new res.body
          end
        end

      end

      # method: login
      #
      # JSON Result Examples:
      # { :errorMessage => "User Account Locked" }
      # { :tokenId => "TOKEN", :successUrl => "https://www.myunfpa.org/" }
      # { :errorMessage => "Authentication Error!!" }
      def login(username=nil,password=nil)
        raise OpenAM::Auth::Error.new('login username is invalid') if username.nil?
        raise OpenAM::Auth::Error.new('login password is invalid') if password.nil?

        res = OpenAM::Auth::HTTP.post(
          build_api(
            @config.login_api,
            realm: @config.realm
          ),
          headers: {
            'X-OpenAM-Username' => username,
            'X-OpenAM-Password' => password,
            'Content-Type' => 'application/json'
          }
        )

        if res.body.nil? || res.contentType.nil?
          raise OpenAM::Auth::Error.new('login request results invalid')
        else
          if res.contentType =~ /\Aapplication\/json/
            JsonResult.new res.body
          end
        end
      end

      # method: user_info
      #
      def user_info(token=nil)
        raise OpenAM::Auth::Error.new('token value is invalid') if token.nil?

        c_name = cookie_name

        res = OpenAM::Auth::HTTP.post(
          build_api(@config.user_api),
          headers: {
            c_name => token,
            'Content-Type' => 'application/json'
          }
        )

        if res.body.nil? || res.contentType.nil?
          raise OpenAM::Auth::Error.new('user info request results invalid')
        else
          if res.contentType =~ /\Aapplication\/json/
            JsonResult.new res.body
          end
        end
      end

      # method: verify_token
      #
      def verify_token(token=nil)
        raise OpenAM::Auth::Error.new('token value is invalid') if token.nil?

        res = OpenAM::Auth::HTTP.post(
          build_api(@config.token_api),
          body: {
            'tokenid' => token
          }
        )

        if res.body.nil? || res.contentType.nil?
          raise OpenAM::Auth::Error.new('verify token request results invalid')
        else
          if res.contentType =~ /\Atext\/plain/
            if /true\Z/.match(res.body)
              return true
            end
          end
        end

        return false
      end

      # method: login_url
      #
      def login_url(goto=nil)
        build_api(
          @config.login_uri,
          realm: @config.realm,
          goto: goto
        )
      end

      # method: logout_url
      #
      def logout_url(goto=nil)
        build_api(
          @config.logout_uri,
          realm: @config.realm,
          goto: goto
        )
      end

      private

      def get_cookie_name
        res = OpenAM::Auth::HTTP.get(build_api(@config.cookie_api))

        if res.body.nil? || res.contentType.nil?
          raise OpenAM::Auth::Error.new('returned cookie name is invalid')
        else
          if res.contentType =~ /\Atext\/plain/
            if res.body =~ /\Astring\=.*\Z/
              res.body.gsub(/\Astring\=/, "")
            else
              @config.cookie_name
            end
          else
            @config.cookie_name
          end
        end
      end

      def build_api(uri=nil,*opts)
        raise OpenAM::Auth::Error.new('configured OpenAM URL is invalid') if @config.url.nil?
        raise OpenAM::Auth::Error.new('requested URI is invalid') if uri.nil?

        api = @config.url.clone

        if uri.instance_of? Hash
          uri_c = uri.clone

          raise OpenAM::Auth::Error.new('requested URI is invalid') if uri_c[:uri].nil?
          api << uri_c.delete(:uri)

          if !uri_c.empty?
            if !opts.nil? && !opts.first.nil?
              opts.first.merge! uri_c
            else
              opts.push uri_c
            end
          end
        elsif uri.instance_of? String
          api << uri
        else
          raise OpenAM::Auth::Error.new('requested URI is invalid')
        end

        p = URI::Parser.new
        api = p.escape(api)

        if !opts.nil? && !opts.first.nil?
          if opts.first.instance_of? Hash
            params = []
            opts.first.each do |k,v|
              if !v.nil?
                p = URI::Parser.new(:RESERVED => '')
                v = p.escape(v)
                params.push "#{k}=#{v}"
              end
            end
            api << "?"
            api << params.join('&')
          end
        end

        api
      end

      class JsonResult

        def initialize(json=nil)
          @json_data = {}

          if json.nil?
            raise OpenAM::Auth::Error.new('JSON data is invalid')
          else
            @raw_json = json
            JSON.parse(json).each do |i|
              key = i.shift
              value = i.shift
              @json_data[key.to_sym] = value
            end
          end
        end

        def to_s
          @raw_json
        end

        def [](key)
          @json_data[key.to_sym]
        end

        def udefine(key,*opts)
          if !key.nil?
            key = key.to_s
            @json_data[key.to_sym]
          end
        end
        alias method_missing udefine

      end

    end
  end
end


text:
This document describes how Mozc obfuscates user data and how it works, in each platform.

## Background

Input methods stores some internal data in the local storage.  Most are stored in plain text, but Mozc applies AES256 CBC to obfuscate some data that are considered to be important for user privacy.

## Overview

As is described above, most of data are stored in plain text.  Only the following data are obfuscated with AES256 CBC:

  * User history: stores some of users input text, which is used to suggest text from the users history.
  * client ID: the ID to be marked for usage stats.  Not used if the user does not opt-in the "sending usage stats" configuration.

## Detailed Design

We have three policies for obfuscation:

  * We do not use a product-wide "master key".  Instead, each key of data encryption is different on each machine.  We want to prevent "casual" leaks such like uploading/publishing on the web, so "no product-wide common keys".
  * We trusts processes which runs with the user ID.  That is, if some malicious cracker succeeds to inject a virus on the machine and it knows Mozc quite well, the data encryption framework can't help.
  * Again, we want to prevent only "casual" leaks.  If you really want to protect your data, you should enable file-system encryption at OS-level.

Then, we take the following process:

  * Mozc firstly created a "password file", which stores the key and salt of AES256 CBC.
  * Then apply AES256 CBC for the data to be obfuscated.

Then, how to protect the "password file"?  It's up to the platform.

  * Windows: Uses [CryptProtectData](http://msdn.microsoft.com/en-us/library/windows/desktop/aa380261.aspx) and [CryptUnprotectData](http://msdn.microsoft.com/en-us/library/windows/desktop/aa380882.aspx) APIs, which guarantee the data is encrypted with keys of user credential and the machine id.  It means, the password file has to be differently encrypted if the user id is different and/or the machine is different.
  * Mac: The password file itself is also obfuscated with AES256 CBC, where the key and salt is generated from the machine's serial number and the user's ID in a predictable way, which means that anyone who knows the machine's serial number and the user's ID can read the password file.
  * Linux: no protection.  The key and salt are stored as plain data. Anyone who has the OS-level permission to reed the password file can read it.
  * ChromeOS: same as Linux.  But no protection does not cause a serious issue because the OS encrypts the local disk and no normal processes run in ChromeOS.
  * Android: Same as Linux.  But no protection does not cause a serious issue because other applications do not have the OS-level permission to read the Mozc's file.

## Code Location

  * [src/base/encryptor.cc](../../src/base/encryptor.cc): consists of two code logic of this obfuscation mechanism. One is the platform-neutral AES256 CBC logic and the other is platform-dependent protection logic for the password file.
  * [src/base/password_manager.cc](../../src/base/password_manager.cc): the password management (dealing with the password file). The actual data protection logic for the password file is delegated to encruptor.cc.
  * [src/base/unverified_aes256.cc](../../src/base/unverified_aes256.cc): the AES256 CBC implementation. Our implementation hasn't been verified with any authority such as FIPS.  Please do not use this implementation for any security/privacy critical purpose.
  * [src/base/unverified_sha1.cc](../../src/base/unverified_sha1.cc): the SHA1 implementation. Our implementation hasn't been verified with any authority such as FIPS.  Please do not use this implementation for any security/privacy critical purpose.

## Guide to port Mozc to a new platform

Before starting your work, you have to understand the privacy concern described in this document.  You also need to find out the best way to store password, for your platform.  For example, gnome-keyrings might be a choice for Linux implementation.


text:
#include "param_table_widget.h"

#include <stdio.h>

#include <QDebug>
#include <QFont>
#include <QMimeData>
#include <QTableWidgetItem>
#include <QMenu>
#include <QAction>
#include "prefs_widget.h"

#include "string_utils.h"
#include "byte_buffer.h"

DroppableTableWidget :: DroppableTableWidget (QWidget *parent_p, char row_delimiter, char column_delimter)
: QTableWidget (parent_p)
{
	setAcceptDrops (true);
	SetRowDelimiter (row_delimiter);
	SetColumnDelimiter (column_delimter);
	dtw_unpack_text_content_flag = true;

	setContextMenuPolicy (Qt :: CustomContextMenu);
	connect (this, &DroppableTableWidget :: customContextMenuRequested, this, &DroppableTableWidget :: ShowPopupMenu);
}

void DroppableTableWidget :: SetColumnDelimiter (char delimiter)
{
	dtw_column_delimiter = delimiter;
}

char DroppableTableWidget :: GetColumnDelimiter () const
{
	return dtw_column_delimiter;
}

void DroppableTableWidget :: SetRowDelimiter (char delimiter)
{
	dtw_row_delimiter = delimiter;
}

char DroppableTableWidget :: GetRowDelimiter () const
{
	return dtw_row_delimiter;
}

void DroppableTableWidget :: dropEvent (QDropEvent *event_p)
{
	QList <QUrl> urls = event_p -> mimeData () -> urls ();

	if (! (urls.isEmpty ()))
		{
			QString filename = urls.first ().toLocalFile ();

			if (! ((filename.isEmpty ()) || (filename.isNull ())))
				{
					QByteArray ba = filename.toLocal8Bit ();
					const char * const filename_s = ba.constData ();

					qDebug () << "dropped " << filename;

					LoadText (filename_s);

					qDebug () << "rows " << rowCount () << " cols " << columnCount ();

				}		/* if (! (filename.isEmpty ())) */

		}		/* if (! (urls.isEmpty ())) */
}

void DroppableTableWidget :: dragEnterEvent (QDragEnterEvent *event_p)
{
	event_p -> acceptProposedAction ();
	event_p -> accept ();
}

void DroppableTableWidget :: dragMoveEvent (QDragMoveEvent *event_p)
{
	event_p -> accept ();
}

bool DroppableTableWidget :: dropMimeData (int row, int column, const QMimeData *data_p, Qt :: DropAction action)
{
	return true;
}

void DroppableTableWidget :: ShowPopupMenu (const QPoint &p)
{
	QMenu *menu_p = new QMenu (this);

	QAction *action_p = new QAction (tr ("Clear Table"), this);
	connect (action_p, &QAction :: triggered, this, &DroppableTableWidget :: clear);
	menu_p -> addAction (action_p);

	menu_p->exec (mapToGlobal (p));
}

char *DroppableTableWidget :: GetEntry (const char *start_s, const char *end_s)
{
	char *value_s = NULL;

	if (!end_s)
		{
			size_t l = strlen (start_s);
			end_s = start_s + l;
		}

	if (dtw_unpack_text_content_flag)
		{
			const char *first_value_p = start_s;
			bool found_flag = false;

			while ((!found_flag) && (first_value_p < end_s))
				{
					if (*first_value_p == '\"')
						{
							found_flag = true;
						}

					++ first_value_p;
				}

			if (found_flag)
				{
					if (end_s)
						{
							const char *last_value_p = end_s;

							found_flag = false;

							while ((!found_flag) && (last_value_p > first_value_p))
								{
									if (*last_value_p == '\"')
										{
											found_flag = true;
										}
									else
										{
											-- last_value_p;
										}
								}

							if (found_flag)
								{
									value_s = CopyToNewString (first_value_p, last_value_p - first_value_p, false);
								}
						}
					else
						{
							value_s = CopyToNewString (first_value_p, 0, false);
						}
				}
			else
				{
					if (end_s != start_s)
						{
							value_s = CopyToNewString (start_s, end_s - start_s, false);
						}
				}
		}
	else
		{
			value_s = CopyToNewString (start_s, end_s - start_s, false);
		}

	return value_s;
}

void DroppableTableWidget :: SetRow (const int row, const char *data_s)
{
	const char *current_token_s = data_s;
	const char *next_token_s;
	int col = 0;
	bool loop_flag = true;

	if (row >= rowCount ())
		{
			setRowCount (row + 1);
		}

	while (loop_flag)
		{
			char *value_s = NULL;

			if (*current_token_s != dtw_column_delimiter)
				{
					next_token_s = strchr (current_token_s, dtw_column_delimiter);

					if (next_token_s)
						{
							value_s = GetEntry (current_token_s, next_token_s);

							if (!value_s)
								{
									value_s = CopyToNewString (current_token_s, next_token_s - current_token_s, false);
								}

							current_token_s = next_token_s + 1;
						}
					else
						{
							value_s = GetEntry (current_token_s, NULL);
							loop_flag = false;
						}
				}
			else
				{
					++ current_token_s;
				}

			QTableWidgetItem *item_p = item (row, col);

			if (item_p)
				{
					item_p -> setText (value_s);
				}
			else
				{
					item_p = new QTableWidgetItem (value_s);

					if (col >= columnCount ())
						{
							setColumnCount (col + 1);
						}

					setItem (row, col, item_p);
				}

			if (row == 0)
				{
					QFont font;
					font.setBold (true);

					item_p -> setFont (font);
				}

			qDebug () << "num rows " << rowCount () << " num cols " << columnCount ();

			++ col;

			FreeCopiedString (value_s);
		}		/* while (loop_flag) */
}

bool DroppableTableWidget :: IsTableEmpty () const
{
	const int num_rows = rowCount ();
	const int num_cols = columnCount ();

	for (int i = 0; i < num_rows; ++ i)
		{
			for (int j = 0; j < num_cols; ++ j)
				{
					QTableWidgetItem *item_p = item (i, j);

					if (item_p)
						{
							QString s = item_p -> text ().trimmed ();

							if (s.length () > 0)
								{
									return false;
								}
						}
				}
		}

	return true;
}

char *DroppableTableWidget :: GetValueAsText ()
{
	char *value_s = NULL;

	if (!IsTableEmpty ())
		{
			ByteBuffer *buffer_p = AllocateByteBuffer (1024);

			if (buffer_p)
				{
					const int num_rows = rowCount ();
					const int num_cols = columnCount ();
					bool success_flag = true;

					for (int i = 0; i < num_rows; ++ i)
						{
							for (int j = 0; j < num_cols; ++ j)
								{
									QTableWidgetItem *item_p = item (i, j);

									if (item_p)
										{
											QString s = item_p -> text ().trimmed ();
											QByteArray ba = s.toLocal8Bit ();
											const char *item_value_s = ba.constData ();

											success_flag = AppendStringToByteBuffer (buffer_p, item_value_s);
										}

									if (success_flag)
										{
											success_flag = AppendToByteBuffer (buffer_p, &dtw_column_delimiter, 1);
										}

									if (!success_flag)
										{
											i = num_rows;
											j = num_cols;
										}
								}

							if (success_flag)
								{
									if (!AppendToByteBuffer (buffer_p, &dtw_row_delimiter, 1))
										{
											success_flag = false;
											i = num_rows;
										}
								}
						}

					if (success_flag)
						{
							const char *data_s = GetByteBufferData (buffer_p);

							value_s = CopyToNewString (data_s, 0, false);
							qDebug () << value_s  << endl;
						}

					FreeByteBuffer (buffer_p);
				}

		}		/* if (!IsTableEmpty ()) */

	return value_s;
}

bool DroppableTableWidget :: SetValueFromText (const char *data_s)
{
	bool success_flag = false;

	return success_flag;
}

void DroppableTableWidget :: LoadText (const char *filename_s)
{
	FILE *in_f = fopen (filename_s, "r");

	qDebug () << "rows " << rowCount () << " cols " << columnCount ();

	if (in_f)
		{
			bool loop_flag = true;
			char *buffer_s = NULL;
			int row = 0;

			setSortingEnabled (false);

			while (loop_flag)
				{
					if (GetLineFromFile (in_f, &buffer_s))
						{
							if (*buffer_s)
								{
									SetRow (row, buffer_s);

									++ row;
								}
							else
								{
									loop_flag = false;
								}
						}
					else
					{
						loop_flag = false;
					}
				}

			setSortingEnabled (true);

			fclose (in_f);
		}

	qDebug () << "rows " << rowCount () << " cols " << columnCount ();
}

ParamTableWidget :: ParamTableWidget (Parameter * const param_p, const PrefsWidget * const options_widget_p, QWidget *parent_p)
:		BaseParamWidget (param_p, options_widget_p)
{
	ptw_delimiter = ',';
	ptw_table_p = new DroppableTableWidget (parent_p, '\n', '|');
}

ParamTableWidget ::	~ParamTableWidget ()
{
	delete ptw_table_p;
	ptw_table_p = NULL;
}

void ParamTableWidget :: RemoveConnection ()
{
}

void ParamTableWidget :: SetDefaultValue ()
{
	const char *value_s = bpw_param_p -> pa_default.st_string_value_s;
}

QWidget *ParamTableWidget :: GetQWidget ()
{
	return ptw_table_p;
}

bool ParamTableWidget :: StoreParameterValue ()
{
	bool success_flag  = false;
	char *value_s = ptw_table_p ->  GetValueAsText ();

	if (value_s)
		{
			SetParameterValue (bpw_param_p, value_s, true);

			success_flag = true;
			FreeCopiedString (value_s);
		}

	return success_flag;
}

bool ParamTableWidget :: SetValueFromText (const char *value_s)
{
	bool success_flag  = false;
	const char *current_row_s = value_s;
	const char *next_row_s;
	int row = 0;

	while ((next_row_s = strchr (current_row_s, '\n')) != NULL)
		{
			char *row_s = CopyToNewString (current_row_s, next_row_s - current_row_s, false);

			if (row_s)
				{
					ptw_table_p -> SetRow (row, row_s);
					FreeCopiedString (row_s);
				}

			current_row_s = next_row_s;
			++ row;
		}

	return success_flag;

}

bool ParamTableWidget :: SetValueFromJSON (const json_t * const value_p)
{
	bool success_flag = false;

	if (json_is_string (value_p))
		{
			const char *value_s = json_string_value (value_p);

			success_flag = SetValueFromText (value_s);
		}

	return success_flag;
}

void ParamTableWidget :: ShowErrors (const json_t *errors_p)
{
	if (errors_p)
		{
			if (json_is_array (errors_p))
				{
					size_t i;
					json_t *value_p;
					const int num_columns = ptw_table_p -> columnCount ();

					json_array_foreach (errors_p, i, value_p)
						{
							int row;

							if (GetJSONInteger (value_p, "row", &row))
								{
									const char *error_s = GetJSONString (value_p, "error");
									QBrush b (Qt :: red);

									for (int i = num_columns - 1; i >= 0; -- i)
										{
											QTableWidgetItem *item_p = ptw_table_p -> item (row, i);

											if (item_p)
												{
													item_p -> setBackground (b);
													item_p -> setToolTip (error_s);
												}
										}
								}

						}		/* json_array_foreach (errors_p, i, value_p) */

				}		/* if (json_is_array (errors_p)) */

		}		/* if (errors_p) */
}


text:
using System;
using System.Xml.Serialization;
using Newtonsoft.Json;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace Gedcomx.Model
{
    public class VocabElementList
    {
        private String title;
        private String description;
        private String uri;
        private String id;
        private List<VocabElement> elements = new List<VocabElement>();

        public String Id
        {
            get
            {
                return id;
            }
            set
            {
                id = value;
            }
        }

        public String Title
        {
            get
            {
                return title;
            }
            set
            {
                title = value;
            }
        }

        public String Description
        {
            get
            {
                return description;
            }
            set
            {
                description = value;
            }
        }

        public String Uri
        {
            get
            {
                return uri;
            }
            set
            {
                uri = value;
            }
        }

        public List<VocabElement> Elements
        {
            get
            {
                return elements;
            }
        }

        public VocabElementList AddElement(VocabElement element)
        {
            this.elements.Add(element);
            return this;
        }
    }
}


text:
<!DOCTYPE HTML>
<html xmlns="http://www.w3.org/1999/xhtml"
      xmlns:th="http://www.thymeleaf.org"
      xmlns:layout="http://www.ultraq.net.nz/web/thymeleaf/layout"
      layout:decorator="layout/main">
<head>
    <title>Alternative Payments Java SDK Sample Application</title>
</head>
<body>
    <section layout:fragment="menu" th:include="transaction/fragments/transaction-menu :: transaction-menu">
    </section>
    <section layout:fragment="content">
        <p th:text="'Transaction with ID ' + ${transaction.id} + ' created!'" />
        <ul th:replace="transaction/fragments/one-transaction :: 'one-transaction' (transaction=${transaction})">
        </ul>
    </section>
</body>
</html>


text:
module EmsRefresh::SaveInventoryObjectStorage
  def save_ems_object_storage_inventory(ems, hashes, target = nil)
    target = ems if target.nil?
    log_header = "EMS: [#{ems.name}], id: [#{ems.id}]"

    # Check if the data coming in reflects a complete removal from the ems
    if hashes.blank?
      target.disconnect_inv
      return
    end

    _log.info("#{log_header} Saving EMS Object Storage Inventory...")
    if debug_trace
      require 'yaml'
      _log.debug "#{log_header} hashes:\n#{YAML.dump(hashes)}"
    end

    child_keys = [
      :cloud_object_store_containers,
      :cloud_object_store_objects,
    ]

    # Save and link other subsections
    save_object_storage_child_inventory(ems, hashes, child_keys, target)

    ems.save!
    hashes[:id] = ems.id

    _log.info("#{log_header} Saving EMS Object Storage Inventory...Complete")

    ems
  end

  def save_object_storage_child_inventory(obj, hashes, child_keys, *args)
    child_keys.each { |k| send("save_object_storage_#{k}_inventory", obj, hashes[k], *args) if hashes.key?(k) }
  end

  def save_object_storage_cloud_object_store_containers_inventory(ems, hashes, target = nil)
    target = ems if target.nil?

    ems.cloud_object_store_containers.reset
    deletes = if target == ems
                :use_association
              else
                []
              end

    hashes.each do |h|
      h[:ems_id]          = ems.id
      # h[:cloud_tenant_id] = h.fetch_path(:tenant, :id) # TODO: add "unless h[:cloud_tenant_id]" so can be common?
    end

    save_inventory_multi(ems.cloud_object_store_containers, hashes, deletes, [:ems_ref], nil, :tenant)
    store_ids_for_new_records(ems.cloud_object_store_containers, hashes, :ems_ref)
  end

  def save_object_storage_cloud_object_store_objects_inventory(ems, hashes, target = nil)
    target = ems if target.nil?

    ems.cloud_object_store_objects.reset
    deletes = if target == ems
                :use_association
              else
                []
              end

    hashes.each do |h|
      h[:ems_id]                          = ems.id
      # h[:cloud_tenant_id]                 = h.fetch_path(:tenant, :id) # TODO: add "unless h[:cloud_tenant_id]" so can be common?
      h[:cloud_object_store_container_id] = h.fetch_path(:container, :id)
    end

    save_inventory_multi(ems.cloud_object_store_objects, hashes, deletes, [:ems_ref], nil, [:tenant, :container], true)
    store_ids_for_new_records(ems.cloud_object_store_objects, hashes, :ems_ref)
  end
end


text:
function [ img ] = eigenfaces_reconstruct( efm, weights )
%EIGENFACES_RECONSTRUCT Reconstructs a face specified by a set of weights
%   Given a trained Eigenfaces model and a set of weights, this function
%   reconstructs the original face.
%   Math: F = Mean + V * W';

    img = efm.eigenfaces * weights' + efm.meanface';
    img = reshape(img, efm.imagesize);
end


text:
<?php
// +----------------------------------------------------------------------
// | YunCMS
// +----------------------------------------------------------------------
// | Copyright (c) 2006-2016 http://www.yunalading.com All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: jabber <2898117012@qq.com>
// +----------------------------------------------------------------------

namespace app\install\controller;

use app\common\controller\BaseController;

/**
 * Class InstallBaseController
 * @package app\install\controller
 */
abstract class InstallBaseController extends BaseController {
    /**
     * InstallBaseController constructor.
     */
    public function __construct() {
        parent::__construct();
    }
}


text:
package com.dailystudio.development;

import java.io.File;

import com.dailystudio.BuildConfig;

import android.os.Environment;
import android.text.TextUtils;
import android.util.Log;

public class Logger {

	private static enum LogToken {
		LOG_D,
		LOG_SD,
		LOG_W,
		LOG_I,
		LOG_E,
	}

	private static final String SUPPRESS_FILE = "dslog_suppress";
	private static final String FORCE_FILE = "dslog_force";

	private static final String UNKNOWN_METHOD = "UnknownMethod";
	private static final String UNKNOWN_CLASS = "UnknownClass";
	private static final int TRACE_BASE_INDEX = 3;

	private static final String DEBUG_MSG_TEMPL = "%s(): %s";

	private static final String UNKNOWN_TAG = "Unknown";

	private static volatile boolean sLogDebugEnabled = BuildConfig.DEBUG;
	private static volatile boolean sLogSecureDebugEnabled = BuildConfig.DEBUG;

	private static void output(String format, LogToken token, Object... args) {
		final String compose = String.format(DEBUG_MSG_TEMPL,
				getCallingMethodName(2), format);

		if (token == null) {
			token = LogToken.LOG_D;
		}

		String tag = getCallingSimpleClassName(2);
		if (TextUtils.isEmpty(tag)) {
			tag = UNKNOWN_TAG;
		}

		if (token == LogToken.LOG_D
				|| token == LogToken.LOG_SD) {
			Log.d(tag, String.format(compose, args));
		} else if (token == LogToken.LOG_W) {
			Log.w(tag, String.format(compose, args));
		} else if (token == LogToken.LOG_I) {
			Log.i(tag, String.format(compose, args));
		} else if (token == LogToken.LOG_E) {
			Log.e(tag, String.format(compose, args));
		}
	}

	public static void setDebugEnabled(boolean enabled) {
		sLogDebugEnabled = enabled;
	}

	public static boolean isDebugEnabled() {
		return sLogDebugEnabled;
	}

	public static void setSecureDebugEnabled(boolean enabled) {
		sLogSecureDebugEnabled = enabled;
	}

	public static boolean isSecureDebugEnabled() {
		return sLogSecureDebugEnabled;
	}

	public static boolean isDebugSuppressed() {
		return isDebugSuppressed(SUPPRESS_FILE);
	}

	public static boolean isPackageDebugSuppressed(String pkg) {
		if (TextUtils.isEmpty(pkg)) {
			return false;
		}

		StringBuilder sb = new StringBuilder(SUPPRESS_FILE);
		sb.append('.');
		sb.append(pkg);

		return isDebugSuppressed(sb.toString());
	}

	private static boolean isDebugSuppressed(String supTagFile) {
		return isTagFileExisted(supTagFile);
	}

	public static boolean isDebugForced() {
		return isDebugSuppressed(FORCE_FILE);
	}

	public static boolean isPackageDebugForced(String pkg) {
		if (TextUtils.isEmpty(pkg)) {
			return false;
		}

		StringBuilder sb = new StringBuilder(FORCE_FILE);
		sb.append('.');
		sb.append(pkg);

		return isDebugForced(sb.toString());
	}

	private static boolean isDebugForced(String forceTagFile) {
		return isTagFileExisted(forceTagFile);
	}

	private static boolean isTagFileExisted(String tagfile) {
		if (TextUtils.isEmpty(tagfile)) {
			return false;
		}

		final String state = Environment.getExternalStorageState();
		if (Environment.MEDIA_MOUNTED.equals(state)
				|| Environment.MEDIA_MOUNTED_READ_ONLY.equals(state)) {
			final File externalStorage =
					Environment.getExternalStorageDirectory();
			if (externalStorage != null) {
				final File supfile = new File(externalStorage, tagfile);
				if (supfile.exists()
						&& supfile.isFile()) {
					return true;
				}
			}
		}

		return false;
	}

	public static void info(String format, Object... args) {
		output(format, LogToken.LOG_I, args);
	}

	public static void debug(String format, Object... args) {
		if (sLogDebugEnabled) {
			output(format, LogToken.LOG_D, args);
		}
	}

	public static void debugSecure(String format, Object... args) {
		if (sLogSecureDebugEnabled) {
			output(format, LogToken.LOG_SD, args);
		}
	}

	@Deprecated
	public static void warnning(String format, Object... args) {
		output(format, LogToken.LOG_W, args);
	}

	public static void warn(String format, Object... args) {
		output(format, LogToken.LOG_W, args);
	}

	public static void error(String format, Object... args) {
		output(format, LogToken.LOG_E, args);
	}

	public static StackTraceElement getCallingElement(int traceLevel) {
		if (traceLevel < 0) {
			return null;
		}

		StackTraceElement[] elements = Thread.currentThread().getStackTrace();
		if (elements == null) {
			return null;
		}

//		dumpStackTraceElements(elements);

		final int length = elements.length;
		if ((TRACE_BASE_INDEX + traceLevel) >= length) {
			return null;
		}

		return elements[TRACE_BASE_INDEX + traceLevel];
	}

/*	private static void dumpStackTraceElements(StackTraceElement[] elements) {
		if (elements == null) {
			return;
		}

		final int length = elements.length;
		for (int i = 0; i < length; i++) {
			Log.d(Logger.class.getSimpleName(),
					String.format("dumpStackTraceElements(): [%03d]: element[%s]",
							i, elements[i]));
		}
	}
*/
	public static String getCallingMethodName() {
		return getCallingMethodName(1);
	}

	public static String getCallingMethodName(int traceLevel) {
		StackTraceElement element = getCallingElement(traceLevel + 1);
		if (element == null) {
			return UNKNOWN_METHOD;
		}

		return element.getMethodName();
	}

	public static String getCallingClassName() {
		return getCallingClassName(1);
	}

	public static String getCallingSimpleClassName() {
		return getCallingSimpleClassName(1);
	}

	public static String getCallingSimpleClassName(int traceLevel) {
		String className = getCallingClassName(traceLevel + 1);
		if (className == null) {
			return UNKNOWN_CLASS;
		}

		Class<?> kls = null;
		try {
			kls = Class.forName(className);
		} catch (ClassNotFoundException e) {
			kls = null;
		}

		if (kls == null) {
			return UNKNOWN_CLASS;
		}

		return kls.getSimpleName();
	}

	public static String getCallingClassName(int traceLevel) {
		StackTraceElement element = getCallingElement(traceLevel + 1);
		if (element == null) {
			return UNKNOWN_CLASS;
		}

		return element.getClassName();
	}

}


text:
// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.

using System.Collections.Generic;
using System.Threading;
using Microsoft.CodeAnalysis.CSharp.Extensions.ContextQuery;
using Microsoft.CodeAnalysis.CSharp.Utilities;

namespace Microsoft.CodeAnalysis.CSharp.Completion.KeywordRecommenders
{
    internal class ClassKeywordRecommender : AbstractSyntacticSingleKeywordRecommender
    {
        private static readonly ISet<SyntaxKind> s_validModifiers = new HashSet<SyntaxKind>(SyntaxFacts.EqualityComparer)
            {
                SyntaxKind.NewKeyword,
                SyntaxKind.PublicKeyword,
                SyntaxKind.ProtectedKeyword,
                SyntaxKind.InternalKeyword,
                SyntaxKind.PrivateKeyword,
                SyntaxKind.AbstractKeyword,
                SyntaxKind.SealedKeyword,
                SyntaxKind.StaticKeyword,
                SyntaxKind.UnsafeKeyword
            };

        public ClassKeywordRecommender()
            : base(SyntaxKind.ClassKeyword)
        {
        }

        protected override bool IsValidContext(int position, CSharpSyntaxContext context, CancellationToken cancellationToken)
        {
            var syntaxTree = context.SyntaxTree;
            return
                context.IsGlobalStatementContext ||
                context.IsTypeDeclarationContext(
                    validModifiers: s_validModifiers,
                    validTypeDeclarations: SyntaxKindSet.ClassInterfaceStructTypeDeclarations,
                    canBePartial: true,
                    cancellationToken: cancellationToken) ||
                syntaxTree.IsTypeParameterConstraintStartContext(position, context.LeftToken, cancellationToken);
        }
    }
}