text:
// WriteThink.cs
//
// Copyright (c) 2013 Brent Knowles (http://www.brentknowles.com)
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//
// Review documentation at http://www.yourothermind.com for updated implementation notes, license updates
// or other general information/
//
// Author information available at http://www.brentknowles.com or http://www.amazon.com/Brent-Knowles/e/B0035WW7OW
// Full source code: https://github.com/BrentKnowles/YourOtherMind
//###
using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using System.Data;
using System.Text.RegularExpressions;
using System.Collections;
using RulesMess;
using CoreUtilities;

// ToDo: Merge this class with yomspellchecker

namespace WriteThinker
{
    /// <summary>
    /// Purposes
    /// - breaking a body of text into database rows so
    ///   SQL operations can be applied to analyze the text
    ///
    /// Feature #1
    /// - She Said/He Said: select a character and see the lines attributed to them
    ///     with a confidence level.
    ///     Purpose: Allows the writer to look for consistency issues with dialog
    ///
    ///
    ///
    ///
    ///
    ///
    /// ACCURACY
    ///  Erased: 75% dialog
    /// Collector: 54% dialog
    ///
    ///
    /// BUGS
    ///   - There was a bug where if there was not a SPACE between a . and the next sentence that sentence would be skipped
    ///      HOWEVER: It appears the bug only happens when loading from a text file, which only happens during test. Pasting the offensive text
    ///        into a Keeper note we were unable to reproduce the problem (WILL NOT FIX)
    ///
    /// </summary>
    public class WriteThink
    {

        int nSeconds = 0; // time to run last 'rebuild'

        public DataTable table = null; // the table assigned when created
        public DataView view = null;
        public CharacterInDialogClass[] characters = null;
        public  KnowledgeBase kb = null;
        private string GETSPEAKER_NameExpression = "";
        Regex nameRegex = null;

        private string GETSPEAKER_RegexForPronouns = "";
        Regex pronounRegex = null;

        private string GETSPEAKER_HeSaidExpression = "";
        Regex hesaids = null;

        private int GETSPEAKER_CONTEXTWEIGHT = 0;

        static Regex passiveworddetector = new Regex
    (@"\b(am|is|are|was|were|be|been|can|could|shall|should|may|might|must|will|would|seem|became|did|do|does|being)\b", RegexOptions.Compiled);

        #region StaticUtilities
        /// <summary>
        /// Replaces � with "
        ///
        /// Like God intended.
        /// </summary>
        /// <param name="sSource"></param>
        /// <returns></returns>
        static public string ReplaceFunkyQuotesWithNormal(string sSource)
        {
            sSource = sSource.Replace('�', '\"');
            sSource = sSource.Replace('�', '\"');
            sSource = sSource.Replace('�', '\'');
            sSource = sSource.Replace('�', '\'');
            return sSource;
        }
        #endregion

        /// <summary>
        /// shoudl only be called after initializaiton
        ///
        /// will generally be used for unit tests to make sure changes
        /// in parser still mathc previous results
        /// </summary>
        public int GetLines(DataTable computetable, string sFilter)
        {
            if (computetable == null)
            {
                throw new Exception("table not initialized");
            }
            return (int)computetable.Compute("count(id)", sFilter);

        }

        /// <summary>
        /// replaces the datatable and view with new one
        /// whichever function calls this will have to call fadmin.LoadTable
        /// afterwards
        /// </summary>
        /// <param name="sFile"></param>
        public void LoadFromFile(string sFile)
        {
            DataSet ds = new DataSet();
            ds.ReadXml(sFile, XmlReadMode.ReadSchema);
            table = ds.Tables[0];
            ds = null;
        }

        /// <summary>
        /// loads an admingrid form
        /// </summary>
        /// <param name="nSeconds"></param>
        public void AdminGrid(int nSeconds)
        {
            if (table == null || view == null)
            {
                throw new Exception("table via LoadTextIntoDatabase needs to be set before AdmiNGrid()");
            }
            fAdminGrid grid = new fAdminGrid();

            grid.LoadTable(view);

            //ToDo: hook this up to console command
            grid.WindowState = System.Windows.Forms.FormWindowState.Minimized;
            grid.labeltimer.Text = nSeconds.ToString();
            grid.writethink = this;
            grid.Show();

        }

        /// <summary>
        /// checks the sentence for passivity
        ///
        /// Note: This is a crude estimate, not very accurate (Not accurate at all)
        /// </summary>
        /// <param name="sSource"></param>
        /// <returns></returns>
        static public bool IsPassive(string sSource, bool bDialog)
        {
            if (bDialog == true) return false;

            MatchCollection matches = passiveworddetector.Matches(sSource);
            if (matches.Count > 0)
            {

                // look through the passive potentials
                foreach (Match match in matches)
                {
                    int nStart = sSource.IndexOf(match.Value) + match.Value.Length;
                    int nLength = 15;
                    if (nLength > (sSource.Length - nStart))
                    {
                        nLength = sSource.Length - nStart;
                    }
                    string neartext = sSource.Substring(nStart, nLength);
                    if (neartext.IndexOf("ed") > -1)
                    {
                        return true;
                    }
                }

                return false;
            }
            return false;
        }

        /// <summary>
        /// returns true if sSource is a fragment
        /// </summary>
        /// <param name="sSource"></param>
        /// <returns></returns>
        static public bool IsFragment(string sSource)
        {
            return false;
        }

        /// <summary>
        ///  counts the word
        ///
        /// I was doing an inline calculation but it was
        /// not entirely accurate
        /// </summary>
        /// <param name="sSource"></param>
        /// <returns></returns>
        static public int GetWordCount(string sSource)
        {
/*
            int c = 0;
            for (int i = 1; i < sSource.Length; i++)
            {
                if (char.IsWhiteSpace(sSource[i - 1]) == true)
                {
                    if (char.IsLetterOrDigit(sSource[i]) == true ||
                        char.IsPunctuation(sSource[i]))
                    {
                        c++;
                    }
                }
                else if (i >= 3 &&
                    char.IsLetterOrDigit(sSource[i]) &&
                    char.IsPunctuation(sSource[i - 1]) &&
                    char.IsPunctuation(sSource[i - 2]) &&
                    char.IsLetterOrDigit(sSource[i - 3]))
                {
                    c++;
                }
            }
            if (sSource.Length > 2)
            {
                c++;
            }
            return c;*/

            int w;
            //Regex reg = new Regex(@"\w+"); // worked but was a little high
            Regex reg = new Regex(@"[\S]+");
            MatchCollection mc = reg.Matches(sSource);
            if (mc.Count > 0)
                w = mc.Count;
            else
                w = 0;
            return w;

        }

        static public int GetSyllables(string sSource)
        {
            sSource = sSource.ToLower().Trim();
            string pattern = "[aeiouy]+";
            int count = Regex.Matches(sSource, pattern).Count;
            if (sSource.EndsWith("e"))
            {
                if (Regex.IsMatch(sSource, "[aeiouy][^aeiouy]e$") == true)
                {
                    count -= 1;
                }

            }
            if (count < 1)
            {
                count = 1;
            }
            return count;
        }

        #region englishdata

        //ToDo: These should probably be pulled from a database?
        string[] saidisms = new string[20] { "read", "laughed", "voice", "said", "shouted", "shouts", "asked", "asks",
            "responds", "responded", "remarked", "remarks", "spoke","whisper","whispered", "objected","scream", "screamed", "demand", "demanded" };
        #endregion

        private string BuildExpressionForHeSaid(string[] caps)
        {
            // dynamically build a regex match expression
            // for detecting "who said" stuff in dialog
            //ToDo: Will probably want to grab all the capital letters and find matches for them too

        //    string s(\bhe shouted\b)

            string sResult = "";

            string[] identifiers = new string[pronouns.Length + caps.Length];
            Array.Copy(caps, identifiers, caps.Length);
            Array.Copy(pronouns, 0, identifiers, caps.Length , pronouns.Length);
            // remember I
            foreach (string s in identifiers)
            {
                string sMultiple = ""; // we set this to | which in regex means OR but only if more

                foreach (string saidism in saidisms)
                {
                    if (sResult != "")
                    {
                        sMultiple = "|";
                    }
                    sResult = String.Format("{0}{1}\\b{2} {3}\\b", sResult, sMultiple, s, saidism);
                }
            }

            return sResult;
        }
        public string[] GetCapitalLetters(string sSource)
        {
            Regex caps = new Regex(@"(\b[A-Z]\w*)");
            MatchCollection matches = caps.Matches(sSource);
            string[] capsfound = new string[matches.Count];
            int nCount = 0;
            foreach(Match match in matches)
            {

                capsfound[nCount] = match.Value;
                nCount++;
            }
            return capsfound;
        }

        /*DataTable femaleNames = null;
        DataTable maleNames = null;
        DataTable lastNames = null;

        /// <summary>
        ///  This worked but I decided to have the user define characte details instead
        /// </summary>
        /// <param name="sSource"></param>
        /// <returns></returns>
        private string[] GetCapitalLettersFilteredForNames(string sSource)
        {
            // only load Female names once
            if (femaleNames == null)
            {
                DataSet ds = new DataSet();
                ds.ReadXml("C:\\Documents and Settings\\brent\\My Documents\\Keeper\\Data\\Words\\femalenames.xml");
                femaleNames = ds.Tables[0];
            }
            // only load Female names once
            if (maleNames == null)
            {
                DataSet ds = new DataSet();
                ds.ReadXml("C:\\Documents and Settings\\brent\\My Documents\\Keeper\\Data\\Words\\malenames.xml");
                maleNames = ds.Tables[0];
            }
            // only load Female names once
            if (lastNames == null)
            {
                DataSet ds = new DataSet();
                ds.ReadXml("C:\\Documents and Settings\\brent\\My Documents\\Keeper\\Data\\Words\\lastnames.xml");
                lastNames = ds.Tables[0];
            }

            // grabs the capital letters but then removes any that do not
            // appear on any name list

            // ToDo: Need to integrate properly with Keeper? (Once word lists are moved into YOM)
            string[] caps = GetCapitalLetters(sSource);
            if (caps.Length > 0)
            {
                for (int i = caps.Length - 1; i >= 0; i--)
                {
                    bool bFound = false;
                    // walk backwards
                    // delete entries with no match
                    foreach (DataRow row in femaleNames.Rows)
                    {
                        if (row["Word"].ToString() == caps[i])
                        {
                            bFound = true;
                            break; // we found it, okay
                        }

                    } // foreach

                    if (bFound == false)
                    {
                        foreach (DataRow row in maleNames.Rows)
                        {
                            if (row["Word"].ToString() == caps[i])
                            {
                                bFound = true;
                                break; // we found it, okay
                            }
                        }
                    }

                    if (bFound == false)
                    {
                        foreach (DataRow row in lastNames.Rows)
                        {
                            if (row["Word"].ToString() == caps[i])
                            {
                                bFound = true;
                                break; // we found it, okay
                            }
                        }
                    }

                    if (bFound == false)
                    {
                        caps[i] = "";
                    }

                }

            }
            return caps;
        }
        */

        /// <summary>
        /// WIll look thru the passed in character string
        /// </summary>
        private string[] CharacterNames
        {
            get { return new string[2] { "Isabel", "Meredith" }; }

        }

    /// <summary>
    /// generates a regex for pronoune, one of the last
    /// ditch efforts if we still have not found a speaker
    /// </summary>
    /// <returns></returns>
         public string GetRegexForPronouns()
        {
            string sResult = "";

            foreach (string s in pronouns)
            {
                string sMultiple = "";

                if (sResult != "")
                {
                    sMultiple = "|";
                }
                sResult = String.Format("{0}{1}\\b{2}\\b", sResult, sMultiple, s);
            }
            return sResult;
        }
        /// <summary>
        /// tries to guesstimate the speaker of sSource dialog line
        /// </summary>
        /// <param name="sSource"></param>
        /// <param name="sBefore"></param>
        /// <param name="sAfter"></param>
        /// <param name="confidence"></param>
        /// <returns></returns>
        private string GetSpeaker(string sSource, string sBefore, string sAfter,
            string sBeforeSpeaker, out double confidence)
        {

            string sContext = sBefore + " " + sAfter;

            if (characters == null)
            {
                throw new Exception("GetSPeaker - we need a character array");
            }

            if (pronouns == null)
            {
                throw new Exception("pronouns not defined? DId you build the AI table?");
            }

            if (kb == null)
            {
                throw new Exception("knowledge base needs to be built");
            }

            if (GETSPEAKER_RegexForPronouns == "")
            {
                throw new Exception("pronouns were not defined.");
            }

            if (hesaids == null)
            {
                throw new Exception("we need the hesaid regex built");
            }
            if (pronounRegex == null)
            {
                throw new Exception("we need the pronoun regex built");
            }

            if (sSource.IndexOf("Listen,") > -1)
            {
              //  int breaker = 9;
            }

            // This worked to get he and shes and whatnot but I wanted to try something
            // more elabroate
            //Regex caps = new Regex(@"(\bhe|\bI\b|\bit\b|\bHe\b|\bShe\b|\bshe\b|(\b[A-Z]\w*))");

            MatchCollection matches = hesaids.Matches(sContext);

            confidence = 0.0;

            ArrayList output = new ArrayList();

            foreach (Match match in matches)
            {
                // remove the 'said' bit from the string
                string sValue = match.Value;

                //ToDo: Potential slow (but only about one second
                foreach (string s in saidisms)
                {
                    if (sValue.IndexOf(s) > -1) // this did not improve speed
                    {
                        sValue = sValue.Replace(s, "");
                    }
                }

                sValue = sValue.Trim();
                output.Add(sValue);

            }

            // now check for pronouns
          ///  Regex pronouns = new Regex(@"(\bhe|\bI\b|\bit\b|\bHe\b|\bShe\b|\bshe\b)");
          //  matches = pronouns.Matches(sContext);

            /////////////////////////
            //
            // Rules Reference
            // in case I get confused
            //
            // 1. he/she said will trump  (includes pronoun he/she but only if followed with a saidism)
            // 2. then we look for a previous speaker (i.e., if line of dialog before me had a speaker, use that)
            // 2. then we look for a Name
            // 3. then we look for the most likely name
            // ToDo: 4 - just look for presence of pronouns
            // ToDo: 5. If we still do not have a match we try to match a Pronoun with a Name
            //
            //////////////////////////////////////////////////////

            //////////////////////////////////////
            //
            // NO MATCH
            //
            // 1. Now we get into advanced stuff trying to dig up the talker
            //
            //////////////////////////////////////

           // if (matches.Count == 0 /*&& bFoundExactlyOne == false*/)
            {
                // a. previous speaker. If there is a previous speaker lets try that
                if (sBeforeSpeaker != "" && sBeforeSpeaker != "none")
                {
                   // sOutput = sBeforeSpeaker;
                    output.Add(sBeforeSpeaker);
                 //   bFoundExactlyOne = true;
                    confidence = 0.40;
                }

                // Was there a name of a 'Character' on the previous or next line?
                // if so, add to output

                MatchCollection nameMatches = nameRegex.Matches(sBefore + sAfter);
                if (nameMatches.Count > 0)
                {
                    // we found a copy name matches
                    foreach (Match match in nameMatches)
                    {
                        output.Add(match.Value);
                    }

                }

            }

            //////////////////////////////////////////////
            //
            // Pronouns
            //
            // Just look for presence of pronouns if still can't find matches
            //
            /////////////////////////////////////////////
            if (output.Count == 0)
            {

                MatchCollection pronounMatches = pronounRegex.Matches( (sBefore + sAfter).ToLower());
                if (pronounMatches.Count > 0)
                {
                    // we found a copy name matches
                    foreach (Match match in pronounMatches)
                    {
                        output.Add(match.Value);
                    }
                    if (output.Count == 1)
                    {
                        confidence = 0.14;
                       /* bFoundExactlyOne = true;*/
                    }
                    else if (output.Count > 1)
                    {
                        confidence = 0.08;
                    }
                }
            }

            if (output.Count == 1)
            {
                confidence = 0.90;

            }

            ///////////////////////////////////////////////
            //
            // Pronoun Replacement
            //
            // We replace pronouns with proper nouns
            //
            // so at this point we have hesaid with He, Her, et cetera
            // we replace each with a name
            //
            //
            //
            //////////////////////////////////////////////

            // a. If IsPronoun
            for (int matchCount = 0; matchCount < output.Count; matchCount++)
            {

                Relationship bestRelationship = kb.GetMostLikelyDeducedFact(
                     output[matchCount].ToString().ToLower(),
                     sContext,
                     GETSPEAKER_CONTEXTWEIGHT);

                 // now we REPLACE WITH a relationship match if there is one
                if (bestRelationship != null && output[matchCount].ToString() != bestRelationship.B)
                {
                    string sMes = "";// String.Format("Dialog: {0} Context: {1}", sSource, sContext);
                    lg.Instance.Line("WriteThink->GetSpeaker", ProblemType.MESSAGE,
                        output[matchCount] + " has been replaced by "+ bestRelationship.B + " "+
                        sMes
                        );
                    output[matchCount] = bestRelationship.B;
                    confidence = 0.03;
                }

            } // for

            /////////////////////////////////////////
            //
            // TOO MANY MATCHES
            //
            // 1. Now we isolate who might have (logically) spoken
            //
            //   A. Are they the same?
            //   B. Now the same? Pick most likely
            //
            ////////////////////////////////////////////

            // - Too many still? Consolidate down to one if possible
            if (output.Count > 1)
            {
                // Are they all the same? If so, pass with a reasonable confidence
                // sort and if first and lst entry are the same we are just one
                output.Sort();
                if (output[0].ToString() == output[output.Count - 1].ToString())
                {
                    string svalue = output[0].ToString();
                    output.Clear();
                    output.Add(svalue); // add value back into cleared list
                    confidence = 0.50;
                }
            }

            if (output.Count > 1) // only should trigger if we hav emore than 1
            {

                {

                    // now we attempt to see which pronoun, noun, was mentioned most and use that
                    Hashtable hash = new Hashtable();
                    foreach (string s in output)
                    {
                        int nValue = 1;
                        if (hash.ContainsKey(s))
                        {
                            nValue = (int)hash[s];
                            nValue++;
                            hash[s] = nValue;
                        }
                        else
                            hash.Add(s, nValue);
                    }
                    string sMaxValue = "";
                    int nMaxValue = -1;
                    foreach (DictionaryEntry dict in hash)
                    {
                        if ((int)dict.Value > nMaxValue)
                        {
                            sMaxValue = dict.Key.ToString();
                            nMaxValue = (int)dict.Value;
                        }
                    }

                    // okay now we have a max value
                    output.Clear();
                    if (sMaxValue == "I")
                    {
                    //    int breaker = 9;
                    }
                    output.Add(sMaxValue);

                    Console.WriteLine(output.ToString());
                    confidence = 0.15;

                }
            }

            if (output.Count < 1)
            {
                lg.Instance.Line("WriteThink->GetSpeaker", ProblemType.MESSAGE,"no match found " + "pick most popular");
                output.Add(CharacterInDialogClass.GetMostLikelyName(characters, CharacterInDialogClass.gender.any));
            }

            string sOutput = "";
            // transfoer to string
            foreach (string s in output)
            {
                if (output.Count > 1)
                {
                    throw new Exception("At this point there should only be one match");
                }

                // convert returned name to NAME (i.e., an alias of doctor turns into Rutger)

                string sdelimiter = "";
                if (sOutput != "")
                {
                    sdelimiter = ", ";
                }
                sOutput = sOutput + sdelimiter +  CharacterInDialogClass.GetNameFromAlias(s, characters);
            }

            return sOutput;

        }

        /// <summary>
        /// Used to break the dialog apart correctly
        /// </summary>
        /// <param name="lines"></param>
        /// <returns></returns>
        private ArrayList ParseEachLine(string[] lines)
        {
            ArrayList lines2  = new ArrayList();
            bool bMultilinedialog = false;

            foreach (string s in lines)
            {
                try
                {
                    if (s.ToLower().IndexOf("like it") >= 0)
                    {
                      //  int breaker = 9;
                    }
                    Regex r = new Regex("\"");
                    MatchCollection match = r.Matches(s);

                    if (match.Count == 1)
                    {
                        // only one quote means a sentence that has been split
                        // across multiple sentences
                        // just add it and go into dialog mode

                        if (bMultilinedialog == false)
                        {
                            bMultilinedialog = true;
                        }
                        else
                        {
                            bMultilinedialog = false;
                        }

                        // if there's a space after quote we go on our own line
                        int nPos = s.IndexOf("\"");

                        if (nPos + 1 >= s.Length)
                        {
                            // July 2010 - this should not come up often but specific files ending with a two sentence statement
                            // at the end of the file could cause a crash.
                            nPos = nPos - 1;
                        }

                        if (s[nPos + 1] == ' ')
                        {

                            // it might also be in the middle of another string
                            // so we look near it for a space
                            // if so we split into two string
                            lines2.Add(s.Substring(0, nPos + 1));
                            lines2.Add(s.Substring(nPos + 1, s.Length - nPos - 1));

                        }
                        else
                            if (nPos > 0 && s[nPos - 1] == '.')
                            {
                                // if we have a ." match then we keep
                                // the quote as part of the first line
                                int nquoteposition = s.IndexOf("\"");
                                lines2.Add(s.Substring(0, nquoteposition + 1));
                                lines2.Add(s.Substring(nquoteposition + 1, s.Length - nquoteposition - 1));
                            }
                            else
                            {
                                int nquoteposition = s.IndexOf("\"");
                                lines2.Add(s.Substring(0, nquoteposition));

                                // june 11 2009 - my intention here is for the 'second line to get the quote?
                                // so as in He sat down. "This is the end

                                lines2.Add(s.Substring(nquoteposition, s.Length - nquoteposition));

                                // just add line
                                //    lines2.Add(s);

                                // no, copy everything before quote as one string
                                // and everything after as one
                            }
                    }
                    else
                        if (s.IndexOf("\"") > 1)
                        {

                            // go through the substring and every 2 " characters we split
                            int quote = 0;
                            int start = 0;
                            for (int i = 0; i < s.Length; i++)
                            {
                                if (s[i] == '\"')
                                {
                                    quote++;

                                    if (bMultilinedialog == true)
                                    {
                                        // if we are multi line and got here
                                        // we have a situation where we have 3 or another
                                        // uneven number of quotes
                                        //
                                        // basically we have not closed the previous
                                        // string correctly.
                                        // in this case we do act like there's an extra quote
                                        quote++;
                                        bMultilinedialog = false;
                                    }

                                    if (quote == 1)
                                    {
                                        // the start of the sentence needs to be
                                        // removed -- the bit before the quote starts
                                        string sSub = s.Substring(start, i - start);
                                        //if (sSub != "" && sSub != " ")
                                        {
                                            lines2.Add(sSub);
                                        }
                                        start = i;
                                    }
                                    if (quote == 2)
                                    {
                                        // peel off substring and
                                        // add
                                        lines2.Add(s.Substring(start, i - start + 1));
                                        quote = 0;
                                        start = i + 1;
                                    }

                                } // a quote
                                // if at end of strign copy the 'rest'
                                if (i == s.Length - 1)
                                {
                                    int nLengthS = i - start;
                                    if (nLengthS < 0)
                                    {
                                        nLengthS = 0; // sometimes we had less than zero which is a crash
                                    }
                                    string sSub = s.Substring(start, nLengthS);
                                    //if (sSub != "" && sSub != " ")
                                    {
                                        lines2.Add(sSub);
                                        if (sSub.IndexOf("\"") >= 0)
                                        {
                                            // trailing quote found
                                            bMultilinedialog = true;
                                        }
                                    }
                                }

                            }

                            /* string[] newlines = s.Split('\"');
                             foreach (string s1 in newlines)
                             {
                                 if (s1 != "" && s1 != " ")
                                 {
                                     // add quotes back in
                                     lines2.Add("\"" + s1 + "\"");
                                 }
                             }*/
                        }
                        else
                        {
                            if (bMultilinedialog == true)
                            {
                                lines2.Add("\"" + s);
                            }
                            else
                            {
                                lines2.Add(s);
                            }
                        }
                } // try
                catch (Exception ex)
                {
                    CoreUtilities.NewMessage.Show(String.Format("Error: {0} with message {1}", s, ex.ToString()));
                }
            }// loop
            return lines2;
        }

        string[] pronouns = null;

        public string speakerrules = "";

        /// <summary>
        /// ToDo: These will be from a file
        ///
        /// Loads the base AI tables and performs deduction
        ///
        /// Observations:
        ///  1. Too many 'is speakers' degrades qualtiy, especially if that
        ///     word isn't being equated to anything (which makes sense, if I say
        ///     banjo is speaker but then never defined banjo as a character
        ///     it will appear as a hesaidshesaid but never be attached to a character
        ///     --> The more character attachments we can make then the more likely quality will improve
        /// </summary>
        public void BuildAITables()
        {
            if (speakerrules == "")
            {
                throw new Exception("speaker rules has not been set (file");
            }
            kb = new KnowledgeBase();
            kb.LoadFromXmlTable(speakerrules);

            // my pronoun list is areally a list of speakers
            //static string[] pronouns = new string[7] { "I", "friend", "companion", "he", "she", "it","his" };
         /*   kb.Add(new Relationship("he", "is", "speaker"));
            kb.Add(new Relationship("his", "is", "speaker"));
            kb.Add(new Relationship("she", "is", "speaker"));
            kb.Add(new Relationship("her", "is", "speaker"));
            kb.Add(new Relationship("friend", "is", "speaker"));
            kb.Add(new Relationship("companion", "is", "speaker"));
        //    kb.Add(new Relationship("it", "is", "speaker")); = this is so seldom true and it screws up

            kb.Add(new Relationship("read", "has", "speaker")); // no purpose, but making sure only 'is' creates a is speaker rule

            kb.Add(new Relationship("his", "is", "malepronoun"));
            kb.Add(new Relationship("she", "is", "femalepronoun"));
            kb.Add(new Relationship("he", "is", "malepronoun")); // so these would stay in
            kb.Add(new Relationship("her", "is", "femalepronoun")); // so these would stay in

            // we put i in lowercase because all gets converted to this
            kb.Add(new Relationship("i", "is", "pov"));
            kb.Add(new Relationship("my", "is", "pov"));

            */
            kb.Add(new Relationship("malepronoun", "is",
                CharacterInDialogClass.GetMostLikelyName(characters,
                CharacterInDialogClass.gender.male), "", 75, 0, false));
            kb.Add(new Relationship("femalepronoun", "is",
                CharacterInDialogClass.GetMostLikelyName(characters,
                CharacterInDialogClass.gender.female), "", 75, 0, false));
            kb.Add(new Relationship("pov", "is",
                CharacterInDialogClass.GetMostLikelyName(characters,
                CharacterInDialogClass.gender.any), "", 75, 0, false));

            string[] malenames = CharacterInDialogClass.GetAllCharacterNames(characters, CharacterInDialogClass.gender.male);
            foreach (string s in malenames)
            {
                kb.Add(new Relationship("malepronoun", "is",s, "", 50, 0, false));
            }

            string[] femalenames = CharacterInDialogClass.GetAllCharacterNames(characters, CharacterInDialogClass.gender.female);
            foreach (string s in femalenames)
            {
                kb.Add(new Relationship("femalepronoun", "is",s, "", 50, 0, false));
            }

            // now we add all the other males/females but with a lower weight

            kb.Deduction();

            pronouns = kb.GetDefinedFor("speaker");
            //new string[7] { "I", "friend", "companion", "he", "she", "it","his" };
            // Optmiziation
            // Moving some of these queries up here so they don't get executed
            // every loop
            GETSPEAKER_NameExpression = CharacterInDialogClass.GetRegexForNames(characters);
            nameRegex = new Regex(GETSPEAKER_NameExpression);

            GETSPEAKER_RegexForPronouns = GetRegexForPronouns();
             pronounRegex = new Regex(GETSPEAKER_RegexForPronouns);

            string[] sCaps = CharacterInDialogClass.GetAllCharacterNames(characters, CharacterInDialogClass.gender.any);

            GETSPEAKER_HeSaidExpression = BuildExpressionForHeSaid(sCaps);
            hesaids = new Regex(GETSPEAKER_HeSaidExpression, RegexOptions.IgnoreCase);

            Relationship[] contextWeight = kb[new RelationKey("pronouncontextweight", KnowledgeBase.DEDUCED_IS)];

            if (contextWeight == null)
            {
                throw new Exception("pronouncontextweight must be defined inrules datable with an integer value");
            }

            GETSPEAKER_CONTEXTWEIGHT = Int32.Parse(contextWeight[0].B);

        }

        DateTime timer;

        /// <summary>
        /// returns the length of time it has taken LoadTextStringIntoDatabase to run
        /// needs to be called right after that calld
        /// </summary>
        /// <returns></returns>
        public int GetSecondsElapsedSinceLoadTextStringIntoDatabase()
        {
            TimeSpan span = DateTime.Now - timer;
            nSeconds = span.Seconds;
            return span.Seconds;
        }

        public DataTable LoadTextStringIntoDatabase(string sText, CharacterInDialogClass[] _characters)
        {
            return LoadTextStringIntoDatabase(sText, _characters, "", false);
        }
        /// <summary>
        /// returns first numberofwords
        /// </summary>
        /// <param name="input"></param>
        /// <param name="numberWords"></param>
        /// <returns></returns>
        public static string FirstWords(string input, int numberWords)
        {
            try
            {
                // Number of words we still want to display.
                int words = numberWords;
                // Loop through entire summary.
                for (int i = 0; i < input.Length; i++)
                {
                    // Increment words on a space.
                    if (input[i] == ' ' || input[i] == ',' || input[i]=='-' || ';' == input[i] || ':'==input[i])
                    {
                        words--;
                    }
                    // If we have no more words to display, return the substring.
                    if (words == 0)
                    {
                        return input.Substring(0, i);
                    }
                }
            }
            catch (Exception)
            {
                // Log the error.
            }
            return string.Empty;
        }

        /// <summary>
        /// loads text into the database as oppose to LoadTextIntoDatabase which
        /// takes a file
        ///
        ///
        /// </summary>
        /// <param name="sTextString"></param>
        /// <param name="rtf">CUT: if passed in will attempt to do character matching by color (Experimental Spetember 2011)</param>
        /// <param name="KeepTable"> if true we don't rebuild the TABLE, for use with color matching</param>
        /// <param name="OverrideCharacter">Name of the character to override, with color matching system</param>
        /// <param name="_characters"></param>
        /// <returns></returns>
        public DataTable LoadTextStringIntoDatabase(string sText, CharacterInDialogClass[] _characters, string OverrideCharacter, bool KeepTable)
        {
            timer = DateTime.Now;

            this.characters = _characters;
            // Hving null characters is okay, though it will produce almost meaningless results
             if (characters == null)
            {
                 // create a simple character array with one entry
                this.characters = new CharacterInDialogClass[1];
                this.characters[0] = new CharacterInDialogClass();
                this.characters[0].Name = "default";
             //   throw new Exception("Need character information to parse dialog");
            }

            sText = CleanupImportString(sText);

            if (sText == null)
            {
                throw new Exception("text string was null");
            }

            if (false == KeepTable)
            {
                table = new DataTable();
                DataColumn auto = new DataColumn("ID", typeof(int));

                auto.AutoIncrement = true;
                auto.AutoIncrementSeed = 1;// DateTime.Now.Millisecond;
                auto.AutoIncrementStep = 1;
                table.Columns.Add(auto);
                table.PrimaryKey = new DataColumn[] { auto };

                DataColumn text = new DataColumn("Text");
                table.Columns.Add(text);

                table.Columns.Add("Confidence", typeof(double));

                table.Columns.Add("dialog", typeof(bool), "Text Like '%\"%'");
                table.Columns.Add("contraction", typeof(bool), "Text Like '%\'\'%'");
                table.Columns.Add("hesaidshesaid");
                DataColumn charLen = new DataColumn("charLen", typeof(int), "Len(Text)");

                table.Columns.Add(charLen);
                DataColumn wordLen = new DataColumn("wordLen", typeof(int));// "Len(Text)/5");
                table.Columns.Add(wordLen);

                table.Columns.Add("ispassive", typeof(bool));
                table.Columns.Add("isfragment", typeof(bool));
                table.Columns.Add("syllables", typeof(int));
                table.Columns.Add("startswithverb", typeof(bool)); // August 2012
                table.Columns.Add("hidelinebyline", typeof(bool)); // November 2012 - to hide during a line by line search
            } // keep table

            // precleanup: convert quotes
            sText = ReplaceFunkyQuotesWithNormal(sText);

           // a regex with some handling for abbreviations

          //    Regex.Split(sText, @"(?<=['""A-Za-z0-9][\.\!\?])\s+(?=[A-Z])");
                //worked but flawed

            // this is suppose to be faster (using compiled with a constant string)
                Regex splitter = new Regex(@"(?:(?<=\.|\!|\?)(?<!Mr\.|\...|Dr\.)(?<!U\.S\.A\.)\s+(?=[A-Z]))", RegexOptions.Compiled);

                string[] lines = splitter.Split(sText);

            /* Nope, won't work, doesn't match
                if ("" != rtf)
                {
                    rtflines = splitter.Split(rtf); // september 2011 (color coding)
                }
            */

            // * Parse # 2 - now we extract dialog
            ArrayList lines2 = ParseEachLine(lines);

            //ArrayList lines2 = new ArrayList(lines);

            // we have to do 2 loops
            // the first loop removes blanks
            for (int deletepass = lines2.Count - 1; deletepass > -1; deletepass--)
            {
                lines2[deletepass] = lines2[deletepass].ToString().Replace("bbb.", "");
                lines2[deletepass] = lines2[deletepass].ToString().Replace("bbb", "");
                lines2[deletepass] = lines2[deletepass].ToString().Trim();
                if (lines2[deletepass].ToString() == "" || lines2[deletepass].ToString() == " ")
                {
                    lg.Instance.Line("WriteThink->LoadTextStringIntoDatabase", ProblemType.MESSAGE, lines2[deletepass].ToString() + " deleting");
                    lines2.Remove(lines2[deletepass]);

                }
            }
            // tried doing a dialog regex but realized that
            // failed too because they would neve rmatch
            // because the strings are split in one but not the other based on periods

            // set up knowledge table
            // ToDo: Should be loading from file
            if (false == KeepTable)
            {
                BuildAITables();
            }

            int nCount = -1;

            // Major pass; primarily grabbing Speaker but also passive and stuff
            foreach (string s in lines2)
            {
                nCount++;

                /* My attempt a COLOR PASS: Ignore this until I get back to it. I don't think I want to dit
                // september 2011
              //  move this code to where dialog is
                if (true)
                {
                    int indexofrtf = rtf.IndexOf(s.Replace("\"", "").Trim()); // remove quote marks because they are removed
                    string rtf_temp = "";
                    if (indexofrtf > -1)
                    {
                        rtf_temp = rtf.Substring(indexofrtf-4, s.Length + 4);
                    }
                }*/

                string snewline = s;
                //  string snewline = s.Replace("bbb.", "");
                //  snewline = snewline.Replace("bbb", "");
              //  if (snewline != "" && snewline != " ")  no longer needed, we do this in a seperate loop
                {

                    // add core string to datatable
                    DataRow row = table.NewRow();
                    row["hidelinebyline"] = false;
                    row["Text"] = snewline;
                    row["hesaidshesaid"] = "none";
                    row["isfragment"] = IsFragment(snewline);
                    row["wordLen"] = GetWordCount(snewline);

                    bool bIsDialog = snewline.IndexOf("\"") > -1;

                    row["ispassive"] = IsPassive(snewline, bIsDialog);
                    double confidence = 0.0;
                    string sPrevious = "";
                    string sNext = "";
                    string sBeforeSpeaker = "";

                    if (Layout.LayoutDetails.Instance.WordSystemInUse.IsVerb(FirstWords(snewline, 1)))
                    {
                        row["startswithverb"] = true;
                    }
                    else
                    {
                        row["startswithverb"] = false;
                    }

                    // * Code for the color coding
                    if ("" != OverrideCharacter)
                    {
                        // we don't do the next section
                        // if we are overriding a character we ASSUME
                        // we passed dialog in.
                        row["hesaidshesaid"] = OverrideCharacter;
                        confidence = 1.0;
                    }
                    else

                    // don't calculate hesaid unless there is actual dialog
                    if (bIsDialog == true)
                    {
                        if (nCount > 0)
                        {
                            // don't grab lines looking for context if they were dialog lines

                            sPrevious = lines2[nCount - 1].ToString();
                            if (sPrevious.IndexOf("\"") > -1)
                            {
                                sPrevious = ""; //
                            }
                            sBeforeSpeaker = table.Rows[table.Rows.Count - 1]["hesaidshesaid"].ToString();

                        }
                        if (nCount < lines2.Count - 1)
                        {

                            sNext = lines2[nCount + 1].ToString();
                            if (sNext.IndexOf("\"") > -1)
                            {
                                sNext = ""; //
                            }

                        }
                        row["hesaidshesaid"] = GetSpeaker(snewline, sPrevious, sNext, sBeforeSpeaker, out confidence);
                    }
                    row["Confidence"] = confidence;
                    row["syllables"] = GetSyllables(snewline);
                    table.Rows.Add(row);
                    row = null;
                }
            }
            return table;
        }
        /// <summary>
        /// used for cleaning out tokens from the string
        /// </summary>
        /// <param name="sLine"></param>
        /// <returns></returns>
        public string CleanupImportString(string sLine)
        {

            sLine = sLine.Replace("\t", " bbb. "); // remove tabs
            sLine = sLine.Replace("\n", " bbb. "); // remove tabs
            sLine = sLine.Replace("\r", " bbb. "); // remove tabs

            return sLine;
        }

        /// <summary>
        /// Loadsa a file  into database
        /// </summary>
        /// <param name="sTextFile"></param>
        /// <returns></returns>
        public DataTable LoadTextIntoDatabase(string sTextFile, CharacterInDialogClass[] _characters)
        {
            if (File.Exists(sTextFile) == false)
            {
                throw new Exception(String.Format("File {0} does not exist."));
            }

            // load the text into one giant string

            // the encoding is important because it gets rid of characters
            StreamReader file = new StreamReader(sTextFile, System.Text.Encoding.Default);
            string sText = "";
            if (file != null)
            {
                string sLine = file.ReadLine();
                while (sLine != null)
                {
                   // sLine = CleanupImportString(sLine); moved this into LoadTextInto
                    if (sLine == "" || sLine == " ")
                    {
                        sLine = " bbb. "; // need to insert this to have proper parabreaks
                    }
                    sText = sText + sLine;
                    sLine = file.ReadLine();
                }
            }

            file.Close();

            return LoadTextStringIntoDatabase(sText, _characters);

        }
        /// <summary>
        /// returns the number of lines of dialog
        /// </summary>
        /// <param name="table"></param>
        /// <param name="sFilter"></param>
        /// <returns></returns>
        public int GetLinesOfDialog(DataTable table, string sFilter)
        {
            return (int)table.Compute("count(id)", "dialog=true");
        }

        /// <summary>
        /// /shows a stat panel for the current text
        /// </summary>
        public void ShowStatPanel(System.Drawing.Icon icon)
        {

            // all routines here need to use Table
            // so we can filter it at a later point

            string sFilter = "";

            DataTable computeTable = this.table;
            if (view != null)
            {
                computeTable = view.ToTable();
            }
            fStatPanel panel = new fStatPanel();
            panel.Icon = icon;
            if (computeTable.Rows.Count > 0)
            {
                // retrievals
                int nMinLineLength = (int)computeTable.Compute("min(wordLen)", sFilter);
                int nMaxLineLength = (int)computeTable.Compute("max(wordLen)", sFilter);
                int nNumberSentences = GetLines(computeTable, sFilter);
                int nPassiveSentences = (int)computeTable.Compute("count(id)", "ispassive=true");

                int nNumberDialog = GetLinesOfDialog(computeTable, sFilter);
                long nNumberDialogWords = 0;
                try
                {
                    nNumberDialogWords = (long)computeTable.Compute("sum(wordLen)", "dialog=true");
                }
                catch (Exception)
                {
                }

                long nTotalSyllables = (long)computeTable.Compute("sum(syllables)", sFilter);
                long nWords = (long)computeTable.Compute("sum(wordLen)", sFilter);

                // calculations
                float percentdialog = ((float)nNumberDialog / (float)nNumberSentences) * 100;
                float percentdialogwords = ((float)nNumberDialogWords / (float)nWords) * 100;
                float avgwordspersentence = (float)nWords / (float)nNumberSentences;
                float avgsyllable = (float)nTotalSyllables / (float)nNumberSentences;

                double readingease = 206.835 - (1.015 * ((double)nWords / (double)nNumberSentences)) -
                    (84.6 * ((double)nTotalSyllables / (double)nWords));

                double gradelevel = (0.39 * ((double)nWords / (double)nNumberSentences)) +
                    (11.8 * ((double)nTotalSyllables / (double)nWords)) - 15.59;

                float percentpassive = ((float)nPassiveSentences / (float)nNumberSentences) * 100;

                string sFormat = "0.00";

                panel.statPanel1.labelMinLine.Text = nMinLineLength.ToString();
                panel.statPanel1.labellongestsentence.Text = nMaxLineLength.ToString();
                panel.statPanel1.labelnumsentences.Text = nNumberSentences.ToString();
                panel.statPanel1.labelpercentdialog.Text = percentdialog.ToString(sFormat);
                panel.statPanel1.labelwords.Text = nWords.ToString();
                panel.statPanel1.labelreadingease.Text = readingease.ToString(sFormat);
                panel.statPanel1.labelgrade.Text = gradelevel.ToString(sFormat);
                panel.statPanel1.labelpassive.Text = String.Format("{0} %", percentpassive.ToString(sFormat));
                panel.statPanel1.labelavgsyl.Text = avgsyllable.ToString(sFormat);
                panel.statPanel1.labelpercentdialogwords.Text = percentdialogwords.ToString(sFormat);
                panel.statPanel1.labelavgwordsper.Text = avgwordspersentence.ToString(sFormat);
              //  panel.statPanel1.labelDebugTimeTaken.Text = nSeconds.ToString();
                panel.ShowDialog();
            }
            else
            {
                panel.statPanel1.Visible = false;
                panel.labelNoText.Visible = true;
                panel.ShowDialog();

            }

        }

        /// <summary>
        /// compares our current table with a grade table that is passed in.
        ///
        /// We are looking at the hesaidshesaid dialog lines.
        ///
        /// ** We are assuming a perfectly Hand-graded gradeTable. **
        ///
        ///
        /// </summary>
        /// <param name="gradeTable"></param>
        /// <returns>a percent fail. If there are 100 lines and 20 of them different then we return 80</returns>
        public double Grade(DataTable gradeTable)
        {
            double Total = 0; // table.Rows.Count;
            double Mistakes = 0;
            double result = 0.0;

            if (gradeTable.Rows.Count != table.Rows.Count)
            {
                lg.Instance.Line("WriteThink->Grade", ProblemType.MESSAGE, "grade tables do not match " + "failed");
                return 0.0;
            }

            for (int i = 0; i < table.Rows.Count; i++)
            {
                if ((bool)table.Rows[i]["dialog"] == true)
                {
                    Total++; // we only count dialog lines
                    string sNew = table.Rows[i]["hesaidshesaid"].ToString();
                    string sBenchmark = gradeTable.Rows[i]["hesaidshesaid"].ToString();
                    if (sNew != sBenchmark)
                    {
                        lg.Instance.Line("WriteThink->Grade", ProblemType.MESSAGE,

                            String.Format("New: {0}, Old{1}", sNew, sBenchmark) + " "+
                            table.Rows[i]["Text"].ToString());
                        Mistakes++;
                    }
                }
            }
            result = ((Total - Mistakes) / Total) * 100.0;
            return result;

        }
    } // class
    #region archive
    /*
     *
 //  { ",\"", ".\"", "?\"", "!\"", "." },
            // initial seperate one
            // split into an array based on '.'
            //string[] lines = sText.Split(new string[1] {"." }, StringSplitOptions.RemoveEmptyEntries);

          //  string[] lines = sText.Split(new string[1] { "\\""":" },
          //      StringSplitOptions.RemoveEmptyEntries);

         /*   string[] lines =
                Regex.Split(sText, @"(?<=['A-Za-z0-9][""\.\!\?])\s+(?=[A-Z])");

            // - extracting dialog tes t-- not 100% happy with my rough approach
            // This works for simple dialog lines Regex dialog = new Regex("\"([^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Multiline);
             Regex dialog = new Regex("\"([^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Multiline);
             MatchCollection matches = dialog.Matches(sText);

            string[] dialogregex = new string[matches.Count];
           for (int i = 0; i < matches.Count; i++)
            {
                Match match = matches[i];
                dialogregex[i] = matches[i].Value;
            }

            sText = sText.Replace("\"", "");//if I remove quotes linefeeds work properly
            // seems to catch dialog
            // step2 would be to toss this into an array
            // and then when we do the MERGE below we integrate

   // MERGE INTEGRATE (dialog with non-dialog)
            // 1. look through Match list from Dialog-Regex
            // 2. If this string exists in 's' then we remove it from s
            // 3. and add string from Dialog-Regex
            // 4. Trick? Knowing where the line should appear?

            foreach (string dialogstring in dialogregex)
            {
                // can only do this once I actually break stirngs properly
                // remove quotes for purpose of comparision
                string stempdialogstring = dialogstring.Replace('\"', ' ').Trim();

                // #1 easy comparision
                // if identical match, replace, quotes back in
                int nPossibleMatch = lines2.IndexOf(stempdialogstring);
               // string sourcelines = lines2[nPossibleMatch].ToString().Replace("bbb.", "").Replace("bbb", "");
                if (nPossibleMatch > -1)
                {
                    lines2[nPossibleMatch] = dialogstring;
                }

                // #2 - partial match

            }
*/
    #endregion

}


text:
package org.docksidestage.dockside.dbflute.whitebox.cbean.bigartist.myself;

import org.dbflute.cbean.result.ListResultBean;
import org.dbflute.cbean.scoping.SubQuery;
import org.dbflute.exception.SpecifyColumnTwoOrMoreColumnException;
import org.docksidestage.dockside.dbflute.cbean.MemberCB;
import org.docksidestage.dockside.dbflute.cbean.MemberServiceCB;
import org.docksidestage.dockside.dbflute.exbhv.MemberBhv;
import org.docksidestage.dockside.dbflute.exentity.Member;
import org.docksidestage.dockside.dbflute.exentity.MemberService;
import org.docksidestage.dockside.dbflute.exentity.ServiceRank;
import org.docksidestage.dockside.unit.UnitContainerTestCase;

/**
 * @author jflute
 */
public class WxCBMyselfExistsTest extends UnitContainerTestCase {

    // ===================================================================================
    //                                                                           Attribute
    //                                                                           =========
    private MemberBhv memberBhv;

    // ===================================================================================
    //                                                                               Basic
    //                                                                               =====
    public void test_myselfExists_basic() {
        // ## Arrange ##
        ListResultBean<Member> memberList = memberBhv.selectList(cb -> {
            /* ## Act ## */
            cb.setupSelect_MemberStatus();
            cb.query().setMemberStatusCode_Equal_Formalized();
            cb.query().myselfExists(new SubQuery<MemberCB>() {
                public void query(MemberCB subCB) {
                    subCB.query().setMemberName_LikeSearch("S", op -> op.likePrefix());
                }
            });
        });

        // ## Assert ##
        assertFalse(memberList.isEmpty());
        for (Member member : memberList) {
            log(member.getMemberName() + ", " + member.getMemberStatus().get().getMemberStatusName());
            assertTrue(member.isMemberStatusCodeFormalized());
            assertTrue(member.getMemberName().startsWith("S"));
        }
    }

    public void test_myselfExists_OneToOne() {
        // ## Arrange ##
        ListResultBean<Member> memberList = memberBhv.selectList(cb -> {
            /* ## Act ## */
            cb.setupSelect_MemberServiceAsOne().withServiceRank();
            cb.query().setMemberStatusCode_Equal_Formalized();
            cb.query().queryMemberServiceAsOne().myselfExists(new SubQuery<MemberServiceCB>() {
                public void query(MemberServiceCB subCB) {
                    subCB.query().setServiceRankCode_Equal_Gold();
                }
            });
        });

        // ## Assert ##
        assertFalse(memberList.isEmpty());
        for (Member member : memberList) {
            MemberService service = member.getMemberServiceAsOne().get();
            ServiceRank rank = service.getServiceRank().get();
            log(member.getMemberName() + ", " + member.getMemberStatusCode() + ", " + rank.getServiceRankName());
            assertTrue(member.isMemberStatusCodeFormalized());
            assertTrue(service.isServiceRankCodeGold());
        }
    }

    // ===================================================================================
    //                                                                             Specify
    //                                                                             =======
    public void test_myselfExists_specify_basic() {
        // ## Arrange ##
        String memberStatusCode = memberBhv.selectByPK(3).get().getMemberStatusCode();
        ListResultBean<Member> memberList = memberBhv.selectList(cb -> {
            /* ## Act ## */
            cb.setupSelect_MemberStatus();
            cb.query().myselfExists(new SubQuery<MemberCB>() {
                public void query(MemberCB subCB) {
                    subCB.specify().columnMemberStatusCode();
                    subCB.query().setMemberId_Equal(3);
                }
            });
        });

        // ## Assert ##
        assertHasAnyElement(memberList);
        for (Member member : memberList) {
            member.getMemberStatus().alwaysPresent(status -> {
                log(member.getMemberName() + ", " + status.getMemberStatusName());
                assertEquals(memberStatusCode, member.getMemberStatusCode());
            });
        }
    }

    public void test_myselfExists_specify_nested() {
        // ## Arrange ##
        String memberStatusCode = memberBhv.selectByPK(3).get().getMemberStatusCode();
        ListResultBean<Member> memberList = memberBhv.selectList(cb -> {
            /* ## Act ## */
            cb.setupSelect_MemberStatus();
            cb.query().myselfExists(new SubQuery<MemberCB>() {
                public void query(MemberCB subCB) {
                    subCB.specify().columnMemberStatusCode();
                    subCB.query().myselfExists(new SubQuery<MemberCB>() {
                        public void query(MemberCB subCB) {
                            subCB.query().setMemberId_Equal(3);
                        }
                    });
                }
            });
        });

        // ## Assert ##
        assertHasAnyElement(memberList);
        for (Member member : memberList) {
            member.getMemberStatus().alwaysPresent(status -> {
                log(member.getMemberName() + ", " + status.getMemberStatusName());
                assertEquals(memberStatusCode, member.getMemberStatusCode());
            });
        }
    }

    public void test_myselfExists_specify_duplicated() {
        // ## Arrange ##
        MemberCB cb = new MemberCB();
        cb.setupSelect_MemberStatus();

        try {
            cb.query().myselfExists(new SubQuery<MemberCB>() {
                public void query(MemberCB subCB) {
                    subCB.specify().columnFormalizedDatetime();
                    subCB.specify().columnMemberStatusCode();
                    subCB.query().setMemberId_Equal(3);
                }
            });

            // ## Assert ##
            fail();
        } catch (SpecifyColumnTwoOrMoreColumnException e) {
            // OK
            log(e.getMessage());
        }
    }
}

redpajama_set_name: RedPajamaGithub


text:
#include "finite_buffer_network_model.h"

class FiniteBufferNetworkModelEMesh : public FiniteBufferNetworkModel
{
   public:
      FiniteBufferNetworkModelEMesh(Network* net, SInt32 network_id);
      ~FiniteBufferNetworkModelEMesh();

      volatile float getFrequency() { return _frequency; }

      // Output Summary
      void outputSummary(ostream &out);

      static pair<bool,SInt32> computeCoreCountConstraints(SInt32 core_count);
      static pair<bool,vector<core_id_t> > computeMemoryControllerPositions(SInt32 num_memory_controllers);

   private:
      enum NodeType
      {
         EMESH = 1 // Always start at 1
      };

      // Private Functions
      NetworkNode* createNetworkNode();

      void computeEMeshPosition(core_id_t core_id, SInt32& x, SInt32& y);
      core_id_t computeCoreId(SInt32 x, SInt32 y);
      SInt32 computeEMeshDistance(core_id_t sender, core_id_t receiver);

      // Static Functions
      static void computeEMeshTopologyParameters(SInt32& emesh_width, SInt32& emesh_height);

      // Main Routing Function
      void computeOutputEndpointList(HeadFlit* head_flit, NetworkNode* curr_network_node);

      // Event Count Summary
      void outputEventCountersSummary(ostream& out);
      // Contention Counters Summary
      void outputContentionCountersSummary(ostream& out);

      // Private Variables
      volatile float _frequency;

      string _emesh_network;

      // Topology Parameters
      SInt32 _emesh_width;
      SInt32 _emesh_height;
};


text:
require 'bundler/setup'
Bundler.require(:test)
$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__)))                      # test
$LOAD_PATH.unshift(File.expand_path(File.join(File.dirname(__FILE__), '../lib'))) # lib

require 'minitest/autorun'
require 'minitest/colorize'
require 'mocha/setup'
require 'string_hound'

begin
  require 'ruby-debug'
rescue LoadError
end


text:
This is a Node/express application implementing the game server.
It uses a WebSocket per viewer (see 'Viewer' section) and a UDP
socket to communicate with clients.

## Viewer

This is a HTML5 browser application allowing to view the state of
a game via Canvas 2D graphics. It uses a WebSocket for real-time
communication with the server.


text:
using std::accumulate;
using std::back_inserter;
using std::endl;
using std::make_pair;
using std::move;
using std::ostream;
using std::string;
using std::vector;

using execHelper::config::EnvironmentCollection;
using execHelper::config::EnvironmentValue;
using execHelper::config::Path;

namespace filesystem = std::filesystem;

namespace {
inline auto implodeVector(const vector<string>& toImplode,
                          const string& delimiter) -> string {
    string result;
    return accumulate(toImplode.begin(), toImplode.end(), string(),
                      [&delimiter](std::string& a, const std::string& b) {
                          if(a.empty()) {
                              return string(b);
                          }
                          return a.append(delimiter).append(b);
                      });
}

inline auto implodeVector(const vector<string>& toImplode) -> string {
    return implodeVector(toImplode, " ");
}
} // namespace

namespace execHelper::core {
// cppcheck-suppress passedByValue symbolName=subtasks
Task::Task(std::vector<std::string> subtasks,
           // cppcheck-suppress passedByValue symbolName=environment
           config::EnvironmentCollection environment,
           config::Path workingDirectory) noexcept
    : m_task(std::move(subtasks)),
      m_env(std::move(environment)),
      m_workingDirectory(std::move(workingDirectory)) {
    setWorkingDirectory(m_workingDirectory);
}

auto Task::getTask() const noexcept -> const execHelper::core::TaskCollection& {
    return m_task;
}

auto Task::toString() const -> string { return implodeVector(m_task); }

auto Task::getEnvironment() const noexcept -> const EnvironmentCollection& {
    return m_env;
}

void Task::setWorkingDirectory(const Path& workingDirectory) noexcept {
    LOG(trace) << "Changing working directory of task to " << workingDirectory;
    m_workingDirectory = workingDirectory;

    // Set the PWD environment variable to the working directory
    appendToEnvironment(
        make_pair("PWD", filesystem::absolute(m_workingDirectory).string()));
}

auto Task::getWorkingDirectory() const noexcept -> const Path& {
    return m_workingDirectory;
}

auto Task::append(const string& taskPart) noexcept -> bool {
    m_task.push_back(taskPart);
    return true;
}

auto Task::append(string&& taskPart) noexcept -> bool {
    m_task.push_back(move(taskPart));
    return true;
}

auto Task::append(const TaskCollection& taskPart) noexcept -> bool {
    m_task.reserve(m_task.size() + taskPart.size());
    m_task.insert(std::end(m_task), std::begin(taskPart), std::end(taskPart));
    return true;
}

auto Task::append(TaskCollection&& taskPart) noexcept -> bool {
    m_task.reserve(m_task.size() + taskPart.size());
    move(taskPart.begin(), taskPart.end(), back_inserter(m_task));
    return true;
}

auto Task::setEnvironment(const EnvironmentCollection& env) noexcept -> bool {
    m_env = env;
    setWorkingDirectory(m_workingDirectory);
    return true;
}

auto Task::setEnvironment(EnvironmentCollection&& env) noexcept -> bool {
    m_env = move(env);
    setWorkingDirectory(m_workingDirectory);
    return true;
}

auto Task::appendToEnvironment(EnvironmentValue&& newValue) noexcept -> bool {
    if(m_env.count(newValue.first) > 0U) {
        m_env.erase(newValue.first);
    }
    m_env.emplace(newValue);
    return true;
}

auto Task::appendToEnvironment(EnvironmentCollection&& newValue) noexcept
    -> bool {
    for(auto&& value : newValue) {
        appendToEnvironment(value);
    }
    return true;
}

auto Task::operator==(const Task& other) const noexcept -> bool {
    return (m_task == other.m_task &&
            m_workingDirectory == other.m_workingDirectory &&
            m_env == other.m_env);
}

auto Task::operator!=(const Task& other) const noexcept -> bool {
    return !(*this == other);
}

auto operator<<(ostream& os, const Task& task) noexcept -> ostream& {
    TaskCollection subtasks = task.getTask();
    EnvironmentCollection environment = task.getEnvironment();
    os << string("Task {");

    os << string("Environment(") << environment.size() << "): {";
    for(const auto& envVar : environment) {
        os << string(" ") << envVar.first << ": " << envVar.second << ";";
    }
    os << string("} ");

    os << string("Command(") << subtasks.size() << "): {";
    for(const auto& subTask : subtasks) {
        os << string(" ") << subTask;
    }
    os << string("} ");
    os << string("Working-dir: {") << task.getWorkingDirectory().string()
       << "}";
    os << string("}");
    os << endl;
    return os;
}
} // namespace execHelper::core


text:
/**
 * @file        test/security-manager-tests.cpp
 * @author      Radoslaw Bartosiak <r.bartosiak@samsung.com>
 * @version     1.0
 * @brief       Security manager tests
 */

#include <iostream>
#include <boost/test/unit_test.hpp>
#include <boost/test/unit_test_log.hpp>
#include <boost/test/results_reporter.hpp>
#include <colour_log_formatter.h>
#include <dpl/log/log.h>

struct TestConfig {
    TestConfig()
    {
        boost::unit_test::unit_test_log.set_threshold_level(
            boost::unit_test::log_test_units);
        boost::unit_test::results_reporter::set_level(boost::unit_test::SHORT_REPORT);
        boost::unit_test::unit_test_log.set_formatter(new SecurityManager::colour_log_formatter);
    }
    ~TestConfig()
    {
    }
};

struct LogSetup {
    LogSetup()
    {
        SecurityManager::Singleton<SecurityManager::Log::LogSystem>::Instance().SetTag("SECURITY_MANAGER_TESTS");
    }
    ~LogSetup() {}
};

BOOST_GLOBAL_FIXTURE(TestConfig);
BOOST_GLOBAL_FIXTURE(LogSetup);


text:
"""
Base classes for MIDI I/O.
"""

class Device(dict):
    """
    PortMidi device.
    """
    def __init__(self, name, input, output, **kw):
        self.name = name
        self.input = input
        self.output = output
        for (name, value) in kw.items():
            setattr(self, name, value)

    def __repr__(self):
        args = []
        for (name, value) in self.__dict__.items():
            if not name.startswith('_'):
                arg = '%s=%r' % (name, value)
                args.append(arg)

        args = ', '.join(args)

        return 'Device(%s)' % args

def make_device_query(get_all_devices):
    """
    Create a query function for devices.

    Takes a function that returns all devices, and
    returns a function that can be called to filters
    these. The returned function calls get_all_devices()
    on each invocation.
    """

    def get_devices(**query):
        """
        Somewhat experimental function to get device info
        as a list of Device objects.
        """

        # Todo: raise exception on illegal query arguments
        # Todo: explain query in docstring

        devices = []

        for dev in get_all_devices():
            for (name, value) in query.items():
                if hasattr(dev, name):
                    if getattr(dev, name) != value:
                        break
            else:
                devices.append(dev)

        return devices

    return get_devices

class Port:
    pass

class Input(Port):
    """
    Abstract base class for MIDI Input.
    """

    def poll(self):
        """
        Return the number of messages ready to be received.
        """

        return self._parse()

    def recv(self):
        """
        Return the next pending message, or None if there are no messages.
        """

        self._parse()
        return self._parser.get_msg()

    def __iter__(self):
        """
        Iterate through pending messages.
        """

        self._parse()
        for msg in self._parser:
            yield msg

class Output(Port):
    """
    Abstract base class for MIDI Input.
    """

    def send(self, msg):
        """Send a message on the output port"""

        # For now, just send the message.
        # This can be expanded later with a delay parameter
        # that delays the mesage, or filtering.

        self._send(msg)


text:
package org.knowm.xchange.paymium.service;

import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import org.knowm.xchange.Exchange;
import org.knowm.xchange.paymium.PaymiumAuthenticated;
import org.knowm.xchange.paymium.dto.account.PaymiumOrder;
import si.mazi.rescu.RestProxyFactory;

public class PaymiumTradeServiceRaw extends PaymiumBaseService {

  protected PaymiumAuthenticated paymiumAuthenticated;

  public PaymiumTradeServiceRaw(Exchange exchange) {

    super(exchange);

    this.paymiumAuthenticated =
        RestProxyFactory.createProxy(
            org.knowm.xchange.paymium.PaymiumAuthenticated.class,
            exchange.getExchangeSpecification().getSslUri(),
            getClientConfig());
  }

  public List<PaymiumOrder> getPaymiumOrders(Long offset, Integer limit) throws IOException {
    return paymiumAuthenticated.getOrders(
        apiKey,
        signatureCreator,
        exchange.getNonceFactory(),
        offset,
        limit,
        Arrays.asList("LimitOrder"),
        null);
  }
}


text:
layout: article
title: "Importancia de la accesibilidad"
description: "La accesibilidad no es una característica."
introduction: "La accesibilidad no es una característica. Los usuarios sordos o invidentes no podrán disfrutar de un vídeo si no este no lleva subtítulos o audiodescripción. El tiempo que lleva añadir estos elementos en el vídeo no es nada comparado con la mala experiencia que puedes llegar a ofrecer a los usuarios. Ofréceles a todos los usuarios al menos una experiencia básica."
article:
  written_on: 2014-04-16
  updated_on: 2014-04-29
  order: 4
collection: videos
authors:
  - samdutton
key-takeaways:
  add-a-video:
    - Usa el elemento de vídeo que se cargará y que descodificará y reproducirá el vídeo en tu sitio.
    - Produce vídeos en varios formatos para llegar a varias plataformas móviles.
    - Dales un tamaño correcto a los vídeos y asegúrate de que no sobrepasen los elementos que los contienen.
    - La accesibilidad es importante; añade el elemento de pista como elemento secundario del elemento de vídeo.
remember:
  media-fragments:
    - El API de Media Fragments es compatible con la mayoría de las plataformas, pero no con iOS.
    - Asegúrate de que las solicitudes de intervalo de bytes sean compatibles con tu servidor. Las solicitudes de intervalo se habilitan de manera predeterminada en la mayoría de los servidores, pero algunos servicios de alojamiento pueden inhabilitarlas.
  dont-overflow:
    - No fuerces el tamaño de los elementos cuando este resulte en una relación de aspecto distinta a la del vídeo original. Los elementos achatados o estirados no quedan bien.
  accessibility-matters:
    - El elemento de pista es compatible con Chrome para Android, con iOS Safari y con todos los navegadores actuales de ordenador, excepto con Firefox (consulta caniuse.com/track). También hay varios Polyfill disponibles. Recomendamos Playr o Captionator.
  construct-video-streams:
    - MSE es compatible con Chrome y Opera en Android, y con Internet Explorer 11 y Chrome en ordenador. También está previsto que sea compatible con la Firefox.
  optimize:
    - Imágenes
    - Optimizar la eficacia del contenido
---

{% wrap content%}

{% include modules/toc.liquid %}

## Incluir subtítulos para mejorar la accesibilidad

Para que los medios sean más accesibles en los dispositivos móviles, incluye subtítulos o descripciones mediante el elemento de pista.

{% include modules/remember.liquid title="Important" list=page.remember.accessibility-matters %}

Al usar el elemento de pista, los subtítulos quedan así:

## Añadir un elemento de pista

Es muy sencillo añadir subtítulos en tu vídeo; solo tienes que añadir un elemento de pista como elemento secundario del elemento de vídeo:

{% include_code _code/track.html track html %}

El atributo `src` del elemento de pista asigna una ubicación al archivo de pista.

## Definir subtítulos en el archivo de pista

Un archivo de pista se compone de pies de entrada con tiempos en formato WebVTT:

    WEBVTT

    00:00.000 --> 00:04.000
    Un hombre sentado en la rama de un árbol usando un portátil.

    00:05.000 --> 00:08.000
    La rama se rompe y cae.

    ...

{% include modules/nextarticle.liquid %}

{% endwrap %}


text:
<?php

namespace Magento\Customer\Block\Adminhtml\Edit\Tab\View;

use Magento\Customer\Controller\RegistryConstants;
use Magento\TestFramework\Helper\Bootstrap;

/**
 * Class OrdersTest
 *
 * @magentoAppArea adminhtml
 */
class OrdersTest extends \PHPUnit_Framework_TestCase
{
    /**
     * The orders block under test.
     *
     * @var Orders
     */
    private $block;

    /**
     * Core registry.
     *
     * @var \Magento\Framework\Registry
     */
    private $coreRegistry;

    /**
     * Execute per test initialization.
     */
    public function setUp()
    {
        $objectManager = Bootstrap::getObjectManager();
        $objectManager->get('Magento\Framework\App\State')->setAreaCode('adminhtml');

        $this->coreRegistry = $objectManager->get('Magento\Framework\Registry');
        $this->coreRegistry->register(RegistryConstants::CURRENT_CUSTOMER_ID, 1);

        $this->block = $objectManager->get(
            'Magento\Framework\View\LayoutInterface'
        )->createBlock(
            'Magento\Customer\Block\Adminhtml\Edit\Tab\View\Orders',
            '',
            ['coreRegistry' => $this->coreRegistry]
        );
        $this->block->getPreparedCollection();
    }

    /**
     * Execute post test cleanup.
     */
    public function tearDown()
    {
        $this->coreRegistry->unregister(RegistryConstants::CURRENT_CUSTOMER_ID);
        $this->block->setCollection(null);
    }

    /**
     * Verify that the correct Url is return for a row in the orders grid.
     */
    public function testGetRowUrl()
    {
        $row = new \Magento\Framework\Object(['id' => 1]);
        $this->assertContains('sales/order/view/order_id/1', $this->block->getRowUrl($row));
    }

    /**
     * Verify that the grid headers are visible.
     */
    public function testGetHeadersVisibility()
    {
        $this->assertTrue($this->block->getHeadersVisibility());
    }

    /**
     * Verify the integrity of the orders collection.
     */
    public function testGetCollection()
    {
        $collection = $this->block->getCollection();
        $this->assertEquals(0, $collection->getSize());
        $this->assertEquals(5, $collection->getPageSize());
        $this->assertEquals(1, $collection->getCurPage());
    }

    /**
     * Check the empty grid Html.
     */
    public function testToHtmlEmptyOrders()
    {
        $this->assertEquals(0, $this->block->getCollection()->getSize());
        $this->assertContains("We couldn't find any records.", $this->block->toHtml());
    }

    /**
     * Verify the contents of the grid Html when there is a sales order.
     *
     * @magentoDataFixture Magento/Customer/_files/customer.php
     * @magentoDataFixture Magento/Sales/_files/order.php
     * @magentoDataFixture Magento/Customer/_files/sales_order.php
     */
    public function testToHtmlWithOrders()
    {
        $html = $this->block->toHtml();
        $this->assertContains('100000001', $html);
        $this->assertContains('firstname lastname', $html);
        $this->assertEquals(1, $this->block->getCollection()->getSize());
    }
}


text:
Description
-----------

An example of a https://projects.spring.io/spring-boot/ stateless web-application deployed using `vagrant` and `virtualbox`.

The setup consists of three application `app` servers, two highly-available http://www.haproxy.org/ load-balancers `lb`, and a management server.
The high-availability is achieved using a `pacemaker`/`corosync` cluster http://clusterlabs.org/.
Application deployments and system configurations are automated with https://www.ansible.com/ and monitoring is performed using https://prometheus.io/.

All servers run CentOS 7. This setup assumes a `vagrant` box with the shared folder enabled by default, mounted on the guests as `/vagrant`.

The setup requires 3GB RAM.

Tested on:

- Ubuntu 16.04 x86_64, Vagrant 1.9.5

----------------------
Configuration overview
----------------------

`vagrant` reserves enp0s3 (the first interface) and this cannot be currently changed (see https://github.com/mitchellh/vagrant/issues/2093).

The application servers and load-balancers provide services on the private network on the enp0s8 (the second) interfaces.
The Virtual IP assigned by the `pacemaker` cluster to one of the load-balancers
is used as the backend for an `apache` reverse proxy running on the management server port 80.
`vagrant` port forwarding feature is used to expose the `apache` port 80 on the management server to the `vagrant` host.

In a real-life scenarios, apart from separate networks handing different traffic (management, application),
there would be no need for an `apache` reverse proxy on the management server.
In the current setup the management server is the SPOF (single point of failure), in addition to the obvious SPOF of the `vagrant` host.

                                                    Active
                                                 load-balancer

                                               -----------------
                                               | lb1           |
                                               |               |
                                               | haproxy       |
                             Virtual IP        | node_exporter |
                         192.168.123.20/24-----| pacemaker     |
                                               | corosync      |
                                               -----------------
                                                   |       |
          Management                               |       |                              Application
            server              enp0s3 10.0.2.0/24 |       | enp0s8 192.168.123.21/24       servers
                                                   |       |
       -----------------                           |       |                           -----------------
       | mgt1          |    enp0s3 10.0.2.0/24    -----------   enp0s3 10.0.2.0/24     | appX: X=1..3  |
       |               |--------------------------|         |--------------------------|               |
       | apache        |                          | Vagrant |                          | spring-boot   |
       | prometheus    | enp0s8 192.168.123.31/24 |  HOST   | enp0s8 192.168.123.1X/24 | node_exporter |
       | node_exporter |--------------------------|         |--------------------------|               |
       |               |                          -----------                          |               |
       |               |                           |       |                           -----------------
       -----------------                           |       |
                                enp0s3 10.0.2.0/24 |       | enp0s8 192.168.123.22/24
                                                   |       |
                                                   |       |
                                               -----------------
                                               | lb2           |
                                               |               |
                                               | haproxy       |
                                               | node_exporter |
                                               | pacemaker     |
                                               | corosync      |
                                               -----------------

                                                    Passive
                                                 load-balancer

------------
Sample Usage
------------

Install `virtualbox`, including "VirtualBox Extension Pack" https://www.virtualbox.org/wiki/Downloads and
`vagrant` https://www.vagrantup.com/downloads.html

Checkout the project repo, and the external `ansible` modules:

        $ git clone https://github.com/marcindulak/vagrant-haproxy-pcs-ansible-tutorial-centos7.git
        $ cd vagrant-haproxy-pcs-ansible-tutorial-centos7
        $ cd ansible/roles/external
        $ git clone https://github.com/marcindulak/ansible-pacemaker styopa.pacemaker
        $ # ansible-galaxy install --roles-path . -r requirements.yml  # if you have ansible on the Vagrant host
        $ cd -

Install the required `vagrant` plugins and bring up the VMs:

        $ vagrant plugin install landrush
        $ vagrant up || :
        $ sleep 30  # wait for the corosync quorum

This will start the VMs, install `ansible` on them, setup key-based authentication for the `vagrant` user used for `ansible`,
and deploy https://github.com/spring-guides/gs-spring-boot/ application on the `app` VMs.

The health of the VMs can be monitored with `prometheus` API on `mgt`:9090:

        $ vagrant ssh mgt1.mydomain -c "curl -sgG --data-urlencode query='up{instance=~\"app.*\", job=\"node_exporter\"}' localhost:9090/api/v1/query | python -m json.tool"

`prometheus` port 9090 exposes also a web-gui service which provides the same information as API in so called console, and additionally allows one to create simple graphs.
The `prometheus` port is also configured to be forwarded by `vagrant` and is available at `localhost`:49090.
The screenshots below show how the same API data are presented in the console and graph.

![up_console](https://raw.github.com/marcindulak/vagrant-haproxy-pcs-ansible-tutorial-centos7/master/screenshots/up_console.png)

![up_graph](https://raw.github.com/marcindulak/vagrant-haproxy-pcs-ansible-tutorial-centos7/master/screenshots/up_graph.png)

For an introduction to `prometheus` see "An introduction to monitoring and alerting with time-series at scale, with Prometheus" https://www.youtube.com/watch?v=gNmWzkGViAY. `prometheus` provides it's own alerting system, but it is not configured here.
See "PromCon 2016: Alerting in the Prometheus Universe - Fabian Reinartz" https://www.youtube.com/watch?v=yrK6z3fpu1E for more details about alerting in `prometheus`.

The initial setup of the VMs and `ansible` playbook use internet downloads (one should use a local proxy of software mirrors instead) and most likely the initial configuration of some components will fail.
Further `ansible` runs should be performed from the `mgt` server to hopefully correct the initial problems:

        $ vagrant ssh mgt1.mydomain -c "ansible-playbook -i /vagrant/ansible/hosts.yml /vagrant/ansible/playbook.yml"

Alternatively, this `ansible` playbook could be let run by `vagrant` by:

        $ vagrant provision mgt1.mydomain

The application should be accesible now on `mgt`:80 which points to the Virtual IP managed by
the `pacemaker`/`corosync` cluster, and the corresponding port forwarded by `vagrant` to `localhost`:40080:

        $ vagrant ssh mgt1.mydomain -c "curl loadbalancer:80"
        $ curl localhost:40080

Note that the `ansible` [springboot](ansible/roles/internal/springboot) task modifies any `build.gradle` in order to build
a "fully executable" jar https://docs.spring.io/spring-boot/docs/current/reference/html/deployment-install.html
and to expose a basic `prometheus` metrics on the `app` servers using https://moelholm.com/2017/02/06/spring-boot-prometheus-actuator-endpoint/.
This is a bad practice, but it's fine for the purpose of this tutorial, instead of forking https://github.com/spring-guides.
Exposing `spring-boot` metrics to `prometheus` is very experimental, but an effort in this direction has been noticed by the developers of `spring-boot`
https://github.com/spring-projects/spring-boot/issues/8130.

Due to the fact that all VMs share a single private network, the individual `app` servers, as well as the `lb` (load-balancers) can be queried from the `mgt` server:

        $ vagrant ssh mgt1.mydomain -c "curl app1:8080"
        $ vagrant ssh mgt1.mydomain -c "curl lb1:80"

Let's deploy a different (https://github.com/spring-guides/gs-testing-web) spring boot application to a subset of `app` nodes.
First, modify the `ansible` playbook to point to the new application:

        $ cp -pf ansible/playbook.yml ansible/playbook.yml.rollback
        $ sed -i 's/gs-spring-boot/gs-testing-web/' ansible/playbook.yml

then run the playbook on the subset of nodes (here just a single node `app1.mydomain`), disabling the old service first:

        $ vagrant ssh mgt1.mydomain -c "ansible app1.mydomain -i /vagrant/ansible/hosts.yml -a 'systemctl stop gs-spring-boot' --become"
        $ vagrant ssh mgt1.mydomain -c "ansible app1.mydomain -i /vagrant/ansible/hosts.yml -a 'systemctl disable gs-spring-boot' --become"
        $ vagrant ssh mgt1.mydomain -c "ansible-playbook -i /vagrant/ansible/hosts.yml /vagrant/ansible/playbook.yml -l app1.mydomain"
        $ sleep 30

Verify the new application (which prints "Hello World") is deployed on the given node:

        $ vagrant ssh mgt1.mydomain -c "curl app1:8080" | grep "Hello World"

One could alternatively completely destroy the given `app` server and make a fresh deployment. This time using the rollback playbook:

        $ vagrant destroy -f app1.mydomain
        $ vagrant up app1.mydomain
        $ vagrant ssh mgt1.mydomain -c "ansible-playbook -i /vagrant/ansible/hosts.yml /vagrant/ansible/playbook.yml.rollback"
        $ sleep 30

Verify the old application (which prints "Greetings from Spring Boot!") is deployed on the given node:

        $ vagrant ssh mgt1.mydomain -c "curl app1:8080" | grep "Greetings from Spring Boot!"

Let's simulate a failure of the active load-balancer to see whether the service continues to be available.

First discover which of the `lb` servers actively holds the Virtual IP, and destroy that VM:

        $ ACTIVE_LB=`vagrant ssh lb1.mydomain -c "sudo su - -c \"pcs status | grep virtual-ip | cut -d: -f5 | cut -d' ' -f2 | tr -d '\n'\""`
        $ if `echo $ACTIVE_LB | grep -q lb1`; then PASSIVE_LB=lb2.mydomain; else PASSIVE_LB=lb1.mydomain; fi
        $ vagrant destroy -f $ACTIVE_LB

Verify that the loadbalancer has been moved to the remaining `lb` VM and the application is still available:

        $ vagrant ssh $PASSIVE_LB -c "sudo su - -c 'pcs status'"
        $ vagrant ssh mgt1.mydomain -c "curl loadbalancer:80"

Bring back, and configure the failed load-balancer. We need to copy the `/etc/corosync/corosync.conf` otherwise the `ansible` pacemaker
module will try to `pcs cluster setup` again (it's a missing logic in the `ansible` module). After copying the file one has to handle the SELinux context.

        $ vagrant up $ACTIVE_LB
        $ vagrant ssh $PASSIVE_LB -c "scp -o StrictHostKeyChecking=no -p /etc/corosync/corosync.conf $ACTIVE_LB:/tmp"
        $ vagrant ssh $ACTIVE_LB -c "sudo su - -c 'mkdir /etc/corosync&& mv /tmp/corosync.conf /etc/corosync&& chown root.root /etc/corosync/corosync.conf'"
        $ vagrant ssh $ACTIVE_LB -c "sudo su - -c 'semanage fcontext -a -s system_u -t etc_t /etc/corosync/corosync.conf'"

        $ vagrant ssh mgt1.mydomain -c "ansible-playbook -i /vagrant/ansible/hosts.yml /vagrant/ansible/playbook.yml.rollback"
        $ vagrant ssh $ACTIVE_LB -c "sudo su - -c 'pcs status'"

A note about STONITH (Shoot The Other Node In The Head): the application is stateless and the only resource managed by
the `pacemaker`/`corosync` cluster is the assignment of the Virtual IP to one of the `lb` servers.
In case where `corosync` detects a failure of the active `lb` node and `pacemaker` moves the Virtual IP to the other `lb` server
there is no risk of any data corruption due to the failed `lb` server being still alive.

When done, destroy the test machines with:

        $ vagrant destroy -f

------------
Dependencies
------------

https://github.com/vagrant-landrush/landrush

-------
License
-------

BSD 2-clause

----
Todo
----

1. few of the `ansible` modules are still used in a non-idempotent way.

2. the template files created by ansible-galaxy init should be edited.

3. there seem to be problems with the pacemaker ansible module: https://github.com/styopa/ansible-pacemaker/pull/8 https://github.com/styopa/ansible-pacemaker/issues/9 causing random failures

--------
Problems
--------