text:
using System.Diagnostics;
using System.Globalization;
using System.Runtime.InteropServices;
using System.Text;

namespace System.Data
{
    internal static class LocaleMapper
    {
        private const string ApiWinCoreLocalization = "api-ms-win-core-localization-l1-2-0.dll";
        private const string ApiWinCoreLocalizationObsolete = "api-ms-win-core-localization-obsolete-l1-2-0.dll";

        private const int LOCALE_NAME_MAX_LENGTH = 85;
        private const int ERROR_INVALID_PARAMETER = 0x57;
        private const uint LOCALE_IDEFAULTANSICODEPAGE = 0x00001004;
        private const uint LOCALE_RETURN_NUMBER = 0x20000000;

        [DllImport(ApiWinCoreLocalization, SetLastError = true, CharSet = CharSet.Unicode)]
        private static extern int GetLocaleInfoEx(string lpLocaleName, uint LCType, [Out] out uint lpLCData, int cchData);

        [DllImport(ApiWinCoreLocalizationObsolete, SetLastError = true, CharSet = CharSet.Unicode)]
        private static extern int LCIDToLocaleName(uint Locale, StringBuilder lpName, int cchName, int dwFlags);

        [DllImport(ApiWinCoreLocalization, SetLastError = true, CharSet = CharSet.Unicode)]
        private static extern uint LocaleNameToLCID(string lpName, uint dwFlags);

        public static string LcidToLocaleNameInternal(int lcid)
        {
            StringBuilder localName = new StringBuilder(LOCALE_NAME_MAX_LENGTH);

            if (LCIDToLocaleName(unchecked((uint)lcid), localName, localName.Capacity, 0) != 0)
            {
                return localName.ToString();
            }
            else
            {
                // LCIDToLocaleName should not return any other errors if we have sufficient buffer space
                int win32Error = Marshal.GetLastWin32Error();
                Debug.Assert(win32Error == ERROR_INVALID_PARAMETER, string.Format("Unknown error returned by LCIDToLocaleName: {0}. Lcid: {1}", win32Error, lcid));

                throw new CultureNotFoundException("lcid", lcid.ToString(), message: null);
            }
        }

        public static int LocaleNameToAnsiCodePage(string localeName)
        {
            uint ansiCodePage;
            if (GetLocaleInfoEx(localeName, LOCALE_RETURN_NUMBER | LOCALE_IDEFAULTANSICODEPAGE, out ansiCodePage, sizeof(uint)) != 0)
            {
                return unchecked((int)ansiCodePage);
            }
            else
            {
                // GetLocaleInfoEx should not return any other errors if we have sufficient buffer space
                int win32Error = Marshal.GetLastWin32Error();
                Debug.Assert(win32Error == ERROR_INVALID_PARAMETER, string.Format("Unknown error returned by GetLocaleInfoEx: {0}. LocaleName: {1}", win32Error, localeName));

                throw new CultureNotFoundException("localeName", localeName, message: null);
            }
        }

        public static int GetLcidForLocaleName(string localeName)
        {
            Debug.Assert(localeName != null, "Locale name should never be null");

            uint lcid = LocaleNameToLCID(localeName, 0);
            if (lcid != 0)
            {
                return unchecked((int)lcid);
            }
            else
            {
                // LocaleNameToLCID should not return any other errors
                int win32Error = Marshal.GetLastWin32Error();
                Debug.Assert(win32Error == ERROR_INVALID_PARAMETER, string.Format("Unknown error returned by LocaleNameToLCID: {0}. LocaleName: {1}", win32Error, localeName));

                throw new CultureNotFoundException("localeName", localeName, message: null);
            }
        }

    }
}


text:
/*
  Loki IndexedDb Adapter (need to include this script to use it)

  Console Usage can be used for management/diagnostic, here are a few examples :
  adapter.getDatabaseList(); // with no callback passed, this method will log results to console
  adapter.saveDatabase('UserDatabase', JSON.stringify(myDb));
  adapter.loadDatabase('UserDatabase'); // will log the serialized db to console
  adapter.deleteDatabase('UserDatabase');
*/

(function (root, factory) {
    if (typeof define === 'function' && define.amd) {
        // AMD
        define([], factory);
    } else if (typeof exports === 'object') {
        // Node, CommonJS-like
        module.exports = factory();
    } else {
        // Browser globals (root is window)
        root.LokiIndexedAdapter = factory();
    }
}(this, function () {
  return (function() {

    /**
     * Loki persistence adapter class for indexedDb.
     *     This class fulfills abstract adapter interface which can be applied to other storage methods.
     *     Utilizes the included LokiCatalog app/key/value database for actual database persistence.
     *     Indexeddb is highly async, but this adapter has been made 'console-friendly' as well.
     *     Anywhere a callback is omitted, it should return results (if applicable) to console.
     *     IndexedDb storage is provided per-domain, so we implement app/key/value database to
     *     allow separate contexts for separate apps within a domain.
     *
     * @example
     * var idbAdapter = new LokiIndexedAdapter('finance');
     *
     * @constructor LokiIndexedAdapter
     *
     * @param {string} appname - (Optional) Application name context can be used to distinguish subdomains, 'loki' by default
     * @param {object=} options Configuration options for the adapter
     * @param {boolean} options.closeAfterSave Whether the indexedDB database should be closed after saving.
     */
    function LokiIndexedAdapter(appname, options)
    {
      this.app = 'loki';
      this.options = options || {};

      if (typeof (appname) !== 'undefined')
      {
        this.app = appname;
      }

      // keep reference to catalog class for base AKV operations
      this.catalog = null;

      if (!this.checkAvailability()) {
        throw new Error('indexedDB does not seem to be supported for your environment');
      }
    }

    /**
     * Used for closing the indexeddb database.
     */
    LokiIndexedAdapter.prototype.closeDatabase = function ()
    {
      if (this.catalog && this.catalog.db) {
        this.catalog.db.close();
        this.catalog.db = null;
      }
    };

    /**
     * Used to check if adapter is available
     *
     * @returns {boolean} true if indexeddb is available, false if not.
     * @memberof LokiIndexedAdapter
     */
    LokiIndexedAdapter.prototype.checkAvailability = function()
    {
      if (typeof indexedDB !== 'undefined' && indexedDB) return true;

      return false;
    };

    /**
     * Retrieves a serialized db string from the catalog.
     *
     * @example
     * // LOAD
     * var idbAdapter = new LokiIndexedAdapter('finance');
     * var db = new loki('test', { adapter: idbAdapter });
     *   db.loadDatabase(function(result) {
     *   console.log('done');
     * });
     *
     * @param {string} dbname - the name of the database to retrieve.
     * @param {function} callback - callback should accept string param containing serialized db string.
     * @memberof LokiIndexedAdapter
     */
    LokiIndexedAdapter.prototype.loadDatabase = function(dbname, callback)
    {
      var appName = this.app;
      var adapter = this;

      // lazy open/create db reference so dont -need- callback in constructor
      if (this.catalog === null || this.catalog.db === null) {
        this.catalog = new LokiCatalog(function(cat) {
          adapter.catalog = cat;

          adapter.loadDatabase(dbname, callback);
        });

        return;
      }

      // lookup up db string in AKV db
      this.catalog.getAppKey(appName, dbname, function(result) {
        if (typeof (callback) === 'function') {
          if (result.id === 0) {
            callback(null);
            return;
          }
          callback(result.val);
        }
        else {
          // support console use of api
          console.log(result.val);
        }
      });
    };

    // alias
    LokiIndexedAdapter.prototype.loadKey = LokiIndexedAdapter.prototype.loadDatabase;

    /**
     * Saves a serialized db to the catalog.
     *
     * @example
     * // SAVE : will save App/Key/Val as 'finance'/'test'/{serializedDb}
     * var idbAdapter = new LokiIndexedAdapter('finance');
     * var db = new loki('test', { adapter: idbAdapter });
     * var coll = db.addCollection('testColl');
     * coll.insert({test: 'val'});
     * db.saveDatabase();  // could pass callback if needed for async complete
     *
     * @param {string} dbname - the name to give the serialized database within the catalog.
     * @param {string} dbstring - the serialized db string to save.
     * @param {function} callback - (Optional) callback passed obj.success with true or false
     * @memberof LokiIndexedAdapter
     */
    LokiIndexedAdapter.prototype.saveDatabase = function(dbname, dbstring, callback)
    {
      var appName = this.app;
      var adapter = this;

      function saveCallback(result) {
        if (result && result.success === true) {
          callback(null);
        }
        else {
          callback(new Error("Error saving database"));
        }

        if (adapter.options.closeAfterSave) {
          adapter.closeDatabase();
        }
      }

      // lazy open/create db reference so dont -need- callback in constructor
      if (this.catalog === null || this.catalog.db === null) {
        this.catalog = new LokiCatalog(function(cat) {
          adapter.saveDatabase(dbname, dbstring, saveCallback);
        });

        return;
      }

      // set (add/update) entry to AKV database
      this.catalog.setAppKey(appName, dbname, dbstring, saveCallback);
    };

    // alias
    LokiIndexedAdapter.prototype.saveKey = LokiIndexedAdapter.prototype.saveDatabase;

    /**
     * Deletes a serialized db from the catalog.
     *
     * @example
     * // DELETE DATABASE
     * // delete 'finance'/'test' value from catalog
     * idbAdapter.deleteDatabase('test', function {
     *   // database deleted
     * });
     *
     * @param {string} dbname - the name of the database to delete from the catalog.
     * @param {function=} callback - (Optional) executed on database delete
     * @memberof LokiIndexedAdapter
     */
    LokiIndexedAdapter.prototype.deleteDatabase = function(dbname, callback)
    {
      var appName = this.app;
      var adapter = this;

      // lazy open/create db reference and pass callback ahead
      if (this.catalog === null || this.catalog.db === null) {
        this.catalog = new LokiCatalog(function(cat) {
          adapter.catalog = cat;

          adapter.deleteDatabase(dbname, callback);
        });

        return;
      }

      // catalog was already initialized, so just lookup object and delete by id
      this.catalog.getAppKey(appName, dbname, function(result) {
        var id = result.id;

        if (id !== 0) {
          adapter.catalog.deleteAppKey(id, callback);
        } else if (typeof (callback) === 'function') {
          callback({ success: true });
        }
      });
    };

    // alias
    LokiIndexedAdapter.prototype.deleteKey = LokiIndexedAdapter.prototype.deleteDatabase;

    /**
     * Removes all database partitions and pages with the base filename passed in.
     * This utility method does not (yet) guarantee async deletions will be completed before returning
     *
     * @param {string} dbname - the base filename which container, partitions, or pages are derived
     * @memberof LokiIndexedAdapter
     */
    LokiIndexedAdapter.prototype.deleteDatabasePartitions = function(dbname) {
      var self=this;
      this.getDatabaseList(function(result) {
        result.forEach(function(str) {
          if (str.startsWith(dbname)) {
            self.deleteDatabase(str);
          }
        });
      });
    };

    /**
     * Retrieves object array of catalog entries for current app.
     *
     * @example
     * idbAdapter.getDatabaseList(function(result) {
     *   // result is array of string names for that appcontext ('finance')
     *   result.forEach(function(str) {
     *     console.log(str);
     *   });
     * });
     *
     * @param {function} callback - should accept array of database names in the catalog for current app.
     * @memberof LokiIndexedAdapter
     */
    LokiIndexedAdapter.prototype.getDatabaseList = function(callback)
    {
      var appName = this.app;
      var adapter = this;

      // lazy open/create db reference so dont -need- callback in constructor
      if (this.catalog === null || this.catalog.db === null) {
        this.catalog = new LokiCatalog(function(cat) {
          adapter.catalog = cat;

          adapter.getDatabaseList(callback);
        });

        return;
      }

      // catalog already initialized
      // get all keys for current appName, and transpose results so just string array
      this.catalog.getAppKeys(appName, function(results) {
        var names = [];

        for(var idx = 0; idx < results.length; idx++) {
          names.push(results[idx].key);
        }

        if (typeof (callback) === 'function') {
          callback(names);
        }
        else {
          names.forEach(function(obj) {
            console.log(obj);
          });
        }
      });
    };

    // alias
    LokiIndexedAdapter.prototype.getKeyList = LokiIndexedAdapter.prototype.getDatabaseList;

    /**
     * Allows retrieval of list of all keys in catalog along with size
     *
     * @param {function} callback - (Optional) callback to accept result array.
     * @memberof LokiIndexedAdapter
     */
    LokiIndexedAdapter.prototype.getCatalogSummary = function(callback)
    {
      var appName = this.app;
      var adapter = this;

      // lazy open/create db reference
      if (this.catalog === null || this.catalog.db === null) {
        this.catalog = new LokiCatalog(function(cat) {
          adapter.catalog = cat;

          adapter.getCatalogSummary(callback);
        });

        return;
      }

      // catalog already initialized
      // get all keys for current appName, and transpose results so just string array
      this.catalog.getAllKeys(function(results) {
        var entries = [];
        var obj,
          size,
          oapp,
          okey,
          oval;

        for(var idx = 0; idx < results.length; idx++) {
          obj = results[idx];
          oapp = obj.app || '';
          okey = obj.key || '';
          oval = obj.val || '';

          // app and key are composited into an appkey column so we will mult by 2
          size = oapp.length * 2 + okey.length * 2 + oval.length + 1;

          entries.push({ "app": obj.app, "key": obj.key, "size": size });
        }

        if (typeof (callback) === 'function') {
          callback(entries);
        }
        else {
          entries.forEach(function(obj) {
            console.log(obj);
          });
        }
      });
    };

    /**
     * LokiCatalog - underlying App/Key/Value catalog persistence
     *    This non-interface class implements the actual persistence.
     *    Used by the IndexedAdapter class.
     */
    function LokiCatalog(callback)
    {
      this.db = null;
      this.initializeLokiCatalog(callback);
    }

    LokiCatalog.prototype.initializeLokiCatalog = function(callback) {
      var openRequest = indexedDB.open('LokiCatalog', 1);
      var cat = this;

      // If database doesn't exist yet or its version is lower than our version specified above (2nd param in line above)
      openRequest.onupgradeneeded = function(e) {
        var thisDB = e.target.result;
        if (thisDB.objectStoreNames.contains('LokiAKV')) {
          thisDB.deleteObjectStore('LokiAKV');
        }

        if(!thisDB.objectStoreNames.contains('LokiAKV')) {
          var objectStore = thisDB.createObjectStore('LokiAKV', { keyPath: 'id', autoIncrement:true });
          objectStore.createIndex('app', 'app', {unique:false});
          objectStore.createIndex('key', 'key', {unique:false});
          // hack to simulate composite key since overhead is low (main size should be in val field)
          // user (me) required to duplicate the app and key into comma delimited appkey field off object
          // This will allow retrieving single record with that composite key as well as
          // still supporting opening cursors on app or key alone
          objectStore.createIndex('appkey', 'appkey', {unique:true});
        }
      };

      openRequest.onsuccess = function(e) {
        cat.db = e.target.result;

        if (typeof (callback) === 'function') callback(cat);
      };

      openRequest.onerror = function(e) {
        throw e;
      };
    };

    LokiCatalog.prototype.getAppKey = function(app, key, callback) {
      var transaction = this.db.transaction(['LokiAKV'], 'readonly');
      var store = transaction.objectStore('LokiAKV');
      var index = store.index('appkey');
      var appkey = app + "," + key;
      var request = index.get(appkey);

      request.onsuccess = (function(usercallback) {
        return function(e) {
          var lres = e.target.result;

          if (lres === null || typeof(lres) === 'undefined') {
            lres = {
              id: 0,
              success: false
            };
          }

          if (typeof(usercallback) === 'function') {
            usercallback(lres);
          }
          else {
            console.log(lres);
          }
        };
      })(callback);

      request.onerror = (function(usercallback) {
        return function(e) {
          if (typeof(usercallback) === 'function') {
            usercallback({ id: 0, success: false });
          }
          else {
            throw e;
          }
        };
      })(callback);
    };

    LokiCatalog.prototype.getAppKeyById = function (id, callback, data) {
      var transaction = this.db.transaction(['LokiAKV'], 'readonly');
      var store = transaction.objectStore('LokiAKV');
      var request = store.get(id);

      request.onsuccess = (function(data, usercallback){
        return function(e) {
          if (typeof(usercallback) === 'function') {
            usercallback(e.target.result, data);
          }
          else {
            console.log(e.target.result);
          }
        };
      })(data, callback);
    };

    LokiCatalog.prototype.setAppKey = function (app, key, val, callback) {
      var transaction = this.db.transaction(['LokiAKV'], 'readwrite');
      var store = transaction.objectStore('LokiAKV');
      var index = store.index('appkey');
      var appkey = app + "," + key;
      var request = index.get(appkey);

      // first try to retrieve an existing object by that key
      // need to do this because to update an object you need to have id in object, otherwise it will append id with new autocounter and clash the unique index appkey
      request.onsuccess = function(e) {
        var res = e.target.result;

        if (res === null || res === undefined) {
          res = {
            app:app,
            key:key,
            appkey: app + ',' + key,
            val:val
          };
        }
        else {
          res.val = val;
        }

        var requestPut = store.put(res);

        requestPut.onerror = (function(usercallback) {
          return function(e) {
            if (typeof(usercallback) === 'function') {
              usercallback({ success: false });
            }
            else {
              console.error('LokiCatalog.setAppKey (set) onerror');
              console.error(request.error);
            }
          };

        })(callback);

        requestPut.onsuccess = (function(usercallback) {
          return function(e) {
            if (typeof(usercallback) === 'function') {
              usercallback({ success: true });
            }
          };
        })(callback);
      };

      request.onerror = (function(usercallback) {
        return function(e) {
          if (typeof(usercallback) === 'function') {
            usercallback({ success: false });
          }
          else {
            console.error('LokiCatalog.setAppKey (get) onerror');
            console.error(request.error);
          }
        };
      })(callback);
    };

    LokiCatalog.prototype.deleteAppKey = function (id, callback) {
      var transaction = this.db.transaction(['LokiAKV'], 'readwrite');
      var store = transaction.objectStore('LokiAKV');
      var request = store.delete(id);

      request.onsuccess = (function(usercallback) {
        return function(evt) {
          if (typeof(usercallback) === 'function') usercallback({ success: true });
        };
      })(callback);

      request.onerror = (function(usercallback) {
        return function(evt) {
          if (typeof(usercallback) === 'function') {
            usercallback({ success: false });
          }
          else {
            console.error('LokiCatalog.deleteAppKey raised onerror');
            console.error(request.error);
          }
        };
      })(callback);
    };

    LokiCatalog.prototype.getAppKeys = function(app, callback) {
      var transaction = this.db.transaction(['LokiAKV'], 'readonly');
      var store = transaction.objectStore('LokiAKV');
      var index = store.index('app');

      // We want cursor to all values matching our (single) app param
      var singleKeyRange = IDBKeyRange.only(app);

      // To use one of the key ranges, pass it in as the first argument of openCursor()/openKeyCursor()
      var cursor = index.openCursor(singleKeyRange);

      // cursor internally, pushing results into this.data[] and return
      // this.data[] when done (similar to service)
      var localdata = [];

      cursor.onsuccess = (function(data, callback) {
        return function(e) {
          var cursor = e.target.result;
          if (cursor) {
            var currObject = cursor.value;

            data.push(currObject);

            cursor.continue();
          }
          else {
            if (typeof(callback) === 'function') {
              callback(data);
            }
            else {
              console.log(data);
            }
          }
        };
      })(localdata, callback);

      cursor.onerror = (function(usercallback) {
        return function(e) {
          if (typeof(usercallback) === 'function') {
            usercallback(null);
          }
          else {
            console.error('LokiCatalog.getAppKeys raised onerror');
            console.error(e);
          }
        };
      })(callback);

    };

    // Hide 'cursoring' and return array of { id: id, key: key }
    LokiCatalog.prototype.getAllKeys = function (callback) {
      var transaction = this.db.transaction(['LokiAKV'], 'readonly');
      var store = transaction.objectStore('LokiAKV');
      var cursor = store.openCursor();

      var localdata = [];

      cursor.onsuccess = (function(data, callback) {
        return function(e) {
          var cursor = e.target.result;
          if (cursor) {
            var currObject = cursor.value;

            data.push(currObject);

            cursor.continue();
          }
          else {
            if (typeof(callback) === 'function') {
              callback(data);
            }
            else {
              console.log(data);
            }
          }
        };
      })(localdata, callback);

      cursor.onerror = (function(usercallback) {
        return function(e) {
          if (typeof(usercallback) === 'function') usercallback(null);
        };
      })(callback);

    };

    return LokiIndexedAdapter;

  }());
}));

redpajama_set_name: RedPajamaGithub


text:
package ch.epfl.bbp.uima.pdf.cleanup;

import static ch.epfl.bbp.StringUtils.print;

import java.io.LineNumberReader;
import java.io.StringReader;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * This class allows to decide whether an hyphen should be removed from a line.
 *
 *
Rules

 *

 *
At the end of lines, there should be at least 3 characters behind and 2
 * forward

 *
Ignore if numbers? (for the moment yes)

 *
Ignore if capital letters forward? (for the moment no)

 *

 *
Comments

 *

 *
First rule not respected in PDFs, so cannot be applied (sticked to: 2
 * behind and 2 forward)

 *
Be careful to remove ligatures before, because not in [A-Za-z]

 *
Accents are not considered (in an other code, range [A-Za-z\300-\377])

 *

 *
 *
Possible hyphens in English (see Unicode, chapter 6,
 * Table 6-3 and http://en.wikipedia.org/wiki/Hyphens#Unicode)

 *

 *
U+002D HYPHEN-MINUS (mainly used)

 *
U+00AD SOFT HYPHEN

 *
U+2010 ‐ hyphen (HTML: ‐)

 *

 *
 * @author orianne.rollier@epfl.ch
 * @author renaud.richardet@epfl.ch
 */
public class HyphenRemover {
    private static Logger LOG = LoggerFactory.getLogger(HyphenRemover.class);

    /** Matches a dash (all kinds, see above) at the end of a line */
    public static final Pattern patternHyphen = Pattern
            .compile("[\\u002D\\u00AD\\u2010]\\s*$");

    private static final String WORD_SEPARATOR = "\\W+";
    private static final Pattern numberEnd = Pattern.compile("[0-9]-$");
    private static final Pattern numberStart = Pattern.compile("^[0-9]");
    private static final Pattern greekEnd = Pattern.compile("\\p{InGreek}-$");
    private static final Pattern greekStart = Pattern.compile("^\\p{InGreek}+");

    /**
     * Decides whether a word should be dehyphenated
     *
     * @param w1
     *            word behind
     * @param w2
     *            word forward
     * @return returns true if the word should be dehyphenated
     */
    private static boolean shouldDehyphenate(String w1, String w2) {

        boolean dehyphenate = true;
        String rule = "DH"; // choice by default, dehyphenate

        if (w1.length() <= 1) {
            // w1 has only one char
            dehyphenate = false;
            rule = "N11";
        } else if (w2.length() <= 1) {
            // w2 has only one char
            dehyphenate = false;
            rule = "N12";
        } else if (numberEnd.matcher(w1).find()) {
            // number at the end of w1
            dehyphenate = false;
            rule = "N21";
        } else if (numberStart.matcher(w2).find()) {
            // number at the beginning of w2
            dehyphenate = false;
            rule = "N22";
        } else if (greekEnd.matcher(w1).find()) {
            // greek letter at the end of w1
            dehyphenate = false;
            rule = "N31";
        } else if (greekStart.matcher(w2).find()) {
            // greek letter at the beginning of w2
            dehyphenate = false;
            rule = "N32";
        }

        if (dehyphenate) {
            String word = w1.substring(0, w1.length() - 1) + w2;
            LOG.debug("\"" + w1 + "\",\"" + w2 + "\",\"" + rule + "\",\""
                    + word + "\"\n");
        } else {
            LOG.debug("\"" + w1 + "\",\"" + w2 + "\",\"" + rule + "\",\"" + w1
                    + w2 + "\"\n");
        }
        return dehyphenate;
    }

    /** Removes ligatures, multiple spaces and hypens from a text file */
    public static String dehyphenate(LineNumberReader reader, String docId) {

        String line;
        StringBuilder sb = new StringBuilder();

        try {
            while ((line = reader.readLine()) != null) {
                String lineOut = line;
                Matcher matcher = patternHyphen.matcher(line);

                // Allows to append consecutive lines with hyphens
                while (matcher.find() && ((line = reader.readLine()) != null)) {
                    String behind[] = lineOut.split(WORD_SEPARATOR);
                    String forward[] = line.trim().split(WORD_SEPARATOR);

                    if (behind.length == 0) {
                        lineOut = lineOut + "\n" + line;
                        continue;
                    }

                    String w1 = behind[(behind.length) - 1];
                    String w2 = forward[0];
                    if (shouldDehyphenate(w1, w2)) {
                        // Remove hyphen from first line and concatenates second
                        lineOut = lineOut.substring(0, lineOut.length() - 1)
                                .trim() + line.trim();
                    } else {
                        // just concanates the two lines
                        lineOut = lineOut + line;
                    }
                    matcher = patternHyphen.matcher(lineOut);
                }

                sb.append(lineOut.trim());
                sb.append("\n");
            }
        } catch (Throwable t) {
            //TODO happens sometimes, e.g. docId 1279669
            LOG.warn("failed to dehyphenate, docId " + docId, print(t));
        }
        return sb.toString();
    }

    /** Removes ligatures, multiple spaces and hypens from a text file */
    public static String dehyphenate(String text, String docId) {
        return dehyphenate(new LineNumberReader(new StringReader(text)), docId);
    }
}


text:
/* See LICENSE for licensing information */

/**
 * \file pubsub.h
 * \brief Macros to implement publish/subscribe abstractions.
 *
 * To use these macros, call DECLARE_PUBSUB_TOPIC() with an identifier to use
 * as your topic.  Below, I'm going to assume you say DECLARE_PUBSUB_TOPIC(T).
 *
 * Doing this will declare the following types:
 *   typedef struct T_event_data_t T_event_data_t; // you define this struct
 *   typedef struct T_subscriber_data_t T_subscriber_data_t; // this one too.
 *   typedef struct T_subscriber_t T_subscriber_t; // opaque
 *   typedef int (*T_subscriber_fn_t)(T_event_data_t*, T_subscriber_data_t*);
 *
 * and it will declare the following functions:
 *     const T_subscriber_t *T_subscribe(T_subscriber_fn_t,
 *                                       T_subscriber_data_t *,
 *                                       unsigned flags,
 *                                       unsigned priority);
 *     int T_unsubscribe(const T_subscriber_t *)
 *
 * Elsewhere you can say DECLARE_NOTIFY_PUBSUB_TOPIC(static, T), which
 * declares:
 *
 *    static int T_notify(T_event_data_t *, unsigned notify_flags);
 *    static void T_clear(void);
 *
 * And in some C file, you would define these functions with:
 *    IMPLEMENT_PUBSUB_TOPIC(static, T).
 *
 * The implementations will be small typesafe wrappers over generic versions
 * of the above functions.
 *
 * To use the typesafe functions, you add any number of subscribers with
 * T_subscribe().  Each has an associated function pointer, data pointer,
 * and priority. Later, you can invoke T_notify() to declare that the
 * event has occurred. Each of the subscribers will be invoked once.
 **/

#ifndef TOR_PUBSUB_H
#define TOR_PUBSUB_H

#include "torint.h"

/**
 * Flag for T_subscribe: die with an assertion failure if the event
 * have ever been published before.  Used when a subscriber must absolutely
 * never have missed an event.
 */
#define SUBSCRIBE_ATSTART (1u<<0)

#define DECLARE_PUBSUB_STRUCT_TYPES(name)                               \
  /* You define this type. */                                           \
  typedef struct name ## _event_data_t name ## _event_data_t;           \
  /* You define this type. */                                           \
  typedef struct name ## _subscriber_data_t name ## _subscriber_data_t;

#define DECLARE_PUBSUB_TOPIC(name)                                      \
  /* This type is opaque. */                                            \
  typedef struct name ## _subscriber_t name ## _subscriber_t;           \
  /* You declare functions matching this type. */                       \
  typedef int (*name ## _subscriber_fn_t)(                              \
                                    name ## _event_data_t *data,        \
                                    name ## _subscriber_data_t *extra); \
  /* Call this function to subscribe to a topic. */                     \
  const name ## _subscriber_t *name ## _subscribe(                      \
                         name##_subscriber_fn_t subscriber,             \
                         name##_subscriber_data_t *extra_data,          \
                         unsigned flags,                                \
                         unsigned priority);                            \
  /* Call this function to unsubscribe from a topic. */                 \
  int name ## _unsubscribe(const name##_subscriber_t *s);

#define DECLARE_NOTIFY_PUBSUB_TOPIC(linkage, name)                          \
  /* Call this function to notify all subscribers. Flags not yet used. */   \
  linkage int name ## _notify(name ## _event_data_t *data, unsigned flags); \
  /* Call this function to release storage held by the topic. */            \
  linkage void name ## _clear(void);

/**
 * Type used to hold a generic function for a subscriber.
 *
 * [Yes, it is safe to cast to this, so long as we cast back to the original
 * type before calling.  From C99: "A pointer to a function of one type may be
 * converted to a pointer to a function of another type and back again; the
 * result shall compare equal to the original pointer."]
*/
typedef int (*pubsub_subscriber_fn_t)(void *, void *);

/**
 * Helper type to implement pubsub abstraction. Don't use this directly.
 * It represents a subscriber.
 */
typedef struct pubsub_subscriber_t {
  /** Function to invoke when the event triggers. */
  pubsub_subscriber_fn_t fn;
  /** Data associated with this subscriber. */
  void *subscriber_data;
  /** Priority for this subscriber. Low priorities happen first. */
  unsigned priority;
  /** Flags set on this subscriber. Not yet used.*/
  unsigned subscriber_flags;
} pubsub_subscriber_t;

/**
 * Helper type to implement pubsub abstraction. Don't use this directly.
 * It represents a topic, and keeps a record of subscribers.
 */
typedef struct pubsub_topic_t {
  /** List of subscribers to this topic. May be NULL. */
  struct smartlist_t *subscribers;
  /** Total number of times that pubsub_notify_() has ever been called on this
   * topic. */
  uint64_t n_events_fired;
  /** True iff we're running 'notify' on this topic, and shouldn't allow
   * any concurrent modifications or events. */
  unsigned locked;
} pubsub_topic_t;

const pubsub_subscriber_t *pubsub_subscribe_(pubsub_topic_t *topic,
                                             pubsub_subscriber_fn_t fn,
                                             void *subscriber_data,
                                             unsigned subscribe_flags,
                                             unsigned priority);
int pubsub_unsubscribe_(pubsub_topic_t *topic, const pubsub_subscriber_t *sub);
void pubsub_clear_(pubsub_topic_t *topic);
typedef int (*pubsub_notify_fn_t)(pubsub_subscriber_t *subscriber,
                                  void *notify_data);
int pubsub_notify_(pubsub_topic_t *topic, pubsub_notify_fn_t notify_fn,
                   void *notify_data, unsigned notify_flags);

#define IMPLEMENT_PUBSUB_TOPIC(notify_linkage, name)                    \
  static pubsub_topic_t name ## _topic_ = { NULL, 0, 0 };               \
  const name ## _subscriber_t *                                         \
  name ## _subscribe(name##_subscriber_fn_t subscriber,                 \
                     name##_subscriber_data_t *extra_data,              \
                     unsigned flags,                                    \
                     unsigned priority)                                 \
  {                                                                     \
    const pubsub_subscriber_t *s;                                       \
    s = pubsub_subscribe_(&name##_topic_,                               \
                          (pubsub_subscriber_fn_t)subscriber,           \
                          extra_data,                                   \
                          flags,                                        \
                          priority);                                    \
    return (const name##_subscriber_t *)s;                              \
  }                                                                     \
  int                                                                   \
  name ## _unsubscribe(const name##_subscriber_t *subscriber)           \
  {                                                                     \
    return pubsub_unsubscribe_(&name##_topic_,                          \
                               (const pubsub_subscriber_t *)subscriber); \
  }                                                                     \
  static int                                                            \
  name##_call_the_notify_fn_(pubsub_subscriber_t *subscriber,           \
                             void *notify_data)                         \
  {                                                                     \
    name ## _subscriber_fn_t fn;                                        \
    fn = (name ## _subscriber_fn_t) subscriber->fn;                     \
    return fn(notify_data, subscriber->subscriber_data);                \
  }                                                                     \
  notify_linkage int                                                    \
  name ## _notify(name ## _event_data_t *event_data, unsigned flags)    \
  {                                                                     \
    return pubsub_notify_(&name##_topic_,                               \
                          name##_call_the_notify_fn_,                   \
                          event_data,                                   \
                          flags);                                       \
  }                                                                     \
  notify_linkage void                                                   \
  name ## _clear(void)                                                  \
  {                                                                     \
    pubsub_clear_(&name##_topic_);                                      \
  }

#endif /* !defined(TOR_PUBSUB_H) */


text:
package com.ashman.fivehundredpx.util;

import android.graphics.Bitmap;

import com.ashman.fivehundredpx.BaseFragmentActivity;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.nio.ByteBuffer;

public class Util {
    //Email regex taken from http://www.regular-expressions.info/email.html
    public static final String EMAIL_REGEX = "[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:" +
            "\\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0" +
            "-9])?\\.)+(?:[A-Z]{2}|com|org|net|edu|gov|mil|biz|info|mobi|nam" +
            "e|aero|asia|jobs|museum)\\b";

    public static File saveBitmapToLocalFile(String directory,
                                 String filename, Bitmap bitmap) {
        try {
            int size = bitmap.getRowBytes() * bitmap.getHeight();
            ByteBuffer buffer = ByteBuffer.allocate(size);
            bitmap.copyPixelsToBuffer(buffer);
            buffer.rewind();
            byte[] bytes = new byte[size];
            buffer.get(bytes, 0, bytes.length);

            File directoryFile = new File(directory);
            File tempFile = File.createTempFile(filename, null, directoryFile);

            FileOutputStream stream = new FileOutputStream(tempFile);
            stream.write(bytes, 0, bytes.length);
            stream.close();

            return tempFile;
        } catch (Exception e) {
            return null;
        }
    }

    public static Bitmap readBitmapFromLocalFile(String directory, int width, int height,
                                                 String bitmapConfigString, String filename) {
        try {
            File directoryFile = new File(directory);
            File file = new File(directoryFile, filename);
            FileInputStream stream = new FileInputStream(file);

            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            byte[] b = new byte[1024];
            int bytesRead;
            while ((bytesRead = stream.read(b)) != -1) {
                bos.write(b, 0, bytesRead);
            }
            byte[] bytes = bos.toByteArray();
            stream.close();
            bos.close();

            Bitmap.Config bitmapConfig = Bitmap.Config.valueOf(bitmapConfigString);
            Bitmap bitmap = Bitmap.createBitmap(width, height, bitmapConfig);
            ByteBuffer buffer = ByteBuffer.wrap(bytes);
            bitmap.copyPixelsFromBuffer(buffer);

            return bitmap;
        } catch (Exception e) {
            return null;
        }
    }

    public static File getLocalFile(BaseFragmentActivity activity, String filename) {
        try {
            File file = new File(activity.getBaseContext().getFilesDir(), filename);
            return file;
        } catch (Exception e) {
            return null;
        }

    }
}


text:
import os
COV = None
if os.environ.get('FLASK_COVERAGE'):
    import coverage
    COV = coverage.coverage(branch=True, include='app/*')
    COV.start()

from app import create_app, db
from app.models import Role, User, Post, Follow, Permission, Comment
from flask_script import Manager, Shell
from flask_migrate import Migrate, MigrateCommand

# 系统中不存在 FLASK_CONFIG 环境变量时,try 语句返回 create_app('default')
# 当存在 FLASK_CONFIG 环境变量,且值不属于 config 的键时,报 KeyError 错,此时通过 except 语句返回 create_app('default')
try:
    app = create_app(os.getenv('FLASK_CONFIG') or 'default')
except KeyError:
    app = create_app('default')

manager = Manager(app)
migrate = Migrate(app, db)

def make_shell_context():
    return dict(app=app, db=db, Role=Role, User=User, Post=Post, Follow=Follow, Permission=Permission, Comment=Comment)

manager.add_command('shell', Shell(make_context=make_shell_context))
manager.add_command('db', MigrateCommand)

@manager.command
def test(coverage=False):
    """Run the unit tests."""
    if coverage and not os.environ.get('FLASK_COVERAGE'):
        import sys
        os.environ['FLASK_COVERAGE'] = '1'
        os.execvp(sys.executable, [sys.executable] + sys.argv)
    import unittest
    tests = unittest.TestLoader().discover('tests')
    unittest.TextTestRunner(verbosity=2).run(tests)
    if COV:
        COV.stop()
        COV.save()
        print('Coverage Summary:')
        COV.report()
        basedir = os.path.abspath(os.path.dirname(__file__))
        covdir = os.path.join(basedir, 'tmp/coverage')
        COV.html_report(directory=covdir)
        print('HTML version: file://{}/index.html'.format(covdir))
        COV.erase()

@manager.command
def profile(length=10, profile_dir=None):
    """Start the application under the code profiler."""
    from werkzeug.contrib.profiler import ProfilerMiddleware
    app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[
                                      length], profile_dir=profile_dir)
    app.run()

@manager.command
def deploy():
    from flask_migrate import upgrade
    from app.models import Role, User

    upgrade()

    Role.insert_roles()
    User.add_self_follows()

if __name__ == '__main__':
    manager.run()


text:
package org.apache.druid.query;

import org.apache.druid.guice.annotations.ExtensionPoint;
import org.apache.druid.segment.Segment;

import java.util.concurrent.ExecutorService;

/**
 * An interface that defines the nitty gritty implementation details of a Query on a Segment
 */
@ExtensionPoint
public interface QueryRunnerFactory<T, QueryType extends Query<T>>
{
  /**
   * Given a specific segment, this method will create a {@link QueryRunner}.
   *
   * The {@link QueryRunner}, when asked, will generate a {@link org.apache.druid.java.util.common.guava.Sequence} of
   * results based on the given segment.  This is the meat of the {@link Query} processing and is where the results are
   * actually generated.  Everything else is just merging and reduction logic.
   *
   * @param   segment The segment to process
   * @return  A {@link QueryRunner} that, when asked, will generate a
   *          {@link org.apache.druid.java.util.common.guava.Sequence} of results based on the given segment
   */
  QueryRunner<T> createRunner(Segment segment);

  /**
   * Runners generated with createRunner() and combined into an Iterable in (time,shardId) order are passed
   * along to this method with an {@link ExecutorService}.  The method should then return a {@link QueryRunner} that,
   * when asked, will use the {@link ExecutorService} to run the base QueryRunners in some fashion.
   *
   * The vast majority of the time, this should be implemented with {@link ChainedExecutionQueryRunner}:
   *
   *     return new ChainedExecutionQueryRunner<>(queryExecutor, toolChest.getOrdering(), queryWatcher, queryRunners);
   *
   * Which will allow for parallel execution up to the maximum number of processing threads allowed.
   *
   * @param queryExecutor   {@link ExecutorService} to be used for parallel processing
   * @param queryRunners    Individual {@link QueryRunner} objects that produce some results
   * @return                a {@link QueryRunner} that, when asked, will use the {@link ExecutorService} to run the base
   *                        {@link QueryRunner} collection.
   */
  QueryRunner<T> mergeRunners(ExecutorService queryExecutor, Iterable<QueryRunner<T>> queryRunners);

  /**
   * Provides access to the {@link QueryToolChest} for this specific {@link Query} type.
   *
   * @return an instance of the toolchest for this specific query type.
   */
  QueryToolChest<T, QueryType> getToolchest();
}


text:
public class Person {

   private final String name;
   private final String profession;
   private final int birthYear;

   /**
    * Construtor de objectos da classe Person.
    * @param name Nome da pessoa
    * @param birthYear Ano de nascimento
    * @param profession Profissao
    */
   public Person(String name,int birthYear,String profession) {
      this.name = name;
      this.birthYear = birthYear;
      this.profession = profession;
   }

   /**
    *  Devolve o nome da pessoa
    */
   public String name() { return name; }

   /**
    *  Devolve o ano de nascimento da pessoa.
    */
   public int birthYear() { return birthYear; }

   /**
    *  Devolve a profissao da pessoa.
    */
   public String profession() { return profession; }
}


text:
<#
The MIT License (MIT)

Copyright (c) 2015 Objectivity Bespoke Software Specialists

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
#>

Import-Module -Name "$PSScriptRoot\..\..\..\PSCI.psd1"

Describe -Tag "PSCI.unit" "Get-UpdateKeyValueCmdParams" {
    InModuleScope PSCI {

        $testFileName = "$PSScriptRoot\Get-UpdateKeyValueCmdParams.test"

        function New-TestFile {
            Set-Content -Path $testFileName -Value @'
key1=value1
 key\2 = value2

[Section]
key3=value3
key4= value4
key5=
key6=value6
app.webservice.password=zzz
'@
        }

        Context "when supplied a file with multiple key=value lines" {

            It "should properly update the lines" {

                try {

                    New-TestFile

                    $configValues = @('key1=newValue1', 'key\2=newValue2', 'key3=value3', 'key5=c:\a\b', 'key6=', 'keyNew=newValue', 'app.webservice.password=123')

                    $params = Get-UpdateKeyValueCmdParams -ConfigFiles $testFileName -ConfigValues $configValues
                    $result = Invoke-Command @params
                    Write-Host $result

                    $content = [IO.File]::ReadAllText($testFileName)
                        $content | Should Be @'
key1=newValue1
 key\2 =newValue2

[Section]
key3=value3
key4= value4
key5=c:\\a\\b
key6=
app.webservice.password=123
keyNew=newValue
'@

                    $result | Should Not Be $null
                    $result.Count | Should Be 8
                    $result[0] | Should Match "Key 'key1' - value set to 'newValue1'"
                    $result[1] | Should Match "Key 'key\\2' - value set to 'newValue2'"
                    $result[2] | Should Match "Key 'key3' - value is already 'value3'"
                    $result[3] | Should Match "Key 'key5' - value set to 'c:\\\\a\\\\b'"
                    $result[4] | Should Match "Key 'key6' - value set to ''"
                    $result[5] | Should Match "Key 'keyNew' not found - adding with value 'newValue'"
                    $result[6] | Should Match "Key 'app.webservice.password' - value set to '123'"

                } finally {
                    Remove-Item -LiteralPath $testFileName -Force -ErrorAction SilentlyContinue
                }

            }
        }

        Context "when FailIfCannotMatch=true and cannot match" {

            It "should fail" {
                try {
                    New-TestFile
                    $params = Get-UpdateKeyValueCmdParams -ConfigFiles $testFileName -ConfigValues 'keyNotFound=test' -FailIfCannotMatch
                    try {
                        Invoke-Command @params
                    } catch {
                        Write-Host $_
                        return
                    }
                    0 | Should Be 1
                } finally {
                    Remove-Item -LiteralPath $testFileName -Force -ErrorAction SilentlyContinue
                }
            }
        }

    }
}


text:
{-# LANGUAGE DeriveGeneric #-}
module Stage (Stage (..), stageString) where

import Development.Shake.Classes
import GHC.Generics (Generic)

-- | A stage refers to a certain compiler in GHC's build process.
--
-- * Stage 0 is the bootstrapping compiler, i.e. the one already installed on
--   the user's system.
--
-- * Stage 1 is built using the stage 0 compiler, using GHC's source code.
--   The result is a compiler that was built by the bootstrapping compiler,
--   with all the features of the new compiler.
--
-- * Stage 2 is built using the stage 1 compiler and GHC's source code. The
--   result is a compiler "built by itself", commonly referred to as
--   /bootstrapping/.
--
-- * Stage 3 uses stage 2 to build from source again. The result should have
--   the same object code as stage 2, which is a good test for the compiler.
--   Since it serves no other purpose than that, the stage 3 build is usually
--   omitted in the build process.
data Stage = Stage0 | Stage1 | Stage2 | Stage3
           deriving (Show, Eq, Ord, Enum, Generic, Bounded)

-- | Prettyprint a 'Stage'.
stageString :: Stage -> String
stageString stage = "stage" ++ show (fromEnum stage)

-- Instances for storing in the Shake database
instance Binary Stage
instance Hashable Stage
instance NFData Stage


text:
package com.helger.html.js;

import javax.annotation.Nonnull;
import javax.annotation.concurrent.NotThreadSafe;

import com.helger.commons.ValueEnforcer;
import com.helger.commons.annotation.Nonempty;
import com.helger.commons.system.ENewLineMode;

/**
 * Default settings for the textual representation of JSDOM objects
 *
 * @author Philip Helger
 */
@NotThreadSafe
public final class JSWriterDefaultSettings
{
  public static final boolean DEFAULT_INDENT_AND_ALIGN = true;
  public static final boolean DEFAULT_GENERATE_TYPE_NAMES = false;
  public static final boolean DEFAULT_GENERATE_COMMENTS = false;
  public static final String DEFAULT_INDENT = "  ";

  private static boolean s_bIndentAndAlign = DEFAULT_INDENT_AND_ALIGN;
  private static boolean s_bGenerateTypeNames = DEFAULT_GENERATE_TYPE_NAMES;
  private static boolean s_bGenerateComments = DEFAULT_GENERATE_COMMENTS;
  private static String s_sIndent = DEFAULT_INDENT;
  private static ENewLineMode s_eNewLineMode = ENewLineMode.DEFAULT;

  private JSWriterDefaultSettings ()
  {}

  public static void setToDefault ()
  {
    s_bIndentAndAlign = DEFAULT_INDENT_AND_ALIGN;
    s_bGenerateTypeNames = DEFAULT_GENERATE_TYPE_NAMES;
    s_bGenerateComments = DEFAULT_GENERATE_COMMENTS;
    s_sIndent = DEFAULT_INDENT;
    s_eNewLineMode = ENewLineMode.DEFAULT;
  }

  public static void setIndentAndAlign (final boolean bIndentAndAlign)
  {
    s_bIndentAndAlign = bIndentAndAlign;
  }

  public static boolean isIndentAndAlign ()
  {
    return s_bIndentAndAlign;
  }

  public static void setGenerateTypeNames (final boolean bGenerateTypeNames)
  {
    s_bGenerateTypeNames = bGenerateTypeNames;
  }

  public static boolean isGenerateTypeNames ()
  {
    return s_bGenerateTypeNames;
  }

  public static void setGenerateComments (final boolean bGenerateComments)
  {
    s_bGenerateComments = bGenerateComments;
  }

  public static boolean isGenerateComments ()
  {
    return s_bGenerateComments;
  }

  /**
   * This is a wrapper around {@link #setIndentAndAlign(boolean)},
   * {@link #setGenerateTypeNames(boolean)} and
   * {@link #setGenerateComments(boolean)}
   *
   * @param bMinimumCodeSize
   *        true for minimum code size
   */
  public static void setMinimumCodeSize (final boolean bMinimumCodeSize)
  {
    setIndentAndAlign (!bMinimumCodeSize);
    setGenerateTypeNames (!bMinimumCodeSize);
    setGenerateComments (!bMinimumCodeSize);
  }

  public static void setIndent (@Nonnull @Nonempty final String sIndent)
  {
    s_sIndent = ValueEnforcer.notEmpty (sIndent, "Indent");
  }

  @Nonnull
  @Nonempty
  public static String getIndent ()
  {
    return s_sIndent;
  }

  public static void setNewLineMode (@Nonnull final ENewLineMode eNewLineMode)
  {
    s_eNewLineMode = ValueEnforcer.notNull (eNewLineMode, "NewLineMode");
  }

  @Nonnull
  public static ENewLineMode getNewLineMode ()
  {
    return s_eNewLineMode;
  }
}


text:
' Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.

Imports System.Threading
Imports Microsoft.CodeAnalysis.Formatting
Imports Microsoft.CodeAnalysis.Formatting.Rules
Imports Microsoft.CodeAnalysis.LanguageServices
Imports Microsoft.CodeAnalysis.Options
Imports Microsoft.CodeAnalysis.Text
Imports Microsoft.CodeAnalysis.Text.Shared.Extensions
Imports Microsoft.CodeAnalysis.VisualBasic.Formatting
Imports Microsoft.CodeAnalysis.VisualBasic.Syntax

Namespace Microsoft.CodeAnalysis.Editor.VisualBasic.Formatting.Indentation
    Partial Friend Class VisualBasicIndentationService
        Private Class Indenter
            Inherits AbstractIndenter

            Public Sub New(syntaxFacts As ISyntaxFactsService,
                           syntaxTree As SyntaxTree,
                           rules As IEnumerable(Of AbstractFormattingRule),
                           optionSet As OptionSet,
                           line As TextLine,
                           cancellationToken As CancellationToken)
                MyBase.New(syntaxFacts, syntaxTree, rules, optionSet, line, cancellationToken)
            End Sub

            Public Overrides Function ShouldUseFormatterIfAvailable() As Boolean
                Return ShouldUseSmartTokenFormatterInsteadOfIndenter(
                    Rules, DirectCast(Root, CompilationUnitSyntax), LineToBeIndented, OptionSet, CancellationToken)
            End Function

            Protected Overrides Function GetDesiredIndentationWorker(
                    token As SyntaxToken,
                    previousLine As TextLine,
                    lastNonWhitespacePosition As Integer) As IndentationResult

                If token.Span.End = lastNonWhitespacePosition + 1 Then
                    Return GetIndentationBasedOnToken(token)
                Else
                    Debug.Assert(token.FullSpan.Contains(lastNonWhitespacePosition))

                    Dim trivia = Root.FindTrivia(lastNonWhitespacePosition)

                    ' preserve the indentation of the comment trivia before a case statement
                    If trivia.Kind = SyntaxKind.CommentTrivia AndAlso trivia.Token.IsKind(SyntaxKind.CaseKeyword) AndAlso trivia.Token.Parent.IsKind(SyntaxKind.CaseStatement) Then
                        Return GetIndentationOfLine(previousLine)
                    End If

                    If trivia.Kind = SyntaxKind.LineContinuationTrivia Then
                        Return GetIndentationBasedOnToken(GetTokenOnLeft(trivia), trivia)
                    End If

                    ' Line ends in comment
                    If trivia.Kind = SyntaxKind.CommentTrivia Then ' Two cases a line ending comment or _ comment
                        Dim firstTrivia As SyntaxTrivia = Tree.GetRoot(CancellationToken).FindTrivia(token.Span.End + 1)
                        ' firstTrivia contains either an _ or a comment, this is the First trivia after the last Token on the line
                        If firstTrivia.Kind = SyntaxKind.LineContinuationTrivia Then
                            Return GetIndentationBasedOnToken(GetTokenOnLeft(firstTrivia), firstTrivia)
                        Else
                            ' This is we have just a comment
                            Return GetIndentationBasedOnToken(GetTokenOnLeft(trivia), trivia)
                        End If
                    End If

                    ' if we are at invalid token (skipped token) at the end of statement, treat it like we are after line continuation
                    If trivia.Kind = SyntaxKind.SkippedTokensTrivia AndAlso trivia.Token.IsLastTokenOfStatement() Then
                        Return GetIndentationBasedOnToken(GetTokenOnLeft(trivia), trivia)
                    End If

                    ' okay, now check whether the trivia is at the beginning of the line
                    Dim firstNonWhitespacePosition = previousLine.GetFirstNonWhitespacePosition()
                    If Not firstNonWhitespacePosition.HasValue Then
                        Return IndentFromStartOfLine(0)
                    End If

                    Dim firstTokenOnLine = Root.FindToken(firstNonWhitespacePosition.Value, findInsideTrivia:=True)
                    If firstTokenOnLine.Kind <> SyntaxKind.None AndAlso firstTokenOnLine.Span.Contains(firstNonWhitespacePosition.Value) Then
                        'okay, beginning of the line is not trivia, use this token as the base token
                        Return GetIndentationBasedOnToken(firstTokenOnLine)
                    End If

                    Return GetIndentationOfLine(previousLine)
                End If
            End Function

            Private Function GetTokenOnLeft(trivia As SyntaxTrivia) As SyntaxToken
                Dim token = trivia.Token
                If token.Span.End <= trivia.SpanStart AndAlso Not token.IsMissing Then
                    Return token
                End If

                Return token.GetPreviousToken()
            End Function

            Private Function GetIndentationBasedOnToken(token As SyntaxToken, Optional trivia As SyntaxTrivia = Nothing) As IndentationResult
                Dim sourceText = LineToBeIndented.Text

                Dim position = GetCurrentPositionNotBelongToEndOfFileToken(LineToBeIndented.Start)

                ' lines must be blank since we got the token from the first non blank line above current position
                If HasLinesBetween(Tree.GetText().Lines.IndexOf(token.Span.End), LineToBeIndented.LineNumber) Then
                    ' if there are blank lines between, return indentation of the owning statement
                    Return GetIndentationOfCurrentPosition(token, position)
                End If

                Dim indentation = GetIndentationFromOperationService(token, position)
                If indentation.HasValue Then
                    Return indentation.Value
                End If

                Dim queryNode = token.GetAncestor(Of QueryClauseSyntax)()
                If queryNode IsNot Nothing Then
                    Dim subQuerySpaces = If(token.IsLastTokenOfStatement(), 0, Me.OptionSet.GetOption(FormattingOptions.IndentationSize, token.Language))
                    Return GetIndentationOfToken(queryNode.GetFirstToken(includeZeroWidth:=True), subQuerySpaces)
                End If

                ' check one more time for query case
                If token.Kind = SyntaxKind.IdentifierToken AndAlso token.HasMatchingText(SyntaxKind.FromKeyword) Then
                    Return GetIndentationOfToken(token)
                End If

                If FormattingHelpers.IsXmlTokenInXmlDeclaration(token) Then
                    Dim xmlDocument = token.GetAncestor(Of XmlDocumentSyntax)()
                    Return GetIndentationOfToken(xmlDocument.GetFirstToken(includeZeroWidth:=True))
                End If

                ' implicit line continuation case
                If IsLineContinuable(token, trivia, position) Then
                    Return GetIndentationFromTokenLineAfterLineContinuation(token, trivia)
                End If

                Return GetIndentationOfCurrentPosition(token, position)
            End Function

            Private Function GetIndentationOfCurrentPosition(token As SyntaxToken, position As Integer) As IndentationResult
                Return GetIndentationOfCurrentPosition(token, position, extraSpaces:=0)
            End Function

            Private Function GetIndentationOfCurrentPosition(token As SyntaxToken, position As Integer, extraSpaces As Integer) As IndentationResult
                ' special case for multi-line string
                Dim containingToken = Tree.FindTokenOnLeftOfPosition(position, CancellationToken)
                If containingToken.IsKind(SyntaxKind.InterpolatedStringTextToken) OrElse
                   containingToken.IsKind(SyntaxKind.InterpolatedStringText) OrElse
                    (containingToken.IsKind(SyntaxKind.CloseBraceToken) AndAlso token.Parent.IsKind(SyntaxKind.Interpolation)) Then
                    Return IndentFromStartOfLine(0)
                End If
                If containingToken.Kind = SyntaxKind.StringLiteralToken AndAlso containingToken.FullSpan.Contains(position) Then
                    Return IndentFromStartOfLine(0)
                End If

                Return IndentFromStartOfLine(Finder.GetIndentationOfCurrentPosition(Tree, token, position, extraSpaces, CancellationToken))
            End Function

            Private Function IsLineContinuable(lastVisibleTokenOnPreviousLine As SyntaxToken, trivia As SyntaxTrivia, position As Integer) As Boolean
                If trivia.Kind = SyntaxKind.LineContinuationTrivia OrElse
                   trivia.Kind = SyntaxKind.SkippedTokensTrivia Then
                    Return True
                End If

                If lastVisibleTokenOnPreviousLine.IsLastTokenOfStatement() Then
                    Return False
                End If

                Dim visibleTokenOnCurrentLine As SyntaxToken = lastVisibleTokenOnPreviousLine.GetNextToken()
                If Not lastVisibleTokenOnPreviousLine.IsKind(SyntaxKind.OpenBraceToken) AndAlso
                    Not lastVisibleTokenOnPreviousLine.IsKind(SyntaxKind.CommaToken) Then
                    If IsCloseBraceOfInitializerSyntax(visibleTokenOnCurrentLine) Then
                        Return False
                    End If
                Else
                    If IsCloseBraceOfInitializerSyntax(visibleTokenOnCurrentLine) Then
                        Return True
                    End If
                End If

                If Not ContainingStatementHasDiagnostic(lastVisibleTokenOnPreviousLine.Parent) Then
                    Return True
                End If

                If lastVisibleTokenOnPreviousLine.GetNextToken(includeZeroWidth:=True).IsMissing Then
                    Return True
                End If

                Return False
            End Function

            Private Function IsCloseBraceOfInitializerSyntax(visibleTokenOnCurrentLine As SyntaxToken) As Boolean
                If visibleTokenOnCurrentLine.IsKind(SyntaxKind.CloseBraceToken) Then
                    Dim visibleTokenOnCurrentLineParent = visibleTokenOnCurrentLine.Parent
                    If TypeOf visibleTokenOnCurrentLineParent Is ObjectCreationInitializerSyntax OrElse
                    TypeOf visibleTokenOnCurrentLineParent Is CollectionInitializerSyntax Then
                        Return True
                    End If
                End If

                Return False
            End Function

            Private Function ContainingStatementHasDiagnostic(node As SyntaxNode) As Boolean
                If node Is Nothing Then
                    Return False
                End If

                If node.ContainsDiagnostics Then
                    Return True
                End If

                Dim containingStatement = node.GetAncestorOrThis(Of StatementSyntax)()
                If containingStatement Is Nothing Then
                    Return False
                End If

                Return containingStatement.ContainsDiagnostics()
            End Function

            Private Function GetIndentationFromOperationService(token As SyntaxToken, position As Integer) As IndentationResult?
                ' check operation service to see whether we can determine indentation from it
                If token.Kind = SyntaxKind.None Then
                    Return Nothing
                End If

                Dim indentation = Finder.FromIndentBlockOperations(Tree, token, position, CancellationToken)
                If indentation.HasValue Then
                    Return IndentFromStartOfLine(indentation.Value)
                End If

                ' special case xml text literal before checking alignment operation
                ' VB has different behavior around missing alignment token. for query expression, VB prefers putting
                ' caret aligned with previous query clause, but for xml literals, it prefer them to be ignored and indented
                ' based on current indentation level.
                If token.Kind = SyntaxKind.XmlTextLiteralToken OrElse
                   token.Kind = SyntaxKind.XmlEntityLiteralToken Then
                    Return GetIndentationOfLine(LineToBeIndented.Text.Lines.GetLineFromPosition(token.SpanStart))
                End If

                ' check alignment token indentation
                Dim alignmentTokenIndentation = Finder.FromAlignTokensOperations(Tree, token)
                If alignmentTokenIndentation.HasValue Then
                    Return IndentFromStartOfLine(alignmentTokenIndentation.Value)
                End If

                Return Nothing
            End Function

            Private Function GetIndentationFromTokenLineAfterLineContinuation(token As SyntaxToken, trivia As SyntaxTrivia) As IndentationResult
                Dim sourceText = LineToBeIndented.Text
                Dim position = LineToBeIndented.Start

                position = GetCurrentPositionNotBelongToEndOfFileToken(position)

                Dim currentTokenLine = sourceText.Lines.GetLineFromPosition(token.SpanStart)

                ' error case where the line continuation belongs to a meaningless token such as empty token for skipped text
                If token.Kind = SyntaxKind.EmptyToken Then
                    Dim baseLine = sourceText.Lines.GetLineFromPosition(trivia.SpanStart)
                    Return GetIndentationOfLine(baseLine)
                End If

                Dim xmlEmbeddedExpression = token.GetAncestor(Of XmlEmbeddedExpressionSyntax)()
                If xmlEmbeddedExpression IsNot Nothing Then
                    Dim firstExpressionLine = sourceText.Lines.GetLineFromPosition(xmlEmbeddedExpression.GetFirstToken(includeZeroWidth:=True).SpanStart)
                    Return GetIndentationFromTwoLines(firstExpressionLine, currentTokenLine, token, position)
                End If

                If FormattingHelpers.IsGreaterThanInAttribute(token) Then
                    Dim attribute = token.GetAncestor(Of AttributeListSyntax)()
                    Dim baseLine = sourceText.Lines.GetLineFromPosition(attribute.GetFirstToken(includeZeroWidth:=True).SpanStart)
                    Return GetIndentationOfLine(baseLine)
                End If

                ' if position is between "," and next token, consider the position to be belonged to the list that
                ' owns the ","
                If IsCommaInParameters(token) AndAlso (token.Span.End <= position AndAlso position <= token.GetNextToken().SpanStart) Then
                    Return GetIndentationOfCurrentPosition(token, token.SpanStart)
                End If

                Dim statement = token.GetAncestor(Of StatementSyntax)()

                ' this can happen if only token in the file is End Of File Token
                If statement Is Nothing Then
                    If trivia.Kind <> SyntaxKind.None Then
                        Dim triviaLine = sourceText.Lines.GetLineFromPosition(trivia.SpanStart)
                        Return GetIndentationOfLine(triviaLine, Me.OptionSet.GetOption(FormattingOptions.IndentationSize, token.Language))
                    End If

                    ' no base line to use to calculate the indentation
                    Return IndentFromStartOfLine(0)
                End If

                ' find line where first token of statement is starting on
                Dim firstTokenLine = sourceText.Lines.GetLineFromPosition(statement.GetFirstToken(includeZeroWidth:=True).SpanStart)
                Return GetIndentationFromTwoLines(firstTokenLine, currentTokenLine, token, position)
            End Function

            Private Function IsCommaInParameters(token As SyntaxToken) As Boolean
                Return token.Kind = SyntaxKind.CommaToken AndAlso
                    (TypeOf token.Parent Is ParameterListSyntax OrElse
                     TypeOf token.Parent Is ArgumentListSyntax OrElse
                     TypeOf token.Parent Is TypeParameterListSyntax)
            End Function

            Private Function GetIndentationFromTwoLines(firstLine As TextLine, secondLine As TextLine, token As SyntaxToken, position As Integer) As IndentationResult
                If firstLine.LineNumber = secondLine.LineNumber Then
                    ' things are on same line, put the indentation size
                    Return GetIndentationOfCurrentPosition(token, position, Me.OptionSet.GetOption(FormattingOptions.IndentationSize, token.Language))
                End If

                ' multiline
                Return GetIndentationOfLine(secondLine)
            End Function

            Private Function HasLinesBetween(lineNumber1 As Integer, lineNumber2 As Integer) As Boolean
                Return lineNumber1 + 1 < lineNumber2
            End Function
        End Class
    End Class
End Namespace


text:
import re

from livestreamer.plugin import Plugin
from livestreamer.plugin.api import http, validate
from livestreamer.stream import RTMPStream, HLSStream

CHANNEL_INFO_URL = "http://live.afreeca.com:8057/api/get_broad_state_list.php"
KEEP_ALIVE_URL = "{server}/stream_keepalive.html"
STREAM_INFO_URLS = {
    "rtmp": "http://sessionmanager01.afreeca.tv:6060/broad_stream_assign.html",
    "hls": "http://resourcemanager.afreeca.tv:9090/broad_stream_assign.html"
}
HLS_KEY_URL = "http://api.m.afreeca.com/broad/a/watch"

CHANNEL_RESULT_ERROR = 0
CHANNEL_RESULT_OK = 1

_url_re = re.compile("http(s)?://(\w+\.)?afreeca.com/(?P<username>\w+)")

_channel_schema = validate.Schema(
    {
        "CHANNEL": {
            "RESULT": validate.transform(int),
            "BROAD_INFOS": [{
                "list": [{
                    "nBroadNo": validate.text
                }]
            }]
        }
    },
    validate.get("CHANNEL")
)
_stream_schema = validate.Schema(
    {
        validate.optional("view_url"): validate.url(
            scheme=validate.any("rtmp", "http")
        )
    }
)

class AfreecaTV(Plugin):
    @classmethod
    def can_handle_url(self, url):
        return _url_re.match(url)

    def _get_channel_info(self, username):
        headers = {
            "Referer": self.url
        }
        data = {
            "uid": username
        }
        res = http.post(CHANNEL_INFO_URL, data=data, headers=headers)

        return http.json(res, schema=_channel_schema)

    def _get_hls_key(self, broadcast, username):
        headers = {
            "Referer": self.url
        }
        data = {
            "bj_id": username,
            "broad_no": broadcast
        }
        res = http.post(HLS_KEY_URL, data=data, headers=headers)

        return http.json(res)

    def _get_stream_info(self, broadcast, type):
        params = {
            "return_type": "gs_cdn",
            "use_cors": "true",
            "cors_origin_url": "m.afreeca.com",
            "broad_no": "{broadcast}-mobile-hd-{type}".format(**locals()),
            "broad_key": "{broadcast}-flash-hd-{type}".format(**locals())
        }
        res = http.get(STREAM_INFO_URLS[type], params=params)
        return http.json(res, schema=_stream_schema)

    def _get_hls_stream(self, broadcast, username):
        keyjson = self._get_hls_key(broadcast, username)
        if keyjson["result"] != CHANNEL_RESULT_OK:
            return
        key = keyjson["data"]["hls_authentication_key"]
        info = self._get_stream_info(broadcast, "hls")
        if "view_url" in info:
            return HLSStream(self.session, info["view_url"], params=dict(aid=key))

    def _get_rtmp_stream(self, broadcast):
        info = self._get_stream_info(broadcast, "rtmp")
        if "view_url" in info:
            params = dict(rtmp=info["view_url"])
            return RTMPStream(self.session, params=params, redirect=True)

    def _get_streams(self):
        match = _url_re.match(self.url)
        username = match.group("username")

        channel = self._get_channel_info(username)
        if channel["RESULT"] != CHANNEL_RESULT_OK:
            return

        broadcast = channel["BROAD_INFOS"][0]["list"][0]["nBroadNo"]
        if not broadcast:
            return

        flash_stream = self._get_rtmp_stream(broadcast)
        if flash_stream:
            yield "live", flash_stream

        mobile_stream = self._get_hls_stream(broadcast, username)
        if mobile_stream:
            yield "live", mobile_stream

__plugin__ = AfreecaTV


text:
<?php

namespace yii\console;

use Yii;
use yii\base\Action;
use yii\base\InlineAction;
use yii\base\InvalidRouteException;
use yii\helpers\Console;
use yii\helpers\Inflector;

/**
 * Controller is the base class of console command classes.
 *
 * A console controller consists of one or several actions known as sub-commands.
 * Users call a console command by specifying the corresponding route which identifies a controller action.
 * The `yii` program is used when calling a console command, like the following:
 *
 * ```
 * yii <route> [--param1=value1 --param2 ...]
 * ```
 *
 * where `<route>` is a route to a controller action and the params will be populated as properties of a command.
 * See [[options()]] for details.
 *
 * @property string $help This property is read-only.
 * @property string $helpSummary This property is read-only.
 * @property array $passedOptionValues The properties corresponding to the passed options. This property is
 * read-only.
 * @property array $passedOptions The names of the options passed during execution. This property is
 * read-only.
 *
 * @author Qiang Xue <qiang.xue@gmail.com>
 * @since 2.0
 */
class Controller extends \yii\base\Controller
{
    /**
     * @deprecated since 2.0.13. Use [[ExitCode::OK]] instead.
     */
    const EXIT_CODE_NORMAL = 0;
    /**
     * @deprecated since 2.0.13. Use [[ExitCode::UNSPECIFIED_ERROR]] instead.
     */
    const EXIT_CODE_ERROR = 1;

    /**
     * @var bool whether to run the command interactively.
     */
    public $interactive = true;
    /**
     * @var bool whether to enable ANSI color in the output.
     * If not set, ANSI color will only be enabled for terminals that support it.
     */
    public $color;
    /**
     * @var bool whether to display help information about current command.
     * @since 2.0.10
     */
    public $help;

    /**
     * @var array the options passed during execution.
     */
    private $_passedOptions = [];

    /**
     * Returns a value indicating whether ANSI color is enabled.
     *
     * ANSI color is enabled only if [[color]] is set true or is not set
     * and the terminal supports ANSI color.
     *
     * @param resource $stream the stream to check.
     * @return bool Whether to enable ANSI style in output.
     */
    public function isColorEnabled($stream = \STDOUT)
    {
        return $this->color === null ? Console::streamSupportsAnsiColors($stream) : $this->color;
    }

    /**
     * Runs an action with the specified action ID and parameters.
     * If the action ID is empty, the method will use [[defaultAction]].
     * @param string $id the ID of the action to be executed.
     * @param array $params the parameters (name-value pairs) to be passed to the action.
     * @return int the status of the action execution. 0 means normal, other values mean abnormal.
     * @throws InvalidRouteException if the requested action ID cannot be resolved into an action successfully.
     * @throws Exception if there are unknown options or missing arguments
     * @see createAction
     */
    public function runAction($id, $params = [])
    {
        if (!empty($params)) {
            // populate options here so that they are available in beforeAction().
            $options = $this->options($id === '' ? $this->defaultAction : $id);
            if (isset($params['_aliases'])) {
                $optionAliases = $this->optionAliases();
                foreach ($params['_aliases'] as $name => $value) {
                    if (array_key_exists($name, $optionAliases)) {
                        $params[$optionAliases[$name]] = $value;
                    } else {
                        throw new Exception(Yii::t('yii', 'Unknown alias: -{name}', ['name' => $name]));
                    }
                }
                unset($params['_aliases']);
            }
            foreach ($params as $name => $value) {
                // Allow camelCase options to be entered in kebab-case
                if (!in_array($name, $options, true) && strpos($name, '-') !== false) {
                    $kebabName = $name;
                    $altName = lcfirst(Inflector::id2camel($kebabName));
                    if (in_array($altName, $options, true)) {
                        $name = $altName;
                    }
                }

                if (in_array($name, $options, true)) {
                    $default = $this->$name;
                    if (is_array($default)) {
                        $this->$name = preg_split('/\s*,\s*(?![^()]*\))/', $value);
                    } elseif ($default !== null) {
                        settype($value, gettype($default));
                        $this->$name = $value;
                    } else {
                        $this->$name = $value;
                    }
                    $this->_passedOptions[] = $name;
                    unset($params[$name]);
                    if (isset($kebabName)) {
                        unset($params[$kebabName]);
                    }
                } elseif (!is_int($name)) {
                    throw new Exception(Yii::t('yii', 'Unknown option: --{name}', ['name' => $name]));
                }
            }
        }
        if ($this->help) {
            $route = $this->getUniqueId() . '/' . $id;
            return Yii::$app->runAction('help', [$route]);
        }

        return parent::runAction($id, $params);
    }

    /**
     * Binds the parameters to the action.
     * This method is invoked by [[Action]] when it begins to run with the given parameters.
     * This method will first bind the parameters with the [[options()|options]]
     * available to the action. It then validates the given arguments.
     * @param Action $action the action to be bound with parameters
     * @param array $params the parameters to be bound to the action
     * @return array the valid parameters that the action can run with.
     * @throws Exception if there are unknown options or missing arguments
     */
    public function bindActionParams($action, $params)
    {
        if ($action instanceof InlineAction) {
            $method = new \ReflectionMethod($this, $action->actionMethod);
        } else {
            $method = new \ReflectionMethod($action, 'run');
        }

        $args = array_values($params);

        $missing = [];
        foreach ($method->getParameters() as $i => $param) {
            if ($param->isArray() && isset($args[$i])) {
                $args[$i] = $args[$i] === '' ? [] : preg_split('/\s*,\s*/', $args[$i]);
            }
            if (!isset($args[$i])) {
                if ($param->isDefaultValueAvailable()) {
                    $args[$i] = $param->getDefaultValue();
                } else {
                    $missing[] = $param->getName();
                }
            }
        }

        if (!empty($missing)) {
            throw new Exception(Yii::t('yii', 'Missing required arguments: {params}', ['params' => implode(', ', $missing)]));
        }

        return $args;
    }

    /**
     * Formats a string with ANSI codes.
     *
     * You may pass additional parameters using the constants defined in [[\yii\helpers\Console]].
     *
     * Example:
     *
     * ```
     * echo $this->ansiFormat('This will be red and underlined.', Console::FG_RED, Console::UNDERLINE);
     * ```
     *
     * @param string $string the string to be formatted
     * @return string
     */
    public function ansiFormat($string)
    {
        if ($this->isColorEnabled()) {
            $args = func_get_args();
            array_shift($args);
            $string = Console::ansiFormat($string, $args);
        }

        return $string;
    }

    /**
     * Prints a string to STDOUT.
     *
     * You may optionally format the string with ANSI codes by
     * passing additional parameters using the constants defined in [[\yii\helpers\Console]].
     *
     * Example:
     *
     * ```
     * $this->stdout('This will be red and underlined.', Console::FG_RED, Console::UNDERLINE);
     * ```
     *
     * @param string $string the string to print
     * @return int|bool Number of bytes printed or false on error
     */
    public function stdout($string)
    {
        if ($this->isColorEnabled()) {
            $args = func_get_args();
            array_shift($args);
            $string = Console::ansiFormat($string, $args);
        }

        return Console::stdout($string);
    }

    /**
     * Prints a string to STDERR.
     *
     * You may optionally format the string with ANSI codes by
     * passing additional parameters using the constants defined in [[\yii\helpers\Console]].
     *
     * Example:
     *
     * ```
     * $this->stderr('This will be red and underlined.', Console::FG_RED, Console::UNDERLINE);
     * ```
     *
     * @param string $string the string to print
     * @return int|bool Number of bytes printed or false on error
     */
    public function stderr($string)
    {
        if ($this->isColorEnabled(\STDERR)) {
            $args = func_get_args();
            array_shift($args);
            $string = Console::ansiFormat($string, $args);
        }

        return fwrite(\STDERR, $string);
    }

    /**
     * Prompts the user for input and validates it.
     *
     * @param string $text prompt string
     * @param array $options the options to validate the input:
     *
     *  - required: whether it is required or not
     *  - default: default value if no input is inserted by the user
     *  - pattern: regular expression pattern to validate user input
     *  - validator: a callable function to validate input. The function must accept two parameters:
     *      - $input: the user input to validate
     *      - $error: the error value passed by reference if validation failed.
     *
     * An example of how to use the prompt method with a validator function.
     *
     * ```php
     * $code = $this->prompt('Enter 4-Chars-Pin', ['required' => true, 'validator' => function($input, &$error) {
     *     if (strlen($input) !== 4) {
     *         $error = 'The Pin must be exactly 4 chars!';
     *         return false;
     *     }
     *     return true;
     * }]);
     * ```
     *
     * @return string the user input
     */
    public function prompt($text, $options = [])
    {
        if ($this->interactive) {
            return Console::prompt($text, $options);
        }

        return isset($options['default']) ? $options['default'] : '';
    }

    /**
     * Asks user to confirm by typing y or n.
     *
     * A typical usage looks like the following:
     *
     * ```php
     * if ($this->confirm("Are you sure?")) {
     *     echo "user typed yes\n";
     * } else {
     *     echo "user typed no\n";
     * }
     * ```
     *
     * @param string $message to echo out before waiting for user input
     * @param bool $default this value is returned if no selection is made.
     * @return bool whether user confirmed.
     * Will return true if [[interactive]] is false.
     */
    public function confirm($message, $default = false)
    {
        if ($this->interactive) {
            return Console::confirm($message, $default);
        }

        return true;
    }

    /**
     * Gives the user an option to choose from. Giving '?' as an input will show
     * a list of options to choose from and their explanations.
     *
     * @param string $prompt the prompt message
     * @param array $options Key-value array of options to choose from
     *
     * @return string An option character the user chose
     */
    public function select($prompt, $options = [])
    {
        return Console::select($prompt, $options);
    }

    /**
     * Returns the names of valid options for the action (id)
     * An option requires the existence of a public member variable whose
     * name is the option name.
     * Child classes may override this method to specify possible options.
     *
     * Note that the values setting via options are not available
     * until [[beforeAction()]] is being called.
     *
     * @param string $actionID the action id of the current request
     * @return string[] the names of the options valid for the action
     */
    public function options($actionID)
    {
        // $actionId might be used in subclasses to provide options specific to action id
        return ['color', 'interactive', 'help'];
    }

    /**
     * Returns option alias names.
     * Child classes may override this method to specify alias options.
     *
     * @return array the options alias names valid for the action
     * where the keys is alias name for option and value is option name.
     *
     * @since 2.0.8
     * @see options()
     */
    public function optionAliases()
    {
        return [
            'h' => 'help',
        ];
    }

    /**
     * Returns properties corresponding to the options for the action id
     * Child classes may override this method to specify possible properties.
     *
     * @param string $actionID the action id of the current request
     * @return array properties corresponding to the options for the action
     */
    public function getOptionValues($actionID)
    {
        // $actionId might be used in subclasses to provide properties specific to action id
        $properties = [];
        foreach ($this->options($this->action->id) as $property) {
            $properties[$property] = $this->$property;
        }

        return $properties;
    }

    /**
     * Returns the names of valid options passed during execution.
     *
     * @return array the names of the options passed during execution
     */
    public function getPassedOptions()
    {
        return $this->_passedOptions;
    }

    /**
     * Returns the properties corresponding to the passed options.
     *
     * @return array the properties corresponding to the passed options
     */
    public function getPassedOptionValues()
    {
        $properties = [];
        foreach ($this->_passedOptions as $property) {
            $properties[$property] = $this->$property;
        }

        return $properties;
    }

    /**
     * Returns one-line short summary describing this controller.
     *
     * You may override this method to return customized summary.
     * The default implementation returns first line from the PHPDoc comment.
     *
     * @return string
     */
    public function getHelpSummary()
    {
        return $this->parseDocCommentSummary(new \ReflectionClass($this));
    }

    /**
     * Returns help information for this controller.
     *
     * You may override this method to return customized help.
     * The default implementation returns help information retrieved from the PHPDoc comment.
     * @return string
     */
    public function getHelp()
    {
        return $this->parseDocCommentDetail(new \ReflectionClass($this));
    }

    /**
     * Returns a one-line short summary describing the specified action.
     * @param Action $action action to get summary for
     * @return string a one-line short summary describing the specified action.
     */
    public function getActionHelpSummary($action)
    {
        if ($action === null) {
            return $this->ansiFormat(Yii::t('yii', 'Action not found.'), Console::FG_RED);
        }

        return $this->parseDocCommentSummary($this->getActionMethodReflection($action));
    }

    /**
     * Returns the detailed help information for the specified action.
     * @param Action $action action to get help for
     * @return string the detailed help information for the specified action.
     */
    public function getActionHelp($action)
    {
        return $this->parseDocCommentDetail($this->getActionMethodReflection($action));
    }

    /**
     * Returns the help information for the anonymous arguments for the action.
     *
     * The returned value should be an array. The keys are the argument names, and the values are
     * the corresponding help information. Each value must be an array of the following structure:
     *
     * - required: boolean, whether this argument is required.
     * - type: string, the PHP type of this argument.
     * - default: string, the default value of this argument
     * - comment: string, the comment of this argument
     *
     * The default implementation will return the help information extracted from the doc-comment of
     * the parameters corresponding to the action method.
     *
     * @param Action $action
     * @return array the help information of the action arguments
     */
    public function getActionArgsHelp($action)
    {
        $method = $this->getActionMethodReflection($action);
        $tags = $this->parseDocCommentTags($method);
        $params = isset($tags['param']) ? (array) $tags['param'] : [];

        $args = [];

        /** @var \ReflectionParameter $reflection */
        foreach ($method->getParameters() as $i => $reflection) {
            if ($reflection->getClass() !== null) {
                continue;
            }
            $name = $reflection->getName();
            $tag = isset($params[$i]) ? $params[$i] : '';
            if (preg_match('/^(\S+)\s+(\$\w+\s+)?(.*)/s', $tag, $matches)) {
                $type = $matches[1];
                $comment = $matches[3];
            } else {
                $type = null;
                $comment = $tag;
            }
            if ($reflection->isDefaultValueAvailable()) {
                $args[$name] = [
                    'required' => false,
                    'type' => $type,
                    'default' => $reflection->getDefaultValue(),
                    'comment' => $comment,
                ];
            } else {
                $args[$name] = [
                    'required' => true,
                    'type' => $type,
                    'default' => null,
                    'comment' => $comment,
                ];
            }
        }

        return $args;
    }

    /**
     * Returns the help information for the options for the action.
     *
     * The returned value should be an array. The keys are the option names, and the values are
     * the corresponding help information. Each value must be an array of the following structure:
     *
     * - type: string, the PHP type of this argument.
     * - default: string, the default value of this argument
     * - comment: string, the comment of this argument
     *
     * The default implementation will return the help information extracted from the doc-comment of
     * the properties corresponding to the action options.
     *
     * @param Action $action
     * @return array the help information of the action options
     */
    public function getActionOptionsHelp($action)
    {
        $optionNames = $this->options($action->id);
        if (empty($optionNames)) {
            return [];
        }

        $class = new \ReflectionClass($this);
        $options = [];
        foreach ($class->getProperties() as $property) {
            $name = $property->getName();
            if (!in_array($name, $optionNames, true)) {
                continue;
            }
            $defaultValue = $property->getValue($this);
            $tags = $this->parseDocCommentTags($property);

            // Display camelCase options in kebab-case
            $name = Inflector::camel2id($name, '-', true);

            if (isset($tags['var']) || isset($tags['property'])) {
                $doc = isset($tags['var']) ? $tags['var'] : $tags['property'];
                if (is_array($doc)) {
                    $doc = reset($doc);
                }
                if (preg_match('/^(\S+)(.*)/s', $doc, $matches)) {
                    $type = $matches[1];
                    $comment = $matches[2];
                } else {
                    $type = null;
                    $comment = $doc;
                }
                $options[$name] = [
                    'type' => $type,
                    'default' => $defaultValue,
                    'comment' => $comment,
                ];
            } else {
                $options[$name] = [
                    'type' => null,
                    'default' => $defaultValue,
                    'comment' => '',
                ];
            }
        }

        return $options;
    }

    private $_reflections = [];

    /**
     * @param Action $action
     * @return \ReflectionMethod
     */
    protected function getActionMethodReflection($action)
    {
        if (!isset($this->_reflections[$action->id])) {
            if ($action instanceof InlineAction) {
                $this->_reflections[$action->id] = new \ReflectionMethod($this, $action->actionMethod);
            } else {
                $this->_reflections[$action->id] = new \ReflectionMethod($action, 'run');
            }
        }

        return $this->_reflections[$action->id];
    }

    /**
     * Parses the comment block into tags.
     * @param \Reflector $reflection the comment block
     * @return array the parsed tags
     */
    protected function parseDocCommentTags($reflection)
    {
        $comment = $reflection->getDocComment();
        $comment = "@description \n" . strtr(trim(preg_replace('/^\s*\**( |\t)?/m', '', trim($comment, '/'))), "\r", '');
        $parts = preg_split('/^\s*@/m', $comment, -1, PREG_SPLIT_NO_EMPTY);
        $tags = [];
        foreach ($parts as $part) {
            if (preg_match('/^(\w+)(.*)/ms', trim($part), $matches)) {
                $name = $matches[1];
                if (!isset($tags[$name])) {
                    $tags[$name] = trim($matches[2]);
                } elseif (is_array($tags[$name])) {
                    $tags[$name][] = trim($matches[2]);
                } else {
                    $tags[$name] = [$tags[$name], trim($matches[2])];
                }
            }
        }

        return $tags;
    }

    /**
     * Returns the first line of docblock.
     *
     * @param \Reflector $reflection
     * @return string
     */
    protected function parseDocCommentSummary($reflection)
    {
        $docLines = preg_split('~\R~u', $reflection->getDocComment());
        if (isset($docLines[1])) {
            return trim($docLines[1], "\t *");
        }

        return '';
    }

    /**
     * Returns full description from the docblock.
     *
     * @param \Reflector $reflection
     * @return string
     */
    protected function parseDocCommentDetail($reflection)
    {
        $comment = strtr(trim(preg_replace('/^\s*\**( |\t)?/m', '', trim($reflection->getDocComment(), '/'))), "\r", '');
        if (preg_match('/^\s*@\w+/m', $comment, $matches, PREG_OFFSET_CAPTURE)) {
            $comment = trim(substr($comment, 0, $matches[0][1]));
        }
        if ($comment !== '') {
            return rtrim(Console::renderColoredString(Console::markdownToAnsi($comment)));
        }

        return '';
    }
}