text:
(function () {
    'use strict';

    var when = require('when');

    exports.monad = function () {
        var base = {
            mreturn: function (val) {
                return function (k) {
                    return k(val);
                };
            },

            mbind: function (m, fun) {
                return function (k) {
                    return m(function (val) {
                        return (fun(val))(k);
                    });
                };
            },

            lift: function (m) {
                var im = this.innerMonad;
                return function (k) {
                    return im.mbind(m, k);
                };
            },

            run: function (monad, cont) {
                cont = cont || base.id;
                return monad(cont);
            },

            // callCC f = ContT $ \c -> runContT (f (\a -> ContT $ \_ -> c a)) c
            callCC: function (fun) {
                return function (k) {
                    var kontinue = function (val) {
                        return function (otherCont) {
                            return k(val);
                        };
                    };
                    var m = fun(kontinue);
                    return m(k);
                };
            },

            // The differences between `callCC` and `suspend` are:
            // - `callCC` can't cope with the stack unwinding.
            // - `suspend` can cope with the stack unwinding.
            // - inside the `callCC` arg function, you can `return` the
            //   resolution, or invoke the supplied fun to specify the
            //   resolution. Obviously, returning will only work on
            //   the nearest parent-scoped `callCC`.
            // - inside the `suspend` arg function, `return` will not
            //   work to specify the resolution: this is deliberate as
            //   it is assumed that the stack will unwind (as it would
            //   with returning a value) before the resolution is
            //   known.
            // - `callCC` assumes that value returned by the arg
            //   function is already monadic, and will invoke it
            //   automatically. Thus you do not need to use
            //   `cont.run`.
            // - `suspend` does not make this assumption - it allows
            //   the arg function to be any raw function as it does
            //   not inspect or care about the result (see
            //   above). Thus if you want to write monadically within
            //   the arg function, you must remember to call
            //   `cont.run` explicitly.
            // - Due to this, in `suspend`, the provided function to
            //   receive the resolution is not a monadic value. Thus
            //   if this function escapes from the arg function,
            //   (e.g. `k2 <- suspend(function (k1) { return k1; });`)
            //   then it must subsequently be treated as a non-monadic
            //   function. E.g. invoke with `return k2(val);` rather
            //   then just `k2(val);`
            // - Both allow the resolution to be made multiple times,
            //   but only suspend, by unwrapping the stack, allows
            //   this to be used to construct infinite loops: callCC
            //   will blow the stack.
            suspend: function (fun) {
                return function (k) {
                    var deferred = when.defer(),
                        resolver = deferred.resolver,
                        result = when.defer(),
                        exception = {},
                        shouldThrow = true,
                        switched = 0, // switched acts as a semaphore
                        resumeCont;
                    try {
                        fun(function (val) {
                            var nextDefer = when.defer(),
                                oldResolver = resolver;
                            resolver = nextDefer.resolver;
                            switched += 1;
                            oldResolver.resolve({value:   val,
                                                 promise: nextDefer.promise});
                            if (shouldThrow) {
                                throw exception;
                            }
                        });
                    } catch (e) {
                        if (e !== exception) {
                            throw e;
                        }
                    } finally {
                        shouldThrow = false;
                    }
                    resumeCont = function(val) {
                        // Ensure we process the current invocation
                        // before any future invocation!
                        process.nextTick(function () { val.promise.then(resumeCont); });
                        var iterResult = when.defer();
                        iterResult.promise.then(function (res) {
                            if (0 === switched) {
                                result.resolver.resolve(res);
                            }
                        });
                        switched -= 1;
                        iterResult.resolver.resolve(k(val.value));
                    };
                    deferred.promise.then(resumeCont);
                    return result.promise;
                };
            },

            id: function (x) {
                return x;
            },

            errorSuccessCallback: function (errBack, sucBack) {
                return function (err, res) {
                    if (err) {
                        return errBack(err);
                    } else {
                        return sucBack(res);
                    }
                };
            }
        };

        return function (innerMonad) {
            var contT = Object.create(base);
            contT.innerMonad = innerMonad;
            return contT;
        };
    };
}());


text:
@interface UIButton (VTMagic)<VTMagicReuseProtocol>

@end

redpajama_set_name: RedPajamaGithub


text:
$:.unshift(File.dirname(__FILE__) + '/../lib')
require 'test/unit'
RAILS_ENV = "test"
require File.expand_path(File.join(File.dirname(__FILE__), '../../../../config/environment.rb'))

module StoreTestCase
  @@allowed_handle = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'
  @@allowed_nonce = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"

  def _gen_nonce
    OpenID::CryptUtil.random_string(8, @@allowed_nonce)
  end

  def _gen_handle(n)
    OpenID::CryptUtil.random_string(n, @@allowed_handle)
  end

  def _gen_secret(n, chars=nil)
    OpenID::CryptUtil.random_string(n, chars)
  end

  def _gen_assoc(issued, lifetime=600)
    secret = _gen_secret(20)
    handle = _gen_handle(128)
    OpenID::Association.new(handle, secret, Time.now + issued, lifetime,
                            'HMAC-SHA1')
  end

  def _check_retrieve(url, handle=nil, expected=nil)
    ret_assoc = @store.get_association(url, handle)

    if expected.nil?
      assert_nil(ret_assoc)
    else
      assert_equal(expected, ret_assoc)
      assert_equal(expected.handle, ret_assoc.handle)
      assert_equal(expected.secret, ret_assoc.secret)
    end
  end

  def _check_remove(url, handle, expected)
    present = @store.remove_association(url, handle)
    assert_equal(expected, present)
  end

  def test_store
    server_url = "http://www.myopenid.com/openid"
    assoc = _gen_assoc(issued=0)

    # Make sure that a missing association returns no result
    _check_retrieve(server_url)

    # Check that after storage, getting returns the same result
    @store.store_association(server_url, assoc)
    _check_retrieve(server_url, nil, assoc)

    # more than once
    _check_retrieve(server_url, nil, assoc)

    # Storing more than once has no ill effect
    @store.store_association(server_url, assoc)
    _check_retrieve(server_url, nil, assoc)

    # Removing an association that does not exist returns not present
    _check_remove(server_url, assoc.handle + 'x', false)

    # Removing an association that does not exist returns not present
    _check_remove(server_url + 'x', assoc.handle, false)

    # Removing an association that is present returns present
    _check_remove(server_url, assoc.handle, true)

    # but not present on subsequent calls
    _check_remove(server_url, assoc.handle, false)

    # Put assoc back in the store
    @store.store_association(server_url, assoc)

    # More recent and expires after assoc
    assoc2 = _gen_assoc(issued=1)
    @store.store_association(server_url, assoc2)

    # After storing an association with a different handle, but the
    # same server_url, the handle with the later expiration is returned.
    _check_retrieve(server_url, nil, assoc2)

    # We can still retrieve the older association
    _check_retrieve(server_url, assoc.handle, assoc)

    # Plus we can retrieve the association with the later expiration
    # explicitly
    _check_retrieve(server_url, assoc2.handle, assoc2)

    # More recent, and expires earlier than assoc2 or assoc. Make sure
    # that we're picking the one with the latest issued date and not
    # taking into account the expiration.
    assoc3 = _gen_assoc(issued=2, lifetime=100)
    @store.store_association(server_url, assoc3)

    _check_retrieve(server_url, nil, assoc3)
    _check_retrieve(server_url, assoc.handle, assoc)
    _check_retrieve(server_url, assoc2.handle, assoc2)
    _check_retrieve(server_url, assoc3.handle, assoc3)

    _check_remove(server_url, assoc2.handle, true)

    _check_retrieve(server_url, nil, assoc3)
    _check_retrieve(server_url, assoc.handle, assoc)
    _check_retrieve(server_url, assoc2.handle, nil)
    _check_retrieve(server_url, assoc3.handle, assoc3)

    _check_remove(server_url, assoc2.handle, false)
    _check_remove(server_url, assoc3.handle, true)

    _check_retrieve(server_url, nil, assoc)
    _check_retrieve(server_url, assoc.handle, assoc)
    _check_retrieve(server_url, assoc2.handle, nil)
    _check_retrieve(server_url, assoc3.handle, nil)

    _check_remove(server_url, assoc2.handle, false)
    _check_remove(server_url, assoc.handle, true)
    _check_remove(server_url, assoc3.handle, false)

    _check_retrieve(server_url, nil, nil)
    _check_retrieve(server_url, assoc.handle, nil)
    _check_retrieve(server_url, assoc2.handle, nil)
    _check_retrieve(server_url, assoc3.handle, nil)

    _check_remove(server_url, assoc2.handle, false)
    _check_remove(server_url, assoc.handle, false)
    _check_remove(server_url, assoc3.handle, false)

    assocValid1 = _gen_assoc(-3600, 7200)
    assocValid2 = _gen_assoc(-5)
    assocExpired1 = _gen_assoc(-7200, 3600)
    assocExpired2 = _gen_assoc(-7200, 3600)

    @store.cleanup_associations
    @store.store_association(server_url + '1', assocValid1)
    @store.store_association(server_url + '1', assocExpired1)
    @store.store_association(server_url + '2', assocExpired2)
    @store.store_association(server_url + '3', assocValid2)

    cleaned = @store.cleanup_associations()
    assert_equal(2, cleaned, "cleaned up associations")
  end

  def _check_use_nonce(nonce, expected, server_url, msg='')
    stamp, salt = OpenID::Nonce::split_nonce(nonce)
    actual = @store.use_nonce(server_url, stamp, salt)
    assert_equal(expected, actual, msg)
  end

  def test_nonce
    server_url = "http://www.myopenid.com/openid"
    [server_url, ''].each{|url|
      nonce1 = OpenID::Nonce::mk_nonce

      _check_use_nonce(nonce1, true, url, "#{url}: nonce allowed by default")
      _check_use_nonce(nonce1, false, url, "#{url}: nonce not allowed twice")
      _check_use_nonce(nonce1, false, url, "#{url}: nonce not allowed third time")

      # old nonces shouldn't pass
      old_nonce = OpenID::Nonce::mk_nonce(3600)
      _check_use_nonce(old_nonce, false, url, "Old nonce #{old_nonce.inspect} passed")

    }

    now = Time.now.to_i
    old_nonce1 = OpenID::Nonce::mk_nonce(now - 20000)
    old_nonce2 = OpenID::Nonce::mk_nonce(now - 10000)
    recent_nonce = OpenID::Nonce::mk_nonce(now - 600)

    orig_skew = OpenID::Nonce.skew
    OpenID::Nonce.skew = 0
    count = @store.cleanup_nonces
    OpenID::Nonce.skew = 1000000
    ts, salt = OpenID::Nonce::split_nonce(old_nonce1)
    assert(@store.use_nonce(server_url, ts, salt), "oldnonce1")
    ts, salt = OpenID::Nonce::split_nonce(old_nonce2)
    assert(@store.use_nonce(server_url, ts, salt), "oldnonce2")
    ts, salt = OpenID::Nonce::split_nonce(recent_nonce)
    assert(@store.use_nonce(server_url, ts, salt), "recent_nonce")

    OpenID::Nonce.skew = 1000
    cleaned = @store.cleanup_nonces
    assert_equal(2, cleaned, "Cleaned #{cleaned} nonces")

    OpenID::Nonce.skew = 100000
    ts, salt = OpenID::Nonce::split_nonce(old_nonce1)
    assert(@store.use_nonce(server_url, ts, salt), "oldnonce1 after cleanup")
    ts, salt = OpenID::Nonce::split_nonce(old_nonce2)
    assert(@store.use_nonce(server_url, ts, salt), "oldnonce2 after cleanup")
    ts, salt = OpenID::Nonce::split_nonce(recent_nonce)
    assert(!@store.use_nonce(server_url, ts, salt), "recent_nonce after cleanup")

    OpenID::Nonce.skew = orig_skew

  end
end

class TestARStore < Test::Unit::TestCase
  include StoreTestCase

  def setup
    @store = ActiveRecordStore.new
  end

end


text:
"""An extensible ASCII table reader and writer.

latex.py:
  Classes to read and write LaTeX tables

:Copyright: Smithsonian Astrophysical Observatory (2011)
:Author: Tom Aldcroft (aldcroft@head.cfa.harvard.edu)
"""

from __future__ import absolute_import, division, print_function

import re

from ...extern import six
from ...extern.six.moves import zip
from . import core

latexdicts = {'AA':  {'tabletype': 'table',
                      'header_start': r'\hline \hline', 'header_end': r'\hline',
                      'data_end': r'\hline'},
              'doublelines': {'tabletype': 'table',
                              'header_start': r'\hline \hline', 'header_end': r'\hline\hline',
                              'data_end': r'\hline\hline'},
              'template': {'tabletype': 'tabletype', 'caption': 'caption',
                           'tablealign': 'tablealign',
                           'col_align': 'col_align', 'preamble': 'preamble',
                           'header_start': 'header_start',
                           'header_end': 'header_end', 'data_start': 'data_start',
                           'data_end': 'data_end', 'tablefoot': 'tablefoot',
                           'units': {'col1': 'unit of col1', 'col2': 'unit of col2'}}
              }

RE_COMMENT = re.compile(r'(?<!\\)%')  # % character but not \%

def add_dictval_to_list(adict, key, alist):
    '''
    Add a value from a dictionary to a list

    Parameters
    ----------
    adict : dictionary
    key : hashable
    alist : list
        List where value should be added
    '''
    if key in adict:
        if isinstance(adict[key], six.string_types):
            alist.append(adict[key])
        else:
            alist.extend(adict[key])

def find_latex_line(lines, latex):
    '''
    Find the first line which matches a patters

    Parameters
    ----------
    lines : list
        List of strings
    latex : str
        Search pattern

    Returns
    -------
    line_num : int, None
        Line number. Returns None, if no match was found

    '''
    re_string = re.compile(latex.replace('\\', '\\\\'))
    for i, line in enumerate(lines):
        if re_string.match(line):
            return i
    else:
        return None

class LatexSplitter(core.BaseSplitter):
    '''Split LaTeX table date. Default delimiter is `&`.
    '''
    delimiter = '&'

    def __call__(self, lines):
        last_line = RE_COMMENT.split(lines[-1])[0].strip()
        if not last_line.endswith(r'\\'):
            lines[-1] = last_line + r'\\'

        return super(LatexSplitter, self).__call__(lines)

    def process_line(self, line):
        """Remove whitespace at the beginning or end of line. Also remove
        \\ at end of line"""
        line = RE_COMMENT.split(line)[0]
        line = line.strip()
        if line.endswith(r'\\'):
            line = line.rstrip(r'\\')
        else:
            raise core.InconsistentTableError(r'Lines in LaTeX table have to end with \\')
        return line

    def process_val(self, val):
        """Remove whitespace and {} at the beginning or end of value."""
        val = val.strip()
        if val and (val[0] == '{') and (val[-1] == '}'):
            val = val[1:-1]
        return val

    def join(self, vals):
        '''Join values together and add a few extra spaces for readability'''
        delimiter = ' ' + self.delimiter + ' '
        return delimiter.join(x.strip() for x in vals) + r' \\'

class LatexHeader(core.BaseHeader):
    '''Class to read the header of Latex Tables'''
    header_start = r'\begin{tabular}'
    splitter_class = LatexSplitter

    def start_line(self, lines):
        line = find_latex_line(lines, self.header_start)
        if line is not None:
            return line + 1
        else:
            return None

    def _get_units(self):
        units = {}
        col_units = [col.info.unit for col in self.cols]
        for name, unit in zip(self.colnames, col_units):
            if unit:
                try:
                    units[name] = unit.to_string(format='latex_inline')
                except AttributeError:
                    units[name] = unit
        return units

    def write(self, lines):
        if 'col_align' not in self.latex:
            self.latex['col_align'] = len(self.cols) * 'c'
        if 'tablealign' in self.latex:
            align = '[' + self.latex['tablealign'] + ']'
        else:
            align = ''
        lines.append(r'\begin{' + self.latex['tabletype'] + r'}' + align)
        add_dictval_to_list(self.latex, 'preamble', lines)
        if 'caption' in self.latex:
            lines.append(r'\caption{' + self.latex['caption'] + '}')
        lines.append(self.header_start + r'{' + self.latex['col_align'] + r'}')
        add_dictval_to_list(self.latex, 'header_start', lines)
        lines.append(self.splitter.join(self.colnames))
        units = self._get_units()
        if 'units' in self.latex:
            units.update(self.latex['units'])
        if units:
            lines.append(self.splitter.join([units.get(name, ' ') for name in self.colnames]))
        add_dictval_to_list(self.latex, 'header_end', lines)

class LatexData(core.BaseData):
    '''Class to read the data in LaTeX tables'''
    data_start = None
    data_end = r'\end{tabular}'
    splitter_class = LatexSplitter

    def start_line(self, lines):
        if self.data_start:
            return find_latex_line(lines, self.data_start)
        else:
            return self.header.start_line(lines) + 1

    def end_line(self, lines):
        if self.data_end:
            return find_latex_line(lines, self.data_end)
        else:
            return None

    def write(self, lines):
        add_dictval_to_list(self.latex, 'data_start', lines)
        core.BaseData.write(self, lines)
        add_dictval_to_list(self.latex, 'data_end', lines)
        lines.append(self.data_end)
        add_dictval_to_list(self.latex, 'tablefoot', lines)
        lines.append(r'\end{' + self.latex['tabletype'] + '}')

class Latex(core.BaseReader):
    r'''Write and read LaTeX tables.

    This class implements some LaTeX specific commands.  Its main
    purpose is to write out a table in a form that LaTeX can compile. It
    is beyond the scope of this class to implement every possible LaTeX
    command, instead the focus is to generate a syntactically valid
    LaTeX tables.

    This class can also read simple LaTeX tables (one line per table
    row, no ``\multicolumn`` or similar constructs), specifically, it
    can read the tables that it writes.

    Reading a LaTeX table, the following keywords are accepted:

    **ignore_latex_commands** :
        Lines starting with these LaTeX commands will be treated as comments (i.e. ignored).

    When writing a LaTeX table, the some keywords can customize the
    format.  Care has to be taken here, because python interprets ``\\``
    in a string as an escape character.  In order to pass this to the
    output either format your strings as raw strings with the ``r``
    specifier or use a double ``\\\\``.

    Examples::

        caption = r'My table \label{mytable}'
        caption = 'My table \\\\label{mytable}'

    **latexdict** : Dictionary of extra parameters for the LaTeX output

        * tabletype : used for first and last line of table.
            The default is ``\\begin{table}``.  The following would generate a table,
            which spans the whole page in a two-column document::

                ascii.write(data, sys.stdout, Writer = ascii.Latex,
                            latexdict = {'tabletype': 'table*'})

        * tablealign : positioning of table in text.
            The default is not to specify a position preference in the text.
            If, e.g. the alignment is ``ht``, then the LaTeX will be ``\\begin{table}[ht]``.

        * col_align : Alignment of columns
            If not present all columns will be centered.

        * caption : Table caption (string or list of strings)
            This will appear above the table as it is the standard in
            many scientific publications.  If you prefer a caption below
            the table, just write the full LaTeX command as
            ``latexdict['tablefoot'] = r'\caption{My table}'``

        * preamble, header_start, header_end, data_start, data_end, tablefoot: Pure LaTeX
            Each one can be a string or a list of strings. These strings
            will be inserted into the table without any further
            processing. See the examples below.

        * units : dictionary of strings
            Keys in this dictionary should be names of columns. If
            present, a line in the LaTeX table directly below the column
            names is added, which contains the values of the
            dictionary. Example::

              from astropy.io import ascii
              data = {'name': ['bike', 'car'], 'mass': [75,1200], 'speed': [10, 130]}
              ascii.write(data, Writer=ascii.Latex,
                               latexdict = {'units': {'mass': 'kg', 'speed': 'km/h'}})

            If the column has no entry in the ``units`` dictionary, it defaults
            to the **unit** attribute of the column. If this attribute is not
            specified (i.e. it is None), the unit will be written as ``' '``.

        Run the following code to see where each element of the
        dictionary is inserted in the LaTeX table::

            from astropy.io import ascii
            data = {'cola': [1,2], 'colb': [3,4]}
            ascii.write(data, Writer=ascii.Latex, latexdict=ascii.latex.latexdicts['template'])

        Some table styles are predefined in the dictionary
        ``ascii.latex.latexdicts``. The following generates in table in
        style preferred by A&A and some other journals::

            ascii.write(data, Writer=ascii.Latex, latexdict=ascii.latex.latexdicts['AA'])

        As an example, this generates a table, which spans all columns
        and is centered on the page::

            ascii.write(data, Writer=ascii.Latex, col_align='|lr|',
                        latexdict={'preamble': r'\begin{center}',
                                   'tablefoot': r'\end{center}',
                                   'tabletype': 'table*'})

    **caption** : Set table caption
        Shorthand for::

            latexdict['caption'] = caption

    **col_align** : Set the column alignment.
        If not present this will be auto-generated for centered
        columns. Shorthand for::

            latexdict['col_align'] = col_align

    '''
    _format_name = 'latex'
    _io_registry_format_aliases = ['latex']
    _io_registry_suffix = '.tex'
    _description = 'LaTeX table'

    header_class = LatexHeader
    data_class = LatexData

    def __init__(self, ignore_latex_commands=['hline', 'vspace', 'tableline'],
                 latexdict={}, caption='', col_align=None):

        super(Latex, self).__init__()

        self.latex = {}
        # The latex dict drives the format of the table and needs to be shared
        # with data and header
        self.header.latex = self.latex
        self.data.latex = self.latex
        self.latex['tabletype'] = 'table'
        self.latex.update(latexdict)
        if caption:
            self.latex['caption'] = caption
        if col_align:
            self.latex['col_align'] = col_align

        self.ignore_latex_commands = ignore_latex_commands
        self.header.comment = '%|' + '|'.join(
            [r'\\' + command for command in self.ignore_latex_commands])
        self.data.comment = self.header.comment

    def write(self, table=None):
        self.header.start_line = None
        self.data.start_line = None
        return core.BaseReader.write(self, table=table)

class AASTexHeaderSplitter(LatexSplitter):
    r'''Extract column names from a `deluxetable`_.

    This splitter expects the following LaTeX code **in a single line**:

        \tablehead{\colhead{col1} & ... & \colhead{coln}}
    '''
    def __call__(self, lines):
        return super(LatexSplitter, self).__call__(lines)

    def process_line(self, line):
        """extract column names from tablehead
        """
        line = line.split('%')[0]
        line = line.replace(r'\tablehead', '')
        line = line.strip()
        if (line[0] == '{') and (line[-1] == '}'):
            line = line[1:-1]
        else:
            raise core.InconsistentTableError(r'\tablehead is missing {}')
        return line.replace(r'\colhead', '')

    def join(self, vals):
        return ' & '.join([r'\colhead{' + str(x) + '}' for x in vals])

class AASTexHeader(LatexHeader):
    r'''In a `deluxetable
    <http://fits.gsfc.nasa.gov/standard30/deluxetable.sty>`_ some header
    keywords differ from standard LaTeX.

    This header is modified to take that into account.
    '''
    header_start = r'\tablehead'
    splitter_class = AASTexHeaderSplitter

    def start_line(self, lines):
        return find_latex_line(lines, r'\tablehead')

    def write(self, lines):
        if 'col_align' not in self.latex:
            self.latex['col_align'] = len(self.cols) * 'c'
        if 'tablealign' in self.latex:
            align = '[' + self.latex['tablealign'] + ']'
        else:
            align = ''
        lines.append(r'\begin{' + self.latex['tabletype'] + r'}{' + self.latex['col_align'] + r'}'
                       + align)
        add_dictval_to_list(self.latex, 'preamble', lines)
        if 'caption' in self.latex:
            lines.append(r'\tablecaption{' + self.latex['caption'] + '}')
        tablehead = ' & '.join([r'\colhead{' + name + '}' for name in self.colnames])
        units = self._get_units()
        if 'units' in self.latex:
            units.update(self.latex['units'])
        if units:
            tablehead += r'\\ ' + self.splitter.join([units.get(name, ' ')
                                                      for name in self.colnames])
        lines.append(r'\tablehead{' + tablehead + '}')

class AASTexData(LatexData):
    r'''In a `deluxetable`_ the data is enclosed in `\startdata` and `\enddata`
    '''
    data_start = r'\startdata'
    data_end = r'\enddata'

    def start_line(self, lines):
        return find_latex_line(lines, self.data_start) + 1

    def write(self, lines):
        lines.append(self.data_start)
        lines_length_initial = len(lines)
        core.BaseData.write(self, lines)
        # To remove extra space(s) and // appended which creates an extra new line
        # in the end.
        if len(lines) > lines_length_initial:
            # we compile separately because py2.6 doesn't have a flags keyword in re.sub
            re_final_line = re.compile(r'\s* \\ \\ \s* $', flags=re.VERBOSE)
            lines[-1] = re.sub(re_final_line, '', lines[-1])
        lines.append(self.data_end)
        add_dictval_to_list(self.latex, 'tablefoot', lines)
        lines.append(r'\end{' + self.latex['tabletype'] + r'}')

class AASTex(Latex):
    '''Write and read AASTeX tables.

    This class implements some AASTeX specific commands.
    AASTeX is used for the AAS (American Astronomical Society)
    publications like ApJ, ApJL and AJ.

    It derives from the ``Latex`` reader and accepts the same
    keywords.  However, the keywords ``header_start``, ``header_end``,
    ``data_start`` and ``data_end`` in ``latexdict`` have no effect.
    '''

    _format_name = 'aastex'
    _io_registry_format_aliases = ['aastex']
    _io_registry_suffix = ''  # AASTex inherits from Latex, so override this class attr
    _description = 'AASTeX deluxetable used for AAS journals'

    header_class = AASTexHeader
    data_class = AASTexData

    def __init__(self, **kwargs):
        super(AASTex, self).__init__(**kwargs)
        # check if tabletype was explicitly set by the user
        if not (('latexdict' in kwargs) and ('tabletype' in kwargs['latexdict'])):
            self.latex['tabletype'] = 'deluxetable'


text:
![TVV gif](http://g.recordit.co/sfgN4mDlZn.gif)

## [CHANGELOG](./CHANGELOG.md)

## Version 0.1.0

This is a variant of [Varying Vagrant Vagrants](https://github.com/Varying-Vagrant-Vagrants/VVV) with a couple of customizations specifically for a local development environment.  TVV is installed in the same way you would install [VVV](https://github.com/varying-vagrant-vagrants/vvv) and this setup assumes you are using the [VVV Site Wizard](https://github.com/bradp/vv) to create additional Wordpress installations.

## Installation

1. Install [Virtualbox](https://www.virtualbox.org/wiki/Downloads)

2. Install [Vagrant](https://www.vagrantup.com/downloads.html)

3. Install vagrant-hostsupdater and vagrant-triggers

        vagrant plugin install vagrant-hostsupdater && vagrant plugin install vagrant-triggers

1. Clone TVV into your home folder:

        cd ~ && git clone https://github.com/tpkemme/TVV.git tvagrant

2. Navigate to cloned directory:

        cd ~/tvagrant

3. Start vagrant:

        vagrant up

That's it!

# Features

So how is TVV different from VVV and why is better suited for local development?  TVV is a fork of VVV and will still have access to future VVV releases.  The main difference between TVV and VVV is the addition of a custom skin for the VVV Dashboard.  While the dark theme for the dashboard is aesthetically pleasing, it also includes a link to phpMyAdmin and a Search and Replace plugin that's already configured to work with VVV's MySQL database.  Awesome!

#### Custom Dashboard for VVV

The [VVV Dashboard](https://github.com/topdown/VVV-Dashboard) is already installed in TVV and after your first "vagrant up", you can view it at [vvv.dev](http://vvv.dev).  The dashboard is currently being included within this repo as a [git-subrepo](https://github.com/ingydotnet/git-subrepo) so that I can make changes to the dashboard and still pull updates from the remote repository.  The beauty of using git-subrepo is that I can include repositories inside this repo and the user will only need to clone TVV once to get everything.  Installation of git-subrepo is not necessary and is only needed if you need to pull updates from the VVV Dashboard repo.

#### Dark Theme
The first change made to the VVV Dashboard was giving it a dark theme so it looks better on a computer running primarily dark-themed applications.  The colors were chosen to be easy on the eyes and to match most IDE's dark material themes.  I removed a lot of unecessary borders and tables so the information is easier to digest.

#### Search and Replace DB Plugin
Besides changing the style, I added a link in the top menu of the dashboard to the [Database Search and Replace Script](https://interconnectit.com/products/search-and-replace-for-wordpress-databases/).  This script is configured to work with any database that was created using the [VVV Site Wizard](https://github.com/bradp/vv) (i.e. username: 'root', password: 'root' ).  All you need to do to use it is specify which database you would like to run the script on.

Although this Search and Replace plugin is top notch, it's important to note that it should only be used in development environments.  TVV should never be used in a production environment because this plugin gives anyone with access to your network the ability to alter the databases for all your sites running through TVV.  This tool is specifically for developers working in a local environment.

# Updating

### Updating VVV

Because TVV is a fork of VVV containing a git-subrepo of the VVV Dashboard, you can use TVV as a replacement for VVV while still receiving updates for both repositories.  Pulling upstream updates from VVV into TVV is very simple.  First you'll need to add the original VVV repository as an upstream source.  You'll only need to do step one and two the first time.

1. Navigate to the TVV directory you previously cloned.

        cd ~/tvagrant

2. Add the original VVV repo as an upstream source.

        git remote add upstream https://github.com/varying-vagrant-vagrants/vvv.git

3. Pull updates from remote repo (in this case, the master branch)

    git pull upstream master

### Updating VVV Dashboard

In order to pull upstream updates from the VVV Dashboard, you will need to install [git-subrepo](https://github.com/ingydotnet/git-subrepo).  Although the git-subrepo project is quite new and not widely used, I highly recommend it as an alternative to git submodules and subtrees.

After installing git-subrepo, you can update the VVV Dashboard whenever commits are made to the remote repo:

1.  Navigate to the base TVV directory

        cd ~/tvagrant

2.  Pull changes from remote subrepo

        git subrepo pull www/default/dashboard


text:
<?php

namespace OrcaServices\Heartbeat\Heartbeat\Sensor;

use Migrations\Migrations;
use OrcaServices\Heartbeat\Heartbeat\Sensor;

/**
 * DB Up to Date Sensor
 *
 * This sensor depends on the CakePHP Migrations plugin.
 * Make sure the plugin is loaded before calling this sensor.
 *
 * @link https://github.com/cakephp/migrations/
 */
class DBUpToDate extends Sensor
{
    /**
     * Migration status indicating the migration was executed successfully
     */
    const MIGRATION_STATUS_UP = 'up';

    /**
     * {@inheritdoc}
     */
    protected function _getStatus()
    {
        $dbMigrated = true;
        try {
            $migrations = new Migrations();
            $status = $migrations->status();
            $lastStatus = array_pop($status);
            if ($lastStatus['status'] !== self::MIGRATION_STATUS_UP) {
                $dbMigrated = false;
            }
        } catch (\Exception $e) {
            $dbMigrated = false;
        }

        return $dbMigrated;
    }
}


text:
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

namespace Microsoft.Azure.WebJobs.Extensions.AuthenticationEvents
{
    /// Types of events to listen for and attach a function to.
    public enum EventType
    {
        /// When a token is issued, this event will be called and the ability to append claim to the token is enabled via the response.
        OnTokenIssuanceStart
    }
}


text:
using Rabbit.Transport.Simple.Tcp.Framing;
using System;
using System.Net.Sockets;

namespace Rabbit.Transport.Simple.Tcp.Client
{
    public sealed class TcpSocketSaeaClientConfiguration
    {
        public TcpSocketSaeaClientConfiguration()
        {
            InitialPooledBufferCount = 4;
            ReceiveBufferSize = 8192;
            SendBufferSize = 8192;
            ReceiveTimeout = TimeSpan.Zero;
            SendTimeout = TimeSpan.Zero;
            NoDelay = true;
            LingerState = new LingerOption(false, 0); // The socket will linger for x seconds after Socket.Close is called.

            FrameBuilder = new LengthPrefixedFrameBuilder();
        }

        public int InitialPooledBufferCount { get; set; }
        public int ReceiveBufferSize { get; set; }
        public int SendBufferSize { get; set; }
        public TimeSpan ReceiveTimeout { get; set; }
        public TimeSpan SendTimeout { get; set; }
        public bool NoDelay { get; set; }
        public LingerOption LingerState { get; set; }

        public IFrameBuilder FrameBuilder { get; set; }
    }
}


text:
namespace Free.Core.Drawing
{
	///
	/// Describes a single-chunked image of 1, 3 (RGB) or 4 (RGBX) unsigned 8-bit integers pre pixel.
	///
	/// <remarks>HINT: If the <see cref="PlanarConfiguration"/> of the image is <see cref="PlanarConfiguration.Separated"/>
	/// then there is a chunk for each channel.</remarks>
	public class Image8 : ChunkedImage8
	{
		#region Constructors
		///
		/// Creates a new instance.
		///
		/// <param name="width">The width of the image in pixels. Must be greater zero (0)</param>
		/// <param name="height">The height of the image in pixels. Must be greater zero (0)</param>
		/// <param name="RGB">Indicates whether the image shall be RGB or single-channeled.</param>
		/// <param name="extraChannel">Indicates whether a RGB image shall contain an extra channel, e.g. for alpha. Must be false if <paramref name="RGB"/> is false.</param>
		/// <param name="init">Indicates whether the <see cref="Bits"/> array shall be created and initialize with zeros (0).</param>
		/// <param name="planarConfig">The planar configuration to use. Single-channel images must be continuously.</param>
		public Image8(int width, int height, bool RGB, bool extraChannel, bool init, PlanarConfiguration planarConfig = PlanarConfiguration.Continuously) :
			base(width, height, RGB, extraChannel, init, height, planarConfig)
		{ }

		///
		/// Creates a new instance with a specified color.
		///
		/// <param name="width">The width of the image in pixels. Must be greater zero (0)</param>
		/// <param name="height">The height of the image in pixels. Must be greater zero (0)</param>
		/// <param name="RGB">Indicates whether the image shall be RGB or single-channeled.</param>
		/// <param name="extraChannel">Indicates whether a RGB image shall contain an extra channel, e.g. for alpha. Must be false if <paramref name="RGB"/> is false.</param>
		/// <param name="initColor">The initial color. ( Layout: [I] or [R] [G] [B] (/[X]) )</param>
		/// <param name="planarConfig">The planar configuration to use. Single-channel images must be continuously.</param>
		public Image8(int width, int height, bool RGB, bool extraChannel, byte[] initColor, PlanarConfiguration planarConfig = PlanarConfiguration.Continuously) :
			base(width, height, RGB, extraChannel, initColor, height, planarConfig)
		{ }

		///
		/// Creates a new instance with a specified color.
		///
		/// <param name="width">The width of the image in pixels. Must be greater zero (0)</param>
		/// <param name="height">The height of the image in pixels. Must be greater zero (0)</param>
		/// <param name="RGB">Indicates whether the image shall be RGB or single-channeled.</param>
		/// <param name="extraChannel">Indicates whether a RGB image shall contain an extra channel, e.g. for alpha. Must be false if <paramref name="RGB"/> is false.</param>
		/// <param name="initColor">The initial color. (Layout: 0xXXRRGGBB)</param>
		/// <param name="planarConfig">The planar configuration to use. Single-channel images must be continuously.</param>
		public Image8(int width, int height, bool RGB, bool extraChannel, int initColor = 0, PlanarConfiguration planarConfig = PlanarConfiguration.Continuously) :
			base(width, height, RGB, extraChannel, initColor, height, planarConfig)
		{ }
		#endregion
	}
}


text:
<!--
Description: entry link rel
Expect:      not bozo and entries[0]['links'][0]['rel'] == 'alternate'
-->
<feed version="0.3" xmlns="http://purl.org/atom/ns#">
<entry>
  <link rel="alternate" type="text/html" href="http://www.example.com/"/>
</entry>
</feed>


text:
'use strict';
/*jshint asi: true*/

var test = require('tap').test
    , util = require('util')
    , redeyed = require('..')

function inspect(obj) {
    return util.inspect(obj, false, 5, true)
}

test('properly handles script level return -- no blow up', function (t) {
    var code = [
        , 'return 1;'
    ].join('\n')
        , opts = {Keyword: {'return': '%:^'}}
        , expected = '\n%return^ 1;'
        , res = redeyed(code, opts).code

    t.equals(res, expected, inspect(code) + ' opts: ' + inspect(opts) + ' => ' + inspect(expected))
    t.end()
})


text:
<?php

namespace Magento\Framework\Exception;

/**
 * State Exception
 */
class StateException extends LocalizedException
{
}


text:
<?xml version="1.0" encoding="utf-8"?>
<prince.app.ccm.util.MultiSwipeRefreshLayout xmlns:android="http://schemas.android.com/apk/res/android"
	android:id="@+id/swiperefresh"
	android:layout_width="fill_parent"
	android:layout_height="fill_parent"
	android:background="#D4CFCF"
	android:orientation="vertical" >

	<RelativeLayout
		android:id="@+id/web_frame"
		android:layout_width="match_parent"
		android:layout_height="match_parent" >

		<WebView
			android:id="@+id/text_audio_main"
			android:layout_width="fill_parent"
			android:layout_height="fill_parent"
			android:gravity="center_horizontal|center_vertical"
			android:text="@string/test"
			android:visibility="gone" />

		<ProgressBar
			android:id="@+id/web_progress"
			style="@android:style/Widget.Holo.ProgressBar.Large"
			android:layout_width="wrap_content"
			android:layout_height="wrap_content"
			android:layout_centerHorizontal="true"
			android:layout_centerVertical="true"
			android:indeterminate="true" />

		<ViewStub
			android:id="@+id/stub_internet"
			android:layout_width="fill_parent"
			android:layout_height="fill_parent"
			android:inflatedId="@+id/viewstub_internet"
			android:layout="@layout/layout_nowifi" />
	</RelativeLayout>

</prince.app.ccm.util.MultiSwipeRefreshLayout>


text:
Ti=Non-Compete/Exclusivity

sec=The {_Founders} shall dedicate substantially all of their professional time to the {_Company} for so long as they are in office and shall be bound by a customary 12-month non-compete obligation after that which may be waived by the {_Company} and shall otherwise be compensated by monthly payments equal to {50%_of_their_average_monthly_salary} during their last 12 months of employment.

=[G/Z/ol/Base]


text:
[`<termox/widget/widgets/selectable.hpp>`](../../../include/termox/widget/widgets/selectable.hpp)

Wraps a Widget type to provide select() and unselect() methods to change the
display, so the wrapped Widget is 'Selectable'. The default implementation
applies the Inverse trait to the entire widget on select(), and uninverts on
unselect().

```cpp
template <typename Widget_t,
          typename Select_method   = void,
          typename Unselect_method = void>
class Selectable : public Widget_t {
   public:
    // Provide two methods to modify Widget_t. Signature: void(Widget_t&);
    Selectable(Select_method s, Unselect_method u);

    Selectable(Select_method s, Unselect_method u, typename Widget_t::Parameters p);

    // Change visual to mark as selected.
    void select();

    // Change visual to mark as unselected.
    void unselect();
};
```


text:
import PropTypes from 'prop-types';
import React from 'react';

import createHandler from './createHandler';
import GestureHandlerPropTypes from './GestureHandlerPropTypes';
import PlatformConstants from './PlatformConstants';

export const TapGestureHandler = createHandler(
  'TapGestureHandler',
  {
    ...GestureHandlerPropTypes,
    maxDurationMs: PropTypes.number,
    maxDelayMs: PropTypes.number,
    numberOfTaps: PropTypes.number,
    maxDeltaX: PropTypes.number,
    maxDeltaY: PropTypes.number,
    maxDist: PropTypes.number,
    minPointers: PropTypes.number,
  },
  {}
);

export const FlingGestureHandler = createHandler(
  'FlingGestureHandler',
  {
    ...GestureHandlerPropTypes,
    numberOfPointers: PropTypes.number,
    direction: PropTypes.number,
  },
  {}
);

class ForceTouchFallback extends React.Component {
  componentDidMount() {
    console.warn(
      'ForceTouchGestureHandler is not available on this platform. Please use ForceTouchGestureHandler.forceTouchAvailable to conditionally render other components that would provide a fallback behavior specific to your usecase'
    );
  }
  render() {
    return this.props.children;
  }
}

export const ForceTouchGestureHandler =
  PlatformConstants && PlatformConstants.forceTouchAvailable
    ? createHandler(
        'ForceTouchGestureHandler',
        {
          ...GestureHandlerPropTypes,
          minForce: PropTypes.number,
          maxForce: PropTypes.number,
          feedbackOnActivation: PropTypes.bool,
        },
        {}
      )
    : ForceTouchFallback;

ForceTouchGestureHandler.forceTouchAvailable =
  (PlatformConstants && PlatformConstants.forceTouchAvailable) || false;

export const LongPressGestureHandler = createHandler(
  'LongPressGestureHandler',
  {
    ...GestureHandlerPropTypes,
    minDurationMs: PropTypes.number,
    maxDist: PropTypes.number,
  },
  {}
);

function validatePanGestureHandlerProps(props) {
  if (props.minDeltaX && props.activeOffsetX) {
    throw new Error(
      `It's not supported use minDeltaX with activeOffsetXStart or activeOffsetXEnd`
    );
  }
  if (props.maxDeltaX && props.failOffsetX) {
    throw new Error(
      `It's not supported use minDeltaX with activeOffsetXStart or activeOffsetXEnd`
    );
  }
  if (props.minDeltaY && props.activeOffsetY) {
    throw new Error(
      `It's not supported use minDeltaX with activeOffsetYStart or activeOffsetYEnd`
    );
  }
  if (props.maxDeltaY && props.failOffsetY) {
    throw new Error(
      `It's not supported use minDeltaX with activeOffsetYStart or activeOffsetYEnd`
    );
  }
  if (
    Array.isArray(props.activeOffsetX) &&
    (props.activeOffsetX[0] > 0 || props.activeOffsetX[1] < 0)
  ) {
    throw new Error(
      `First element of activeOffsetX should be negative, a the second one should be positive`
    );
  }

  if (
    Array.isArray(props.activeOffsetY) &&
    (props.activeOffsetY[0] > 0 || props.activeOffsetY[1] < 0)
  ) {
    throw new Error(
      `First element of activeOffsetY should be negative, a the second one should be positive`
    );
  }

  if (
    Array.isArray(props.failOffsetX) &&
    (props.failOffsetX[0] > 0 || props.failOffsetX[1] < 0)
  ) {
    throw new Error(
      `First element of failOffsetX should be negative, a the second one should be positive`
    );
  }

  if (
    Array.isArray(props.failOffsetY) &&
    (props.failOffsetY[0] > 0 || props.failOffsetY[1] < 0)
  ) {
    throw new Error(
      `First element of failOffsetY should be negative, a the second one should be positive`
    );
  }
}

function transformPanGestureHandlerProps(props) {
  const res = { ...props };
  if (props.minDeltaX !== undefined) {
    delete res['minDeltaX'];
    res.activeOffsetXStart = -props.minDeltaX;
    res.activeOffsetXEnd = props.minDeltaX;
  }
  if (props.maxDeltaX !== undefined) {
    delete res['maxDeltaX'];
    res.failOffsetXStart = -props.maxDeltaX;
    res.failOffsetXEnd = props.maxDeltaX;
  }
  if (props.minOffsetX !== undefined) {
    delete res['minOffsetX'];
    if (props.minOffsetX < 0) {
      res.activeOffsetXStart = props.minOffsetX;
    } else {
      res.activeOffsetXEnd = props.minOffsetX;
    }
  }

  if (props.minDeltaY !== undefined) {
    delete res['minDeltaY'];
    res.activeOffsetYStart = -props.minDeltaY;
    res.activeOffsetYEnd = props.minDeltaY;
  }
  if (props.maxDeltaY !== undefined) {
    delete res['maxDeltaY'];
    res.failOffsetYStart = -props.maxDeltaY;
    res.failOffsetYEnd = props.maxDeltaY;
  }

  if (props.minOffsetY !== undefined) {
    delete res['minOffsetY'];
    if (props.minOffsetY < 0) {
      res.activeOffsetYStart = props.minOffsetY;
    } else {
      res.activeOffsetYEnd = props.minOffsetY;
    }
  }

  if (props.activeOffsetX !== undefined) {
    delete res['activeOffsetX'];
    if (Array.isArray(props.activeOffsetX)) {
      res.activeOffsetXStart = props.activeOffsetX[0];
      res.activeOffsetXEnd = props.activeOffsetX[1];
    } else if (props.activeOffsetX < 0) {
      res.activeOffsetXStart = props.activeOffsetX;
    } else {
      res.activeOffsetXEnd = props.activeOffsetX;
    }
  }

  if (props.activeOffsetY !== undefined) {
    delete res['activeOffsetY'];
    if (Array.isArray(props.activeOffsetY)) {
      res.activeOffsetYStart = props.activeOffsetY[0];
      res.activeOffsetYEnd = props.activeOffsetY[1];
    } else if (props.activeOffsetY < 0) {
      res.activeOffsetYStart = props.activeOffsetY;
    } else {
      res.activeOffsetYEnd = props.activeOffsetY;
    }
  }

  if (props.failOffsetX !== undefined) {
    delete res['failOffsetX'];
    if (Array.isArray(props.failOffsetX)) {
      res.failOffsetXStart = props.failOffsetX[0];
      res.failOffsetXEnd = props.failOffsetX[1];
    } else if (props.failOffsetX < 0) {
      res.failOffsetXStart = props.failOffsetX;
    } else {
      res.failOffsetXEnd = props.failOffsetX;
    }
  }

  if (props.failOffsetY !== undefined) {
    delete res['failOffsetY'];
    if (Array.isArray(props.failOffsetY)) {
      res.failOffsetYStart = props.failOffsetY[0];
      res.failOffsetYEnd = props.failOffsetY[1];
    } else if (props.failOffsetY < 0) {
      res.failOffsetYStart = props.failOffsetY;
    } else {
      res.failOffsetYEnd = props.failOffsetY;
    }
  }

  return res;
}

function managePanProps(props) {
  if (__DEV__) {
    validatePanGestureHandlerProps(props);
  }
  return transformPanGestureHandlerProps(props);
}

export const PanGestureHandler = createHandler(
  'PanGestureHandler',
  {
    ...GestureHandlerPropTypes,
    activeOffsetY: PropTypes.oneOfType([
      PropTypes.number,
      PropTypes.arrayOf(PropTypes.number),
    ]),
    activeOffsetX: PropTypes.oneOfType([
      PropTypes.number,
      PropTypes.arrayOf(PropTypes.number),
    ]),
    failOffsetY: PropTypes.oneOfType([
      PropTypes.number,
      PropTypes.arrayOf(PropTypes.number),
    ]),
    failOffsetX: PropTypes.oneOfType([
      PropTypes.number,
      PropTypes.arrayOf(PropTypes.number),
    ]),
    minDist: PropTypes.number,
    minVelocity: PropTypes.number,
    minVelocityX: PropTypes.number,
    minVelocityY: PropTypes.number,
    minPointers: PropTypes.number,
    maxPointers: PropTypes.number,
    avgTouches: PropTypes.bool,
  },
  {},
  managePanProps,
  {
    activeOffsetYStart: true,
    activeOffsetYEnd: true,
    activeOffsetXStart: true,
    activeOffsetXEnd: true,
    failOffsetYStart: true,
    failOffsetYEnd: true,
    failOffsetXStart: true,
    failOffsetXEnd: true,
  }
);
export const PinchGestureHandler = createHandler(
  'PinchGestureHandler',
  GestureHandlerPropTypes,
  {}
);
export const RotationGestureHandler = createHandler(
  'RotationGestureHandler',
  GestureHandlerPropTypes,
  {}
);


text:
https://www.nytimes.com/interactive/2018/05/09/nyregion/subway-crisis-mta-decisions-signals-rules.html
https://www.kaggle.com/pmarcelino/comprehensive-data-exploration-with-python


text:
.views-exposed-form .views-exposed-widget {
  float: left; /* LTR */
  padding: .5em 1em 0 0; /* LTR */
}

.views-exposed-form .views-exposed-widget .form-submit {
  margin-top: 1.6em;
}

.views-exposed-form .form-item,
.views-exposed-form .form-submit {
  margin-top: 0;
  margin-bottom: 0;
}

.views-exposed-form label {
  font-weight: bold;
}

.views-exposed-widgets {
  margin-bottom: .5em;
}

/* table style column align */
.views-align-left {
  text-align: left;
}
.views-align-right {
  text-align: right;
}
.views-align-center {
  text-align: center;
}

/* Remove the border on tbody that system puts in */
.views-view-grid tbody {
  border-top: none;
}

.view .progress-disabled {
  float: none;
}


text:
require 'net/ssh'
require 'net/ssh/telnet'
require 'net/sftp'

require 'ghaki/account/base'
require 'ghaki/core_ext/file/with_temp'

require 'ghaki/net_ssh/errors'
require 'ghaki/net_ssh/shell'
require 'ghaki/net_ssh/ftp'
require 'ghaki/net_ssh/logger'
require 'ghaki/net_ssh/telnet'


text:
{% extends "base.html" %}

{% block title -%}
Wiki Editor
{%- endblock title %}

{% block content %}

Editor

Preview

    <form method="post" class="form">
      {{ form.hidden_tag() }}
      {{ input(form.title, placeholder="Title", class="span7") }}
      {{ input(form.body, placeholder="Markdown", class="span7", rows="20") }}
      {{ input(form.tags, placeholder="Tags (comma separated)", class="span7") }}

          Preview

          Cancel
          <button class="btn btn-success" type="submit">Save</button>

    </form>

{% endblock content %}
{% block sidebar %}

Show all Uploads

Upload Resources

You can upload image for markdown

<form name="form" action="" method="POST" enctype="multipart/form-data">

<input id="file" type="file" size="45" name="file" class="input" />

Please select a file and click Upload button

<button class="button" id="buttonUpload" onclick="return ajaxFileUpload();">Upload</button>

  </form>

Upload list:

{% endblock sidebar %}
{% block postscripts -%}
{{ super() }}
$('#previewlink').on('click', function() {
  var $form = $('.form');
  var $inputs = $form.find('input, textarea, button');
  var $pre = $('#preview');
  var bodycontent = 'title: preview\n\n' + $form.find('textarea').val();
  $inputs.prop('disabled', true);
  $pre.removeClass('alert').removeClass('alert-error').html("Loading...");
  $.ajax({
    url: "{{ url_for('preview') }}",
    type: "POST",
    data: { body: bodycontent },
    success: function(msg) {
      $pre.html(msg);
      console.log(msg);
    },
    error: function() {
  $pre.addClass('alert').addClass('alert-error');
      $pre.html('There was a problem with the preview.');
    },
    complete: function() {
      $inputs.prop('disabled', false);
    }
  });
});
$('#previewbtn').on('click', function(event) {
  event.preventDefault();
  $('#previewlink').click();
});

function ajaxFileUpload(){
        $("#loading")
        .ajaxStart(function(){
            $(this).show();
        })
        .ajaxComplete(function(){
            $(this).hide();
        });
        $.ajaxFileUpload({
            url:'/upload/',
            type:'post',
            secureuri:false,
            fileElementId:'file',
            dataType: 'json',
            success: function (data, status)
            {
                var file = $("#file")
                file.after(file.clone().val(""));
                file.remove();
                console.log(data);
                console.log(status);
                if(!data.error){
                    var list = $("#uploadlist");
                    var html = list.html() + data.filename + "'s url:\n" + data.url + "
";
                    list.html(html)
                }else{
                    alert("upload error.")
              }
            },
            error: function (data, status, e)
            {
                alert("upload error.")
            }
        })
        return false;
    }
{%- endblock postscripts %}


text:
import bottle
import cpyutils.restutils
import lxciaas.nodeinfo as nodeinfo
import lxciaas.lxclib as lxclib

ACCESS_USER = ""
ACCESS_TOKEN = "1e55e7e4-b369-45d2-a484-eb6ea48d5da1"
DEFAULT_LXC_PM_HOST_LISTEN_PORT = 15909
DEFAULT_LXC_PM_HOST_LISTEN_HOST = "localhost"

app = cpyutils.restutils.get_app()

# In order to use basic authentication, it is enough to use the decorator
#   @bottle.auth_basic(check_pass)
# right before the name of the function that is serving a REST route. Then
# You'd have to implement your authentication backend (e.g. database or a
# simple password check)'
def check_pass(username, password):
    return (username == ACCESS_USER) and (password == ACCESS_TOKEN)

# https://realpython.com/blog/python/api-integration-in-python/
# http://docs.python-requests.org/en/master/user/authentication/

@app.route('/')
@bottle.auth_basic(check_pass)
def get_server_info():
    return cpyutils.restutils.response_json(nodeinfo.get_nodeinfo())

@app.route('/:containername', method = 'POST')
def create_container(containername):
    json = {}
    request = None
    try:
        # Grab the JSON info from the request
        json = bottle.request.json

        # We have to add the name of the container because lxclib.Request needs it
        json['name'] = containername

        # Let's try to build a request from the JSON received
        request = lxclib.Request.from_json(json)
    except:
        json = None
        request = None

    if json is None or request is None:
        return cpyutils.restutils.error(400, "Malformed json")

    # Now we'll try to create the container
    result, container = request.create_container()
    if result:
        return True, cpyutils.restutils.response_json(container.serialize())
    else:
        return False, cpyutils.restutils.error(409, "failed to create a container")

@app.route('/:containername', method = 'GET')
def get_container(containername):
    container = lxclib.LXCContainer.pick(containername)
    if container is None:
        return cpyutils.restutils.error(404, "container %s not found" % containername)
    return cpyutils.restutils.response_json(container.serialize())

@app.route('/:containername/start', method = 'PUT')
def start_container(containername):
    container = lxclib.LXCContainer.pick(containername)
    if container is None:
        return cpyutils.restutils.error(404, "container %s not found" % containername)
    container.start()
    if container.state == "RUNNING":
        return cpyutils.restutils.response_json(container.serialize())
    return cpyutils.restutils.error(500, "could not start container %s" % containername)

@app.route('/:containername/stop', method = 'PUT')
def stop_container(containername):
    container = lxclib.LXCContainer.pick(containername)
    if container is None:
        return cpyutils.restutils.error(404, "container %s not found" % containername)
    container.stop()
    if container.state == "STOPPED":
        return cpyutils.restutils.response_json(container.serialize())
    return cpyutils.restutils.error(500, "could not stop container %s" % containername)

@app.route('/:containername', method = 'DELETE')
def delete_container(containername):
    container = lxclib.LXCContainer.pick(containername)
    if container is None:
        return cpyutils.restutils.error(404, "container %s not found" % containername)
    container.destroy()
    if not container.defined:
        return cpyutils.restutils.response_json(container.serialize())
    return cpyutils.restutils.error(500, "could not destroy container %s" % containername)

def main_function():
    r = lxclib.Request.from_json({"name": "mycont", "distribution":"ubuntu","release":"xenial"})
    created, container = r.create_container()
    print created
    print container
    if created:
        container.start()
    #cpyutils.restutils.run(DEFAULT_LXC_PM_HOST_LISTEN_HOST, DEFAULT_LXC_PM_HOST_LISTEN_PORT)

if __name__ == '__main__':
    main_function()


text:
package org.apache.spark.sql.execution.streaming

import java.io.{InterruptedIOException, IOException}
import java.util.UUID
import java.util.concurrent.{CountDownLatch, TimeUnit}
import java.util.concurrent.atomic.AtomicReference
import java.util.concurrent.locks.ReentrantLock

import scala.collection.mutable.{Map => MutableMap}
import scala.collection.mutable.ArrayBuffer
import scala.util.control.NonFatal

import org.apache.hadoop.fs.Path

import org.apache.spark.internal.Logging
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.encoders.RowEncoder
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeMap, CurrentBatchTimestamp, CurrentDate, CurrentTimestamp}
import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan}
import org.apache.spark.sql.execution.{QueryExecution, SQLExecution}
import org.apache.spark.sql.execution.command.StreamingExplainCommand
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.streaming._
import org.apache.spark.util.{Clock, UninterruptibleThread, Utils}

/** States for [[StreamExecution]]'s lifecycle. */
trait State
case object INITIALIZING extends State
case object ACTIVE extends State
case object TERMINATED extends State

/**
 * Manages the execution of a streaming Spark SQL query that is occurring in a separate thread.
 * Unlike a standard query, a streaming query executes repeatedly each time new data arrives at any
 * [[Source]] present in the query plan. Whenever new data arrives, a [[QueryExecution]] is created
 * and the results are committed transactionally to the given [[Sink]].
 *
 * @param deleteCheckpointOnStop whether to delete the checkpoint if the query is stopped without
 *                               errors
 */
class StreamExecution(
    override val sparkSession: SparkSession,
    override val name: String,
    private val checkpointRoot: String,
    analyzedPlan: LogicalPlan,
    val sink: Sink,
    val trigger: Trigger,
    val triggerClock: Clock,
    val outputMode: OutputMode,
    deleteCheckpointOnStop: Boolean)
  extends StreamingQuery with ProgressReporter with Logging {

  import org.apache.spark.sql.streaming.StreamingQueryListener._

  private val pollingDelayMs = sparkSession.sessionState.conf.streamingPollingDelay

  private val minBatchesToRetain = sparkSession.sessionState.conf.minBatchesToRetain
  require(minBatchesToRetain > 0, "minBatchesToRetain has to be positive")

  /**
   * A lock used to wait/notify when batches complete. Use a fair lock to avoid thread starvation.
   */
  private val awaitBatchLock = new ReentrantLock(true)
  private val awaitBatchLockCondition = awaitBatchLock.newCondition()

  private val initializationLatch = new CountDownLatch(1)
  private val startLatch = new CountDownLatch(1)
  private val terminationLatch = new CountDownLatch(1)

  val resolvedCheckpointRoot = {
    val checkpointPath = new Path(checkpointRoot)
    val fs = checkpointPath.getFileSystem(sparkSession.sessionState.newHadoopConf())
    checkpointPath.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toUri.toString
  }

  /**
   * Tracks how much data we have processed and committed to the sink or state store from each
   * input source.
   * Only the scheduler thread should modify this field, and only in atomic steps.
   * Other threads should make a shallow copy if they are going to access this field more than
   * once, since the field's value may change at any time.
   */
  @volatile
  var committedOffsets = new StreamProgress

  /**
   * Tracks the offsets that are available to be processed, but have not yet be committed to the
   * sink.
   * Only the scheduler thread should modify this field, and only in atomic steps.
   * Other threads should make a shallow copy if they are going to access this field more than
   * once, since the field's value may change at any time.
   */
  @volatile
  var availableOffsets = new StreamProgress

  /** The current batchId or -1 if execution has not yet been initialized. */
  protected var currentBatchId: Long = -1

  /** Metadata associated with the whole query */
  protected val streamMetadata: StreamMetadata = {
    val metadataPath = new Path(checkpointFile("metadata"))
    val hadoopConf = sparkSession.sessionState.newHadoopConf()
    StreamMetadata.read(metadataPath, hadoopConf).getOrElse {
      val newMetadata = new StreamMetadata(UUID.randomUUID.toString)
      StreamMetadata.write(newMetadata, metadataPath, hadoopConf)
      newMetadata
    }
  }

  /** Metadata associated with the offset seq of a batch in the query. */
  protected var offsetSeqMetadata = OffsetSeqMetadata(batchWatermarkMs = 0, batchTimestampMs = 0,
    conf = Map(SQLConf.SHUFFLE_PARTITIONS.key ->
      sparkSession.conf.get(SQLConf.SHUFFLE_PARTITIONS).toString))

  override val id: UUID = UUID.fromString(streamMetadata.id)

  override val runId: UUID = UUID.randomUUID

  /**
   * Pretty identified string of printing in logs. Format is
   * If name is set "queryName [id = xyz, runId = abc]" else "[id = xyz, runId = abc]"
   */
  private val prettyIdString =
    Option(name).map(_ + " ").getOrElse("") + s"[id = $id, runId = $runId]"

  /**
   * All stream sources present in the query plan. This will be set when generating logical plan.
   */
  @volatile protected var sources: Seq[Source] = Seq.empty

  /**
   * A list of unique sources in the query plan. This will be set when generating logical plan.
   */
  @volatile private var uniqueSources: Seq[Source] = Seq.empty

  override lazy val logicalPlan: LogicalPlan = {
    assert(microBatchThread eq Thread.currentThread,
      "logicalPlan must be initialized in StreamExecutionThread " +
        s"but the current thread was ${Thread.currentThread}")
    var nextSourceId = 0L
    val toExecutionRelationMap = MutableMap[StreamingRelation, StreamingExecutionRelation]()
    val _logicalPlan = analyzedPlan.transform {
      case streamingRelation@StreamingRelation(dataSource, _, output) =>
        toExecutionRelationMap.getOrElseUpdate(streamingRelation, {
          // Materialize source to avoid creating it in every batch
          val metadataPath = s"$resolvedCheckpointRoot/sources/$nextSourceId"
          val source = dataSource.createSource(metadataPath)
          nextSourceId += 1
          // We still need to use the previous `output` instead of `source.schema` as attributes in
          // "df.logicalPlan" has already used attributes of the previous `output`.
          StreamingExecutionRelation(source, output)
        })
    }
    sources = _logicalPlan.collect { case s: StreamingExecutionRelation => s.source }
    uniqueSources = sources.distinct
    _logicalPlan
  }

  private val triggerExecutor = trigger match {
    case t: ProcessingTime => ProcessingTimeExecutor(t, triggerClock)
    case OneTimeTrigger => OneTimeExecutor()
    case _ => throw new IllegalStateException(s"Unknown type of trigger: $trigger")
  }

  /** Defines the internal state of execution */
  private val state = new AtomicReference[State](INITIALIZING)

  @volatile
  var lastExecution: IncrementalExecution = _

  /** Holds the most recent input data for each source. */
  protected var newData: Map[Source, DataFrame] = _

  @volatile
  private var streamDeathCause: StreamingQueryException = null

  /* Get the call site in the caller thread; will pass this into the micro batch thread */
  private val callSite = Utils.getCallSite()

  /** Used to report metrics to coda-hale. This uses id for easier tracking across restarts. */
  lazy val streamMetrics = new MetricsReporter(
    this, s"spark.streaming.${Option(name).getOrElse(id)}")

  /**
   * The thread that runs the micro-batches of this stream. Note that this thread must be
   * [[org.apache.spark.util.UninterruptibleThread]] to workaround KAFKA-1894: interrupting a
   * running `KafkaConsumer` may cause endless loop.
   */
  val microBatchThread =
    new StreamExecutionThread(s"stream execution thread for $prettyIdString") {
      override def run(): Unit = {
        // To fix call site like "run at <unknown>:0", we bridge the call site from the caller
        // thread to this micro batch thread
        sparkSession.sparkContext.setCallSite(callSite)
        runBatches()
      }
    }

  /**
   * A write-ahead-log that records the offsets that are present in each batch. In order to ensure
   * that a given batch will always consist of the same data, we write to this log *before* any
   * processing is done.  Thus, the Nth record in this log indicated data that is currently being
   * processed and the N-1th entry indicates which offsets have been durably committed to the sink.
   */
  val offsetLog = new OffsetSeqLog(sparkSession, checkpointFile("offsets"))

  /**
   * A log that records the batch ids that have completed. This is used to check if a batch was
   * fully processed, and its output was committed to the sink, hence no need to process it again.
   * This is used (for instance) during restart, to help identify which batch to run next.
   */
  val batchCommitLog = new BatchCommitLog(sparkSession, checkpointFile("commits"))

  /** Whether all fields of the query have been initialized */
  private def isInitialized: Boolean = state.get != INITIALIZING

  /** Whether the query is currently active or not */
  override def isActive: Boolean = state.get != TERMINATED

  /** Returns the [[StreamingQueryException]] if the query was terminated by an exception. */
  override def exception: Option[StreamingQueryException] = Option(streamDeathCause)

  /** Returns the path of a file with `name` in the checkpoint directory. */
  private def checkpointFile(name: String): String =
    new Path(new Path(resolvedCheckpointRoot), name).toUri.toString

  /**
   * Starts the execution. This returns only after the thread has started and [[QueryStartedEvent]]
   * has been posted to all the listeners.
   */
  def start(): Unit = {
    logInfo(s"Starting $prettyIdString. Use $resolvedCheckpointRoot to store the query checkpoint.")
    microBatchThread.setDaemon(true)
    microBatchThread.start()
    startLatch.await()  // Wait until thread started and QueryStart event has been posted
  }

  /**
   * Repeatedly attempts to run batches as data arrives.
   *
   * Note that this method ensures that [[QueryStartedEvent]] and [[QueryTerminatedEvent]] are
   * posted such that listeners are guaranteed to get a start event before a termination.
   * Furthermore, this method also ensures that [[QueryStartedEvent]] event is posted before the
   * `start()` method returns.
   */
  private def runBatches(): Unit = {
    try {
      sparkSession.sparkContext.setJobGroup(runId.toString, getBatchDescriptionString,
        interruptOnCancel = true)
      if (sparkSession.sessionState.conf.streamingMetricsEnabled) {
        sparkSession.sparkContext.env.metricsSystem.registerSource(streamMetrics)
      }

      // `postEvent` does not throw non fatal exception.
      postEvent(new QueryStartedEvent(id, runId, name))

      // Unblock starting thread
      startLatch.countDown()

      // While active, repeatedly attempt to run batches.
      SparkSession.setActiveSession(sparkSession)

      updateStatusMessage("Initializing sources")
      // force initialization of the logical plan so that the sources can be created
      logicalPlan

      // Isolated spark session to run the batches with.
      val sparkSessionToRunBatches = sparkSession.cloneSession()
      // Adaptive execution can change num shuffle partitions, disallow
      sparkSessionToRunBatches.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false")
      offsetSeqMetadata = OffsetSeqMetadata(batchWatermarkMs = 0, batchTimestampMs = 0,
        conf = Map(SQLConf.SHUFFLE_PARTITIONS.key ->
          sparkSessionToRunBatches.conf.get(SQLConf.SHUFFLE_PARTITIONS.key)))

      if (state.compareAndSet(INITIALIZING, ACTIVE)) {
        // Unblock `awaitInitialization`
        initializationLatch.countDown()

        triggerExecutor.execute(() => {
          startTrigger()

          if (isActive) {
            reportTimeTaken("triggerExecution") {
              if (currentBatchId < 0) {
                // We'll do this initialization only once
                populateStartOffsets(sparkSessionToRunBatches)
                sparkSession.sparkContext.setJobDescription(getBatchDescriptionString)
                logDebug(s"Stream running from $committedOffsets to $availableOffsets")
              } else {
                constructNextBatch()
              }
              if (dataAvailable) {
                currentStatus = currentStatus.copy(isDataAvailable = true)
                updateStatusMessage("Processing new data")
                runBatch(sparkSessionToRunBatches)
              }
            }
            // Report trigger as finished and construct progress object.
            finishTrigger(dataAvailable)
            if (dataAvailable) {
              // Update committed offsets.
              batchCommitLog.add(currentBatchId)
              committedOffsets ++= availableOffsets
              logDebug(s"batch ${currentBatchId} committed")
              // We'll increase currentBatchId after we complete processing current batch's data
              currentBatchId += 1
              sparkSession.sparkContext.setJobDescription(getBatchDescriptionString)
            } else {
              currentStatus = currentStatus.copy(isDataAvailable = false)
              updateStatusMessage("Waiting for data to arrive")
              Thread.sleep(pollingDelayMs)
            }
          }
          updateStatusMessage("Waiting for next trigger")
          isActive
        })
        updateStatusMessage("Stopped")
      } else {
        // `stop()` is already called. Let `finally` finish the cleanup.
      }
    } catch {
      case _: InterruptedException | _: InterruptedIOException if state.get == TERMINATED =>
        // interrupted by stop()
        updateStatusMessage("Stopped")
      case e: IOException if e.getMessage != null
        && e.getMessage.startsWith(classOf[InterruptedException].getName)
        && state.get == TERMINATED =>
        // This is a workaround for HADOOP-12074: `Shell.runCommand` converts `InterruptedException`
        // to `new IOException(ie.toString())` before Hadoop 2.8.
        updateStatusMessage("Stopped")
      case e: Throwable =>
        streamDeathCause = new StreamingQueryException(
          toDebugString(includeLogicalPlan = isInitialized),
          s"Query $prettyIdString terminated with exception: ${e.getMessage}",
          e,
          committedOffsets.toOffsetSeq(sources, offsetSeqMetadata).toString,
          availableOffsets.toOffsetSeq(sources, offsetSeqMetadata).toString)
        logError(s"Query $prettyIdString terminated with error", e)
        updateStatusMessage(s"Terminated with exception: ${e.getMessage}")
        // Rethrow the fatal errors to allow the user using `Thread.UncaughtExceptionHandler` to
        // handle them
        if (!NonFatal(e)) {
          throw e
        }
    } finally {
      // Release latches to unblock the user codes since exception can happen in any place and we
      // may not get a chance to release them
      startLatch.countDown()
      initializationLatch.countDown()

      try {
        stopSources()
        state.set(TERMINATED)
        currentStatus = status.copy(isTriggerActive = false, isDataAvailable = false)

        // Update metrics and status
        sparkSession.sparkContext.env.metricsSystem.removeSource(streamMetrics)

        // Notify others
        sparkSession.streams.notifyQueryTermination(StreamExecution.this)
        postEvent(
          new QueryTerminatedEvent(id, runId, exception.map(_.cause).map(Utils.exceptionString)))

        // Delete the temp checkpoint only when the query didn't fail
        if (deleteCheckpointOnStop && exception.isEmpty) {
          val checkpointPath = new Path(resolvedCheckpointRoot)
          try {
            val fs = checkpointPath.getFileSystem(sparkSession.sessionState.newHadoopConf())
            fs.delete(checkpointPath, true)
          } catch {
            case NonFatal(e) =>
              // Deleting temp checkpoint folder is best effort, don't throw non fatal exceptions
              // when we cannot delete them.
              logWarning(s"Cannot delete $checkpointPath", e)
          }
        }
      } finally {
        awaitBatchLock.lock()
        try {
          // Wake up any threads that are waiting for the stream to progress.
          awaitBatchLockCondition.signalAll()
        } finally {
          awaitBatchLock.unlock()
        }
        terminationLatch.countDown()
      }
    }
  }

  /**
   * Populate the start offsets to start the execution at the current offsets stored in the sink
   * (i.e. avoid reprocessing data that we have already processed). This function must be called
   * before any processing occurs and will populate the following fields:
   *  - currentBatchId
   *  - committedOffsets
   *  - availableOffsets
   *  The basic structure of this method is as follows:
   *
   *  Identify (from the offset log) the offsets used to run the last batch
   *  IF last batch exists THEN
   *    Set the next batch to be executed as the last recovered batch
   *    Check the commit log to see which batch was committed last
   *    IF the last batch was committed THEN
   *      Call getBatch using the last batch start and end offsets
   *      // ^^^^ above line is needed since some sources assume last batch always re-executes
   *      Setup for a new batch i.e., start = last batch end, and identify new end
   *    DONE
   *  ELSE
   *    Identify a brand new batch
   *  DONE
   */
  private def populateStartOffsets(sparkSessionToRunBatches: SparkSession): Unit = {
    offsetLog.getLatest() match {
      case Some((latestBatchId, nextOffsets)) =>
        /* First assume that we are re-executing the latest known batch
         * in the offset log */
        currentBatchId = latestBatchId
        availableOffsets = nextOffsets.toStreamProgress(sources)
        /* Initialize committed offsets to a committed batch, which at this
         * is the second latest batch id in the offset log. */
        offsetLog.get(latestBatchId - 1).foreach { secondLatestBatchId =>
          committedOffsets = secondLatestBatchId.toStreamProgress(sources)
        }

        // update offset metadata
        nextOffsets.metadata.foreach { metadata =>
          val shufflePartitionsSparkSession: Int =
            sparkSessionToRunBatches.conf.get(SQLConf.SHUFFLE_PARTITIONS)
          val shufflePartitionsToUse = metadata.conf.getOrElse(SQLConf.SHUFFLE_PARTITIONS.key, {
            // For backward compatibility, if # partitions was not recorded in the offset log,
            // then ensure it is not missing. The new value is picked up from the conf.
            logWarning("Number of shuffle partitions from previous run not found in checkpoint. "
              + s"Using the value from the conf, $shufflePartitionsSparkSession partitions.")
            shufflePartitionsSparkSession
          })
          offsetSeqMetadata = OffsetSeqMetadata(
            metadata.batchWatermarkMs, metadata.batchTimestampMs,
            metadata.conf + (SQLConf.SHUFFLE_PARTITIONS.key -> shufflePartitionsToUse.toString))
          // Update conf with correct number of shuffle partitions
          sparkSessionToRunBatches.conf.set(
            SQLConf.SHUFFLE_PARTITIONS.key, shufflePartitionsToUse.toString)
        }

        /* identify the current batch id: if commit log indicates we successfully processed the
         * latest batch id in the offset log, then we can safely move to the next batch
         * i.e., committedBatchId + 1 */
        batchCommitLog.getLatest() match {
          case Some((latestCommittedBatchId, _)) =>
            if (latestBatchId == latestCommittedBatchId) {
              /* The last batch was successfully committed, so we can safely process a
               * new next batch but first:
               * Make a call to getBatch using the offsets from previous batch.
               * because certain sources (e.g., KafkaSource) assume on restart the last
               * batch will be executed before getOffset is called again. */
              availableOffsets.foreach { ao: (Source, Offset) =>
                val (source, end) = ao
                if (committedOffsets.get(source).map(_ != end).getOrElse(true)) {
                  val start = committedOffsets.get(source)
                  source.getBatch(start, end)
                }
              }
              currentBatchId = latestCommittedBatchId + 1
              committedOffsets ++= availableOffsets
              // Construct a new batch be recomputing availableOffsets
              constructNextBatch()
            } else if (latestCommittedBatchId < latestBatchId - 1) {
              logWarning(s"Batch completion log latest batch id is " +
                s"${latestCommittedBatchId}, which is not trailing " +
                s"batchid $latestBatchId by one")
            }
          case None => logInfo("no commit log present")
        }
        logDebug(s"Resuming at batch $currentBatchId with committed offsets " +
          s"$committedOffsets and available offsets $availableOffsets")
      case None => // We are starting this stream for the first time.
        logInfo(s"Starting new streaming query.")
        currentBatchId = 0
        constructNextBatch()
    }
  }

  /**
   * Returns true if there is any new data available to be processed.
   */
  private def dataAvailable: Boolean = {
    availableOffsets.exists {
      case (source, available) =>
        committedOffsets
            .get(source)
            .map(committed => committed != available)
            .getOrElse(true)
    }
  }

  /**
   * Queries all of the sources to see if any new data is available. When there is new data the
   * batchId counter is incremented and a new log entry is written with the newest offsets.
   */
  private def constructNextBatch(): Unit = {
    // Check to see what new data is available.
    val hasNewData = {
      awaitBatchLock.lock()
      try {
        val latestOffsets: Map[Source, Option[Offset]] = uniqueSources.map { s =>
          updateStatusMessage(s"Getting offsets from $s")
          reportTimeTaken("getOffset") {
            (s, s.getOffset)
          }
        }.toMap
        availableOffsets ++= latestOffsets.filter { case (s, o) => o.nonEmpty }.mapValues(_.get)

        if (dataAvailable) {
          true
        } else {
          noNewData = true
          false
        }
      } finally {
        awaitBatchLock.unlock()
      }
    }
    if (hasNewData) {
      var batchWatermarkMs = offsetSeqMetadata.batchWatermarkMs
      // Update the eventTime watermark if we find one in the plan.
      if (lastExecution != null) {
        lastExecution.executedPlan.collect {
          case e: EventTimeWatermarkExec if e.eventTimeStats.value.count > 0 =>
            logDebug(s"Observed event time stats: ${e.eventTimeStats.value}")
            e.eventTimeStats.value.max - e.delayMs
        }.headOption.foreach { newWatermarkMs =>
          if (newWatermarkMs > batchWatermarkMs) {
            logInfo(s"Updating eventTime watermark to: $newWatermarkMs ms")
            batchWatermarkMs = newWatermarkMs
          } else {
            logDebug(
              s"Event time didn't move: $newWatermarkMs < " +
                s"$batchWatermarkMs")
          }
        }
      }
      offsetSeqMetadata = offsetSeqMetadata.copy(
        batchWatermarkMs = batchWatermarkMs,
        batchTimestampMs = triggerClock.getTimeMillis()) // Current batch timestamp in milliseconds

      updateStatusMessage("Writing offsets to log")
      reportTimeTaken("walCommit") {
        assert(offsetLog.add(
          currentBatchId,
          availableOffsets.toOffsetSeq(sources, offsetSeqMetadata)),
          s"Concurrent update to the log. Multiple streaming jobs detected for $currentBatchId")
        logInfo(s"Committed offsets for batch $currentBatchId. " +
          s"Metadata ${offsetSeqMetadata.toString}")

        // NOTE: The following code is correct because runBatches() processes exactly one
        // batch at a time. If we add pipeline parallelism (multiple batches in flight at
        // the same time), this cleanup logic will need to change.

        // Now that we've updated the scheduler's persistent checkpoint, it is safe for the
        // sources to discard data from the previous batch.
        val prevBatchOff = offsetLog.get(currentBatchId - 1)
        if (prevBatchOff.isDefined) {
          prevBatchOff.get.toStreamProgress(sources).foreach {
            case (src, off) => src.commit(off)
          }
        }

        // It is now safe to discard the metadata beyond the minimum number to retain.
        // Note that purge is exclusive, i.e. it purges everything before the target ID.
        if (minBatchesToRetain < currentBatchId) {
          offsetLog.purge(currentBatchId - minBatchesToRetain)
          batchCommitLog.purge(currentBatchId - minBatchesToRetain)
        }
      }
    } else {
      awaitBatchLock.lock()
      try {
        // Wake up any threads that are waiting for the stream to progress.
        awaitBatchLockCondition.signalAll()
      } finally {
        awaitBatchLock.unlock()
      }
    }
  }

  /**
   * Processes any data available between `availableOffsets` and `committedOffsets`.
   * @param sparkSessionToRunBatch Isolated [[SparkSession]] to run this batch with.
   */
  private def runBatch(sparkSessionToRunBatch: SparkSession): Unit = {
    // Request unprocessed data from all sources.
    newData = reportTimeTaken("getBatch") {
      availableOffsets.flatMap {
        case (source, available)
          if committedOffsets.get(source).map(_ != available).getOrElse(true) =>
          val current = committedOffsets.get(source)
          val batch = source.getBatch(current, available)
          logDebug(s"Retrieving data from $source: $current -> $available")
          Some(source -> batch)
        case _ => None
      }
    }

    // A list of attributes that will need to be updated.
    var replacements = new ArrayBuffer[(Attribute, Attribute)]
    // Replace sources in the logical plan with data that has arrived since the last batch.
    val withNewSources = logicalPlan transform {
      case StreamingExecutionRelation(source, output) =>
        newData.get(source).map { data =>
          val newPlan = data.logicalPlan
          assert(output.size == newPlan.output.size,
            s"Invalid batch: ${Utils.truncatedString(output, ",")} != " +
            s"${Utils.truncatedString(newPlan.output, ",")}")
          replacements ++= output.zip(newPlan.output)
          newPlan
        }.getOrElse {
          LocalRelation(output)
        }
    }

    // Rewire the plan to use the new attributes that were returned by the source.
    val replacementMap = AttributeMap(replacements)
    val triggerLogicalPlan = withNewSources transformAllExpressions {
      case a: Attribute if replacementMap.contains(a) => replacementMap(a)
      case ct: CurrentTimestamp =>
        CurrentBatchTimestamp(offsetSeqMetadata.batchTimestampMs,
          ct.dataType)
      case cd: CurrentDate =>
        CurrentBatchTimestamp(offsetSeqMetadata.batchTimestampMs,
          cd.dataType, cd.timeZoneId)
    }

    reportTimeTaken("queryPlanning") {
      lastExecution = new IncrementalExecution(
        sparkSessionToRunBatch,
        triggerLogicalPlan,
        outputMode,
        checkpointFile("state"),
        currentBatchId,
        offsetSeqMetadata)
      lastExecution.executedPlan // Force the lazy generation of execution plan
    }

    val nextBatch =
      new Dataset(sparkSessionToRunBatch, lastExecution, RowEncoder(lastExecution.analyzed.schema))

    reportTimeTaken("addBatch") {
      SQLExecution.withNewExecutionId(sparkSessionToRunBatch, lastExecution) {
        sink.addBatch(currentBatchId, nextBatch)
      }
    }

    awaitBatchLock.lock()
    try {
      // Wake up any threads that are waiting for the stream to progress.
      awaitBatchLockCondition.signalAll()
    } finally {
      awaitBatchLock.unlock()
    }
  }

  override protected def postEvent(event: StreamingQueryListener.Event): Unit = {
    sparkSession.streams.postListenerEvent(event)
  }

  /** Stops all streaming sources safely. */
  private def stopSources(): Unit = {
    uniqueSources.foreach { source =>
      try {
        source.stop()
      } catch {
        case NonFatal(e) =>
          logWarning(s"Failed to stop streaming source: $source. Resources may have leaked.", e)
      }
    }
  }

  /**
   * Signals to the thread executing micro-batches that it should stop running after the next
   * batch. This method blocks until the thread stops running.
   */
  override def stop(): Unit = {
    // Set the state to TERMINATED so that the batching thread knows that it was interrupted
    // intentionally
    state.set(TERMINATED)
    if (microBatchThread.isAlive) {
      sparkSession.sparkContext.cancelJobGroup(runId.toString)
      microBatchThread.interrupt()
      microBatchThread.join()
      // microBatchThread may spawn new jobs, so we need to cancel again to prevent a leak
      sparkSession.sparkContext.cancelJobGroup(runId.toString)
    }
    logInfo(s"Query $prettyIdString was stopped")
  }

  /**
   * Blocks the current thread until processing for data from the given `source` has reached at
   * least the given `Offset`. This method is intended for use primarily when writing tests.
   */
  private[sql] def awaitOffset(source: Source, newOffset: Offset): Unit = {
    assertAwaitThread()
    def notDone = {
      val localCommittedOffsets = committedOffsets
      !localCommittedOffsets.contains(source) || localCommittedOffsets(source) != newOffset
    }

    while (notDone) {
      awaitBatchLock.lock()
      try {
        awaitBatchLockCondition.await(100, TimeUnit.MILLISECONDS)
        if (streamDeathCause != null) {
          throw streamDeathCause
        }
      } finally {
        awaitBatchLock.unlock()
      }
    }
    logDebug(s"Unblocked at $newOffset for $source")
  }

  /** A flag to indicate that a batch has completed with no new data available. */
  @volatile private var noNewData = false

  /**
   * Assert that the await APIs should not be called in the stream thread. Otherwise, it may cause
   * dead-lock, e.g., calling any await APIs in `StreamingQueryListener.onQueryStarted` will block
   * the stream thread forever.
   */
  private def assertAwaitThread(): Unit = {
    if (microBatchThread eq Thread.currentThread) {
      throw new IllegalStateException(
        "Cannot wait for a query state from the same thread that is running the query")
    }
  }

  /**
   * Await until all fields of the query have been initialized.
   */
  def awaitInitialization(timeoutMs: Long): Unit = {
    assertAwaitThread()
    require(timeoutMs > 0, "Timeout has to be positive")
    if (streamDeathCause != null) {
      throw streamDeathCause
    }
    initializationLatch.await(timeoutMs, TimeUnit.MILLISECONDS)
    if (streamDeathCause != null) {
      throw streamDeathCause
    }
  }

  override def processAllAvailable(): Unit = {
    assertAwaitThread()
    if (streamDeathCause != null) {
      throw streamDeathCause
    }
    awaitBatchLock.lock()
    try {
      noNewData = false
      while (true) {
        awaitBatchLockCondition.await(10000, TimeUnit.MILLISECONDS)
        if (streamDeathCause != null) {
          throw streamDeathCause
        }
        if (noNewData) {
          return
        }
      }
    } finally {
      awaitBatchLock.unlock()
    }
  }

  override def awaitTermination(): Unit = {
    assertAwaitThread()
    terminationLatch.await()
    if (streamDeathCause != null) {
      throw streamDeathCause
    }
  }

  override def awaitTermination(timeoutMs: Long): Boolean = {
    assertAwaitThread()
    require(timeoutMs > 0, "Timeout has to be positive")
    terminationLatch.await(timeoutMs, TimeUnit.MILLISECONDS)
    if (streamDeathCause != null) {
      throw streamDeathCause
    } else {
      !isActive
    }
  }

  /** Expose for tests */
  def explainInternal(extended: Boolean): String = {
    if (lastExecution == null) {
      "No physical plan. Waiting for data."
    } else {
      val explain = StreamingExplainCommand(lastExecution, extended = extended)
      sparkSession.sessionState.executePlan(explain).executedPlan.executeCollect()
        .map(_.getString(0)).mkString("\n")
    }
  }

  override def explain(extended: Boolean): Unit = {
    // scalastyle:off println
    println(explainInternal(extended))
    // scalastyle:on println
  }

  override def explain(): Unit = explain(extended = false)

  override def toString: String = {
    s"Streaming Query $prettyIdString [state = $state]"
  }

  private def toDebugString(includeLogicalPlan: Boolean): String = {
    val debugString =
      s"""|=== Streaming Query ===
          |Identifier: $prettyIdString
          |Current Committed Offsets: $committedOffsets
          |Current Available Offsets: $availableOffsets
          |
          |Current State: $state
          |Thread State: ${microBatchThread.getState}""".stripMargin
    if (includeLogicalPlan) {
      debugString + s"\n\nLogical Plan:\n$logicalPlan"
    } else {
      debugString
    }
  }

  private def getBatchDescriptionString: String = {
    val batchDescription = if (currentBatchId < 0) "init" else currentBatchId.toString
    Option(name).map(_ + "<br/>").getOrElse("") +
      s"id = $id<br/>runId = $runId<br/>batch = $batchDescription"
  }
}

/**
 * A special thread to run the stream query. Some codes require to run in the StreamExecutionThread
 * and will use `classOf[StreamExecutionThread]` to check.
 */
abstract class StreamExecutionThread(name: String) extends UninterruptibleThread(name)


text:
<?php
	$category_display_label = 'primary';
	$category_object = wp_get_post_terms( get_the_ID(), 'course_category' );
	$category_object = $category_object[0]; ?>

	<?php get_template_part('templates/header'); ?>

		<?php if('design' === $category_object->slug)
				$category_display_label = 'danger';
			echo '
';
				echo do_shortcode( '[course_title class="course-single-title" title_tag="h1"]' );
				echo '' . $category_object->name . '';
			echo '
';
			echo do_shortcode( '[course_media course_id="'. get_the_ID() .'" class="course-video-holder"]' );
		?>

<?php previous_post_link('%link', '« Previous lesson'); ?>

<?php next_post_link('%link', 'Next lesson »'); ?>

			<?php if(is_user_logged_in()) : ?>
				<?php echo do_shortcode( '[course_join_button course_id="'. get_the_ID() .'"]' ); ?>
			<?php else : ?>
				<button class="btn btn-primary" id="course-register" href="#">Signup!</button>
			<?php endif; ?>


text:
	  ___|             ___|  \  |  ___|
	 |      _ \   __| |     |\/ |\___ \
	 |     (   |\__ \ |     |   |      |
	\____|\___/ ____/\____|_|  _|_____/

# About

CosCMS is a simple modular framework for building web application or shell applications.

Modules are distrubuted as profiles.

This is the default profile, it includes a

* Account system
* A blog
* Disqus
* Analytics

# Demo

[Demo Site](https://coscms-demo.os-cms.net/account/login/index)

Login with `admin` / `admin`

# Install Requirements

You will need:

* PHP>=5.5 (Tested with PHP7.0)
* PHP extensions: PDO, PDO-mysql, GD, mbstring, intl
* MySQL>=5.5, and
* Apache2 with mod rewrite module enabled

You can install it else where, but this is the quickstart.

# Install

Install

    git clone https://github.com/diversen/coscms

    cd coscms

Update composer packages

    composer update

Create an apache2 host

    ./coscli.sh apache2 --en yoursite.com

Install:

    ./coscli.sh prompt-install --in

You will need a:

* MySQL user (username and password) who can create a database
* Database name to be created
* Server name (yoursite.com - in our example)

Install will proceed, and all modules will be cloned from `git` repos.

At last you are prompted for an email and a password. Enter email and password, but just before logging in, run the following command to set correct file perms

    sudo ./coscli.sh file --chmod-files

In order to change other settings, edit `config/config.ini`

License: MIT