text:
r"""
Util functions (:mod: `qiita_db.util`)
======================================

..currentmodule:: qiita_db.util

This module provides different util functions.

Methods
-------

..autosummary::
    :toctree: generated/

    quote_data_value
    scrub_data
    exists_table
    get_db_files_base_dir
    compute_checksum
    get_files_from_uploads_folders
    get_mountpoint
    insert_filepaths
    check_table_cols
    check_required_columns
    convert_from_id
    convert_to_id
    get_environmental_packages
    get_visibilities
    purge_filepaths
    move_filepaths_to_upload_folder
    move_upload_files_to_trash
    add_message
    get_pubmed_ids_from_dois
"""
# -----------------------------------------------------------------------------
# Copyright (c) 2014--, The Qiita Development Team.
#
# Distributed under the terms of the BSD 3-clause License.
#
# The full license is in the file LICENSE, distributed with this software.
# -----------------------------------------------------------------------------

from __future__ import division
from future.builtins import zip
from future.utils import viewitems
from random import choice
from string import ascii_letters, digits, punctuation
from binascii import crc32
from bcrypt import hashpw, gensalt
from functools import partial
from os.path import join, basename, isdir, relpath, exists
from os import walk, remove, listdir, makedirs, rename
from shutil import move, rmtree, copy as shutil_copy
from json import dumps
from datetime import datetime
from itertools import chain

from qiita_core.exceptions import IncompetentQiitaDeveloperError
import qiita_db as qdb

def params_dict_to_json(options):
    """Convert a dict of parameter key-value pairs to JSON string

    Parameters
    ----------
    options : dict
        The dict of options
    """
    return dumps(options, sort_keys=True, separators=(',', ':'))

def scrub_data(s):
    r"""Scrubs data fields of characters not allowed by PostgreSQL

    disallowed characters:
        '   ;

    Parameters
    ----------
    s : str
        The string to clean up

    Returns
    -------
    str
        The scrubbed string
    """
    ret = s.replace("'", "")
    ret = ret.replace(";", "")
    return ret

def convert_type(obj):
    """Converts a passed item to int, float, or str in that order

    Parameters
    ----------
    obj : object
        object to evaluate

    Returns
    -------
    int, float, or str
        Re-typed information from obj

    Raises
    ------
    IncompetentQiitaDeveloperError
        If the object can't be converted to int, float, or string

    Notes
    -----
    The function first tries to convert to an int. If that fails, it tries to
    convert to a float. If that fails it returns the original string.
    """
    item = None
    if isinstance(obj, datetime):
        item = str(obj)
    else:
        for fn in (int, float, str):
            try:
                item = fn(obj)
            except ValueError:
                continue
            else:
                break
    if item is None:
        raise IncompetentQiitaDeveloperError("Can't convert item of type %s!" %
                                             str(type(obj)))
    return item

def get_artifact_types(key_by_id=False):
    """Gets the list of possible artifact types

    Parameters
    ----------
    key : bool, optional
        Determines the format of the returned dict. Defaults to false.

    Returns
    -------
    dict
        If key_by_id is True, dict is of the form
        {artifact_type_id: artifact_type}
        If key_by_id is False, dict is of the form
        {artifact_type: artifact_type_id}
    """
    with qdb.sql_connection.TRN:
        cols = ('artifact_type_id, artifact_type'
                if key_by_id else 'artifact_type, artifact_type_id')
        sql = "SELECT {} FROM qiita.artifact_type".format(cols)
        qdb.sql_connection.TRN.add(sql)
        return dict(qdb.sql_connection.TRN.execute_fetchindex())

def get_filepath_types(key='filepath_type'):
    """Gets the list of possible filepath types from the filetype table

    Parameters
    ----------
    key : {'filepath_type', 'filepath_type_id'}, optional
        Defaults to "filepath_type". Determines the format of the returned
        dict.

    Returns
    -------
    dict
        - If `key` is "filepath_type", dict is of the form
          {filepath_type: filepath_type_id}
        - If `key` is "filepath_type_id", dict is of the form
          {filepath_type_id: filepath_type}
    """
    with qdb.sql_connection.TRN:
        if key == 'filepath_type':
            cols = 'filepath_type, filepath_type_id'
        elif key == 'filepath_type_id':
            cols = 'filepath_type_id, filepath_type'
        else:
            raise qdb.exceptions.QiitaDBColumnError(
                "Unknown key. Pass either 'filepath_type' or "
                "'filepath_type_id'.")
        sql = 'SELECT {} FROM qiita.filepath_type'.format(cols)
        qdb.sql_connection.TRN.add(sql)
        return dict(qdb.sql_connection.TRN.execute_fetchindex())

def get_data_types(key='data_type'):
    """Gets the list of possible data types from the data_type table

    Parameters
    ----------
    key : {'data_type', 'data_type_id'}, optional
        Defaults to "data_type". Determines the format of the returned dict.

    Returns
    -------
    dict
        - If `key` is "data_type", dict is of the form
          {data_type: data_type_id}
        - If `key` is "data_type_id", dict is of the form
          {data_type_id: data_type}
    """
    with qdb.sql_connection.TRN:
        if key == 'data_type':
            cols = 'data_type, data_type_id'
        elif key == 'data_type_id':
            cols = 'data_type_id, data_type'
        else:
            raise qdb.exceptions.QiitaDBColumnError(
                "Unknown key. Pass either 'data_type_id' or 'data_type'.")
        sql = 'SELECT {} FROM qiita.data_type'.format(cols)
        qdb.sql_connection.TRN.add(sql)
        return dict(qdb.sql_connection.TRN.execute_fetchindex())

def create_rand_string(length, punct=True):
    """Returns a string of random ascii characters

    Parameters
    ----------
    length: int
        Length of string to return
    punct: bool, optional
        Include punctuation as well as letters and numbers. Default True.
    """
    chars = ''.join((ascii_letters, digits))
    if punct:
        chars = ''.join((chars, punctuation))
    return ''.join(choice(chars) for i in range(length))

def hash_password(password, hashedpw=None):
    """Hashes password

    Parameters
    ----------
    password: str
        Plaintext password
    hashedpw: str, optional
        Previously hashed password for bcrypt to pull salt from. If not
        given, salt generated before hash

    Returns
    -------
    str
        Hashed password

    Notes
    -----
    Relies on bcrypt library to hash passwords, which stores the salt as
    part of the hashed password. Don't need to actually store the salt
    because of this.
    """
    # all the encode/decode as a python 3 workaround for bcrypt
    if hashedpw is None:
        hashedpw = gensalt()
    else:
        hashedpw = hashedpw.encode('utf-8')
    password = password.encode('utf-8')
    output = hashpw(password, hashedpw)
    if isinstance(output, bytes):
        output = output.decode("utf-8")
    return output

def check_required_columns(keys, table):
    """Makes sure all required columns in database table are in keys

    Parameters
    ----------
    keys: iterable
        Holds the keys in the dictionary
    table: str
        name of the table to check required columns

    Raises
    ------
    QiitaDBColumnError
        If keys exist that are not in the table
    RuntimeError
        Unable to get columns from database
    """
    with qdb.sql_connection.TRN:
        sql = """SELECT is_nullable, column_name, column_default
                 FROM information_schema.columns WHERE table_name = %s"""
        qdb.sql_connection.TRN.add(sql, [table])
        cols = qdb.sql_connection.TRN.execute_fetchindex()
        # Test needed because a user with certain permissions can query without
        # error but be unable to get the column names
        if len(cols) == 0:
            raise RuntimeError("Unable to fetch column names for table %s"
                               % table)
        required = set(x[1] for x in cols if x[0] == 'NO' and x[2] is None)
        if len(required.difference(keys)) > 0:
            raise qdb.exceptions.QiitaDBColumnError(
                "Required keys missing: %s" % required.difference(keys))

def check_table_cols(keys, table):
    """Makes sure all keys correspond to column headers in a table

    Parameters
    ----------
    keys: iterable
        Holds the keys in the dictionary
    table: str
        name of the table to check column names

    Raises
    ------
    QiitaDBColumnError
        If a key is found that is not in table columns
    RuntimeError
        Unable to get columns from database
    """
    with qdb.sql_connection.TRN:
        sql = """SELECT column_name FROM information_schema.columns
                 WHERE table_name = %s"""
        qdb.sql_connection.TRN.add(sql, [table])
        cols = qdb.sql_connection.TRN.execute_fetchflatten()
        # Test needed because a user with certain permissions can query without
        # error but be unable to get the column names
        if len(cols) == 0:
            raise RuntimeError("Unable to fetch column names for table %s"
                               % table)
        if len(set(keys).difference(cols)) > 0:
            raise qdb.exceptions.QiitaDBColumnError(
                "Non-database keys found: %s" % set(keys).difference(cols))

def get_table_cols(table):
    """Returns the column headers of table

    Parameters
    ----------
    table : str
        The table name

    Returns
    -------
    list of str
        The column headers of `table`
    """
    with qdb.sql_connection.TRN:
        sql = """SELECT column_name FROM information_schema.columns
                 WHERE table_name=%s AND table_schema='qiita'"""
        qdb.sql_connection.TRN.add(sql, [table])
        return qdb.sql_connection.TRN.execute_fetchflatten()

def exists_table(table):
    r"""Checks if `table` exists on the database

    Parameters
    ----------
    table : str
        The table name to check if exists

    Returns
    -------
    bool
        Whether `table` exists on the database or not
    """
    with qdb.sql_connection.TRN:
        sql = """SELECT exists(
                    SELECT * FROM information_schema.tables
                    WHERE table_name=%s)"""
        qdb.sql_connection.TRN.add(sql, [table])
        return qdb.sql_connection.TRN.execute_fetchlast()

def get_db_files_base_dir():
    r"""Returns the path to the base directory of all db files

    Returns
    -------
    str
        The path to the base directory of all db files
    """
    with qdb.sql_connection.TRN:
        qdb.sql_connection.TRN.add("SELECT base_data_dir FROM settings")
        return qdb.sql_connection.TRN.execute_fetchlast()

def get_work_base_dir():
    r"""Returns the path to the base directory of all db files

    Returns
    -------
    str
        The path to the base directory of all db files
    """
    with qdb.sql_connection.TRN:
        qdb.sql_connection.TRN.add("SELECT base_work_dir FROM settings")
        return qdb.sql_connection.TRN.execute_fetchlast()

def compute_checksum(path):
    r"""Returns the checksum of the file pointed by path

    Parameters
    ----------
    path : str
        The path to compute the checksum

    Returns
    -------
    int
        The file checksum
    """
    crc = 0
    filepaths = []
    if isdir(path):
        for name, dirs, files in walk(path):
            join_f = partial(join, name)
            filepaths.extend(list(map(join_f, files)))
    else:
        filepaths.append(path)

    for fp in filepaths:
        with open(fp, "Ub") as f:
            # Go line by line so we don't need to load the entire file
            for line in f:
                if crc is None:
                    crc = crc32(line)
                else:
                    crc = crc32(line, crc)
    # We need the & 0xffffffff in order to get the same numeric value across
    # all python versions and platforms
    return crc & 0xffffffff

def get_files_from_uploads_folders(study_id):
    """Retrieve files in upload folders

    Parameters
    ----------
    study_id : str
        The study id of which to retrieve all upload folders

    Returns
    -------
    list
        List of the filepaths for upload for that study
    """
    study_id = str(study_id)
    fp = []
    for pid, p in get_mountpoint("uploads", retrieve_all=True):
        t = join(p, study_id)
        if exists(t):
            fp.extend([(pid, f)
                       for f in listdir(t)
                       if not f.startswith('.') and not isdir(join(t, f))])

    return fp

def move_upload_files_to_trash(study_id, files_to_move):
    """Move files to a trash folder within the study_id upload folder

    Parameters
    ----------
    study_id : int
        The study id
    files_to_move : list
        List of tuples (folder_id, filename)

    Raises
    ------
    QiitaDBError
        If folder_id or the study folder don't exist and if the filename to
        erase matches the trash_folder, internal variable
    """
    trash_folder = 'trash'
    folders = {k: v for k, v in get_mountpoint("uploads", retrieve_all=True)}

    for fid, filename in files_to_move:
        if filename == trash_folder:
            raise qdb.exceptions.QiitaDBError(
                "You can not erase the trash folder: %s" % trash_folder)

        if fid not in folders:
            raise qdb.exceptions.QiitaDBError(
                "The filepath id: %d doesn't exist in the database" % fid)

        foldername = join(folders[fid], str(study_id))
        if not exists(foldername):
            raise qdb.exceptions.QiitaDBError(
                "The upload folder for study id: %d doesn't exist" % study_id)

        trashpath = join(foldername, trash_folder)
        if not exists(trashpath):
            makedirs(trashpath)

        fullpath = join(foldername, filename)
        new_fullpath = join(foldername, trash_folder, filename)

        if not exists(fullpath):
            raise qdb.exceptions.QiitaDBError(
                "The filepath %s doesn't exist in the system" % fullpath)

        rename(fullpath, new_fullpath)

def get_mountpoint(mount_type, retrieve_all=False, retrieve_subdir=False):
    r""" Returns the most recent values from data directory for the given type

    Parameters
    ----------
    mount_type : str
        The data mount type
    retrieve_all : bool, optional
        Retrieve all the available mount points or just the active one.
        Default: False.
    retrieve_subdir : bool, optional
        Retrieve the subdirectory column. Default: False.

    Returns
    -------
    list
        List of tuple, where: [(id_mountpoint, filepath_of_mountpoint)]
    """
    with qdb.sql_connection.TRN:
        if retrieve_all:
            sql = """SELECT data_directory_id, mountpoint, subdirectory
                     FROM qiita.data_directory
                     WHERE data_type=%s ORDER BY active DESC"""
        else:
            sql = """SELECT data_directory_id, mountpoint, subdirectory
                     FROM qiita.data_directory
                     WHERE data_type=%s AND active=true"""
        qdb.sql_connection.TRN.add(sql, [mount_type])
        db_result = qdb.sql_connection.TRN.execute_fetchindex()
        basedir = get_db_files_base_dir()
        if retrieve_subdir:
            result = [(d, join(basedir, m), s) for d, m, s in db_result]
        else:
            result = [(d, join(basedir, m)) for d, m, _ in db_result]
        return result

def get_mountpoint_path_by_id(mount_id):
    r""" Returns the mountpoint path for the mountpoint with id = mount_id

    Parameters
    ----------
    mount_id : int
        The mountpoint id

    Returns
    -------
    str
        The mountpoint path
    """
    with qdb.sql_connection.TRN:
        sql = """SELECT mountpoint FROM qiita.data_directory
                 WHERE data_directory_id=%s"""
        qdb.sql_connection.TRN.add(sql, [mount_id])
        mountpoint = qdb.sql_connection.TRN.execute_fetchlast()
        return join(get_db_files_base_dir(), mountpoint)

def insert_filepaths(filepaths, obj_id, table, filepath_table,
                     move_files=True, copy=False):
    r"""Inserts `filepaths` in the database.

    Since the files live outside the database, the directory in which the files
    lives is controlled by the database, so it moves the filepaths from
    its original location to the controlled directory.

    Parameters
    ----------
    filepaths : iterable of tuples (str, int)
        The list of paths to the raw files and its filepath type identifier
    obj_id : int
        Id of the object calling the functions. Disregarded if move_files
        is False
    table : str
        Table that holds the file data.
    filepath_table : str
        Table that holds the filepath information
    move_files : bool, optional
        Whether or not to move the given filepaths to the db filepaths
        default: True
    copy : bool, optional
        If `move_files` is true, whether to actually move the files or just
        copy them

    Returns
    -------
    list of int
        List of the filepath_id in the database for each added filepath
    """
    with qdb.sql_connection.TRN:
        new_filepaths = filepaths

        dd_id, mp, subdir = get_mountpoint(table, retrieve_subdir=True)[0]
        base_fp = join(get_db_files_base_dir(), mp)

        if move_files:
            db_path = partial(join, base_fp)
            if subdir:
                # Generate the new filepaths, format:
                # mountpoint/obj_id/original_name
                dirname = db_path(str(obj_id))
                if not exists(dirname):
                    makedirs(dirname)
                new_filepaths = [
                    (join(dirname, basename(path)), id_)
                    for path, id_ in filepaths]
            else:
                # Generate the new fileapths. format:
                # mountpoint/DataId_OriginalName
                new_filepaths = [
                    (db_path("%s_%s" % (obj_id, basename(path))), id_)
                    for path, id_ in filepaths]
            # Move the original files to the controlled DB directory
            transfer_function = shutil_copy if copy else move
            for old_fp, new_fp in zip(filepaths, new_filepaths):
                    transfer_function(old_fp[0], new_fp[0])
                    # In case the transaction executes a rollback, we need to
                    # make sure the files have not been moved
                    qdb.sql_connection.TRN.add_post_rollback_func(
                        move, new_fp[0], old_fp[0])

        def str_to_id(x):
            return (x if isinstance(x, (int, long))
                    else convert_to_id(x, "filepath_type"))
        paths_w_checksum = [(basename(path), str_to_id(id_),
                            compute_checksum(path))
                            for path, id_ in new_filepaths]
        # Create the list of SQL values to add
        values = [[path, pid, checksum, 1, dd_id]
                  for path, pid, checksum in paths_w_checksum]
        # Insert all the filepaths at once and get the filepath_id back
        sql = """INSERT INTO qiita.{0}
                    (filepath, filepath_type_id, checksum,
                     checksum_algorithm_id, data_directory_id)
                 VALUES (%s, %s, %s, %s, %s)
                 RETURNING filepath_id""".format(filepath_table)
        idx = qdb.sql_connection.TRN.index
        qdb.sql_connection.TRN.add(sql, values, many=True)
        # Since we added the query with many=True, we've added len(values)
        # queries to the transaction, so the ids are in the last idx queries
        return list(chain.from_iterable(
            chain.from_iterable(qdb.sql_connection.TRN.execute()[idx:])))

def retrieve_filepaths(obj_fp_table, obj_id_column, obj_id, sort=None,
                       fp_type=None):
    """Retrieves the filepaths for the given object id

    Parameters
    ----------
    obj_fp_table : str
        The name of the table that links the object and the filepath
    obj_id_column : str
        The name of the column that represents the object id
    obj_id : int
        The object id
    sort : {'ascending', 'descending'}, optional
        The direction in which the results are sorted, using the filepath id
        as sorting key. Default: None, no sorting is applied
    fp_type: str, optional
        Retrieve only the filepaths of the matching filepath type

    Returns
    -------
    list of (int, str, str)
        The list of (filepath id, filepath, filepath_type) attached to the
        object id
    """

    def path_builder(db_dir, filepath, mountpoint, subdirectory, obj_id):
        if subdirectory:
            return join(db_dir, mountpoint, str(obj_id), filepath)
        else:
            return join(db_dir, mountpoint, filepath)

    sql_sort = ""
    if sort == 'ascending':
        sql_sort = " ORDER BY filepath_id"
    elif sort == 'descending':
        sql_sort = " ORDER BY filepath_id DESC"
    elif sort is not None:
        raise qdb.exceptions.QiitaDBError(
            "Unknown sorting direction: %s. Please choose from 'ascending' or "
            "'descending'" % sort)

    sql_args = [obj_id]

    sql_type = ""
    if fp_type:
        sql_type = " AND filepath_type=%s"
        sql_args.append(fp_type)

    with qdb.sql_connection.TRN:
        sql = """SELECT filepath_id, filepath, filepath_type, mountpoint,
                        subdirectory
                 FROM qiita.filepath
                    JOIN qiita.filepath_type USING (filepath_type_id)
                    JOIN qiita.data_directory USING (data_directory_id)
                    JOIN qiita.{0} USING (filepath_id)
                 WHERE {1} = %s{2}{3}""".format(obj_fp_table, obj_id_column,
                                                sql_type, sql_sort)
        qdb.sql_connection.TRN.add(sql, sql_args)
        results = qdb.sql_connection.TRN.execute_fetchindex()
        db_dir = get_db_files_base_dir()

        return [(fpid, path_builder(db_dir, fp, m, s, obj_id), fp_type_)
                for fpid, fp, fp_type_, m, s in results]

def purge_filepaths():
    r"""Goes over the filepath table and remove all the filepaths that are not
    used in any place
    """
    with qdb.sql_connection.TRN:
        # Get all the (table, column) pairs that reference to the filepath
        # table. Adapted from http://stackoverflow.com/q/5347050/3746629
        sql = """SELECT R.TABLE_NAME, R.column_name
            FROM INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE u
            INNER JOIN INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS FK
                ON U.CONSTRAINT_CATALOG = FK.UNIQUE_CONSTRAINT_CATALOG
                AND U.CONSTRAINT_SCHEMA = FK.UNIQUE_CONSTRAINT_SCHEMA
                AND U.CONSTRAINT_NAME = FK.UNIQUE_CONSTRAINT_NAME
            INNER JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE R
                ON R.CONSTRAINT_CATALOG = FK.CONSTRAINT_CATALOG
                AND R.CONSTRAINT_SCHEMA = FK.CONSTRAINT_SCHEMA
                AND R.CONSTRAINT_NAME = FK.CONSTRAINT_NAME
            WHERE U.COLUMN_NAME = 'filepath_id'
                AND U.TABLE_SCHEMA = 'qiita'
                AND U.TABLE_NAME = 'filepath'"""
        qdb.sql_connection.TRN.add(sql)

        union_str = " UNION ".join(
            ["SELECT %s FROM qiita.%s WHERE %s IS NOT NULL" % (col, table, col)
             for table, col in qdb.sql_connection.TRN.execute_fetchindex()])
        # Get all the filepaths from the filepath table that are not
        # referenced from any place in the database
        sql = """SELECT filepath_id, filepath, filepath_type, data_directory_id
            FROM qiita.filepath FP JOIN qiita.filepath_type FPT
                ON FP.filepath_type_id = FPT.filepath_type_id
            WHERE filepath_id NOT IN (%s)""" % union_str
        qdb.sql_connection.TRN.add(sql)

        # We can now go over and remove all the filepaths
        sql = "DELETE FROM qiita.filepath WHERE filepath_id=%s"
        db_results = qdb.sql_connection.TRN.execute_fetchindex()
        for fp_id, fp, fp_type, dd_id in db_results:
            qdb.sql_connection.TRN.add(sql, [fp_id])

            # Remove the data
            fp = join(get_mountpoint_path_by_id(dd_id), fp)
            if exists(fp):
                if fp_type is 'directory':
                    func = rmtree
                else:
                    func = remove
                qdb.sql_connection.TRN.add_post_commit_func(func, fp)

        qdb.sql_connection.TRN.execute()

def move_filepaths_to_upload_folder(study_id, filepaths):
    r"""Goes over the filepaths list and moves all the filepaths that are not
    used in any place to the upload folder of the study

    Parameters
    ----------
    study_id : int
        The study id to where the files should be returned to
    filepaths : list
        List of filepaths to move to the upload folder
    """
    with qdb.sql_connection.TRN:
        uploads_fp = join(get_mountpoint("uploads")[0][1], str(study_id))
        path_builder = partial(join, uploads_fp)

        # We can now go over and remove all the filepaths
        sql = """DELETE FROM qiita.filepath WHERE filepath_id=%s"""
        for fp_id, fp, fp_type in filepaths:
            qdb.sql_connection.TRN.add(sql, [fp_id])

            if fp_type == 'html_summary':
                qdb.sql_connection.TRN.add_post_commit_func(
                    remove, fp)
            else:
                destination = path_builder(basename(fp))

                qdb.sql_connection.TRN.add_post_rollback_func(
                    move, destination, fp)
                move(fp, destination)

        qdb.sql_connection.TRN.execute()

def get_filepath_id(table, fp):
    """Return the filepath_id of fp

    Parameters
    ----------
    table : str
        The table type so we can search on this one
    fp : str
        The full filepath

    Returns
    -------
    int
        The filepath id forthe given filepath

    Raises
    ------
    QiitaDBError
        If fp is not stored in the DB.
    """
    with qdb.sql_connection.TRN:
        _, mp = get_mountpoint(table)[0]
        base_fp = join(get_db_files_base_dir(), mp)

        sql = "SELECT filepath_id FROM qiita.filepath WHERE filepath=%s"
        qdb.sql_connection.TRN.add(sql, [relpath(fp, base_fp)])
        fp_id = qdb.sql_connection.TRN.execute_fetchindex()

        # check if the query has actually returned something
        if not fp_id:
            raise qdb.exceptions.QiitaDBError(
                "Filepath not stored in the database")

        # If there was a result it was a single row and and single value,
        # hence access to [0][0]
        return fp_id[0][0]

def filepath_id_to_rel_path(filepath_id):
    """Gets the relative to the base directory of filepath_id

    Returns
    -------
    str
        The relative path for the given filepath id
    """
    with qdb.sql_connection.TRN:
        sql = """SELECT mountpoint, filepath, subdirectory, artifact_id
                 FROM qiita.filepath
                    JOIN qiita.data_directory USING (data_directory_id)
                    LEFT JOIN qiita.artifact_filepath USING (filepath_id)
                 WHERE filepath_id = %s"""
        qdb.sql_connection.TRN.add(sql, [filepath_id])
        mp, fp, sd, a_id = qdb.sql_connection.TRN.execute_fetchindex()[0]
        if sd:
            result = join(mp, str(a_id), fp)
        else:
            result = join(mp, fp)
        # It should be only one row
        return result

def filepath_ids_to_rel_paths(filepath_ids):
    """Gets the full paths, relative to the base directory

    Parameters
    ----------
    filepath_ids : list of int

    Returns
    -------
    dict where keys are ints and values are str
        {filepath_id: relative_path}
    """
    if not filepath_ids:
        return {}

    with qdb.sql_connection.TRN:
        sql = """SELECT filepath_id, mountpoint, filepath, subdirectory,
                        artifact_id
                 FROM qiita.filepath
                    JOIN qiita.data_directory USING (data_directory_id)
                    LEFT JOIN qiita.artifact_filepath USING (filepath_id)
                 WHERE filepath_id IN %s"""
        qdb.sql_connection.TRN.add(sql, [tuple(filepath_ids)])
        res = {}
        for row in qdb.sql_connection.TRN.execute_fetchindex():
            if row[3]:
                res[row[0]] = join(row[1], str(row[4]), row[2])
            else:
                res[row[0]] = join(row[1], row[2])
        return res

def convert_to_id(value, table, text_col=None):
    """Converts a string value to its corresponding table identifier

    Parameters
    ----------
    value : str
        The string value to convert
    table : str
        The table that has the conversion
    text_col : str, optional
        Column holding the string value. Defaults to same as table name.

    Returns
    -------
    int
        The id correspinding to the string

    Raises
    ------
    QiitaDBLookupError
        The passed string has no associated id
    """
    text_col = table if text_col is None else text_col
    with qdb.sql_connection.TRN:
        sql = "SELECT {0}_id FROM qiita.{0} WHERE {1} = %s".format(
            table, text_col)
        qdb.sql_connection.TRN.add(sql, [value])
        _id = qdb.sql_connection.TRN.execute_fetchindex()
        if not _id:
            raise qdb.exceptions.QiitaDBLookupError(
                "%s not valid for table %s" % (value, table))
        # If there was a result it was a single row and and single value,
        # hence access to [0][0]
        return _id[0][0]

def convert_from_id(value, table):
    """Converts an id value to its corresponding string value

    Parameters
    ----------
    value : int
        The id value to convert
    table : str
        The table that has the conversion

    Returns
    -------
    str
        The string correspinding to the id

    Raises
    ------
    QiitaDBLookupError
        The passed id has no associated string
    """
    with qdb.sql_connection.TRN:
        sql = "SELECT {0} FROM qiita.{0} WHERE {0}_id = %s".format(table)
        qdb.sql_connection.TRN.add(sql, [value])
        string = qdb.sql_connection.TRN.execute_fetchindex()
        if not string:
            raise qdb.exceptions.QiitaDBLookupError(
                "%s not valid for table %s" % (value, table))
        # If there was a result it was a single row and and single value,
        # hence access to [0][0]
        return string[0][0]

def get_count(table):
    """Counts the number of rows in a table

    Parameters
    ----------
    table : str
        The name of the table of which to count the rows

    Returns
    -------
    int
    """
    with qdb.sql_connection.TRN:
        sql = "SELECT count(1) FROM %s" % table
        qdb.sql_connection.TRN.add(sql)
        return qdb.sql_connection.TRN.execute_fetchlast()

def check_count(table, exp_count):
    """Checks that the number of rows in a table equals the expected count

    Parameters
    ----------
    table : str
        The name of the table of which to count the rows
    exp_count : int
        The expected number of rows in the table

    Returns
    -------
    bool
    """
    obs_count = get_count(table)
    return obs_count == exp_count

def get_environmental_packages():
    """Get the list of available environmental packages

    Returns
    -------
    list of (str, str)
        The available environmental packages. The first string is the
        environmental package name and the second string is the table where
        the metadata for the environmental package is stored
    """
    with qdb.sql_connection.TRN:
        qdb.sql_connection.TRN.add("SELECT * FROM qiita.environmental_package")
        return qdb.sql_connection.TRN.execute_fetchindex()

def get_visibilities():
    """Get the list of available visibilities for artifacts

    Returns
    -------
    list of str
        The available visibilities
    """
    with qdb.sql_connection.TRN:
        qdb.sql_connection.TRN.add("SELECT visibility FROM qiita.visibility")
        return qdb.sql_connection.TRN.execute_fetchflatten()

def get_timeseries_types():
    """Get the list of available timeseries types

    Returns
    -------
    list of (int, str, str)
        The available timeseries types. Each timeseries type is defined by the
        tuple (timeseries_id, timeseries_type, intervention_type)
    """
    with qdb.sql_connection.TRN:
        sql = "SELECT * FROM qiita.timeseries_type ORDER BY timeseries_type_id"
        qdb.sql_connection.TRN.add(sql)
        return qdb.sql_connection.TRN.execute_fetchindex()

def get_pubmed_ids_from_dois(doi_ids):
    """Get the dict of pubmed ids from a list of doi ids

    Parameters
    ----------
    doi_ids : list of str
        The list of doi ids

    Returns
    -------
    dict of {doi: pubmed_id}
        Return dict of doi and pubmed ids

    Notes
    -----
    If doi doesn't exist it will not return that {key: value} pair
    """
    with qdb.sql_connection.TRN:
        sql = "SELECT doi, pubmed_id FROM qiita.publication WHERE doi IN %s"
        qdb.sql_connection.TRN.add(sql, [tuple(doi_ids)])
        return {row[0]: row[1]
                for row in qdb.sql_connection.TRN.execute_fetchindex()}

def check_access_to_analysis_result(user_id, requested_path):
    """Get filepath IDs for a particular requested_path, if user has access

    This function is only applicable for analysis results.

    Parameters
    ----------
    user_id : str
        The ID (email address) that identifies the user
    requested_path : str
        The path that the user requested

    Returns
    -------
    list of int
        The filepath IDs associated with the requested path
    """
    with qdb.sql_connection.TRN:
        # Get all filepath ids associated with analyses that the user has
        # access to where the filepath is the base_requested_fp from above.
        # There should typically be only one matching filepath ID, but for
        # safety we allow for the possibility of multiple.
        sql = """SELECT fp.filepath_id
                 FROM qiita.analysis_job aj JOIN (
                    SELECT analysis_id FROM qiita.analysis A
                    JOIN qiita.analysis_status stat
                    ON A.analysis_status_id = stat.analysis_status_id
                    WHERE stat.analysis_status_id = 6
                    UNION
                    SELECT analysis_id FROM qiita.analysis_users
                    WHERE email = %s
                    UNION
                    SELECT analysis_id FROM qiita.analysis WHERE email = %s
                 ) ids ON aj.analysis_id = ids.analysis_id
                 JOIN qiita.job_results_filepath jrfp ON
                    aj.job_id = jrfp.job_id
                 JOIN qiita.filepath fp ON jrfp.filepath_id = fp.filepath_id
                 WHERE fp.filepath = %s"""
        qdb.sql_connection.TRN.add(sql, [user_id, user_id, requested_path])

        return qdb.sql_connection.TRN.execute_fetchflatten()

def infer_status(statuses):
    """Infers an object status from the statuses passed in

    Parameters
    ----------
    statuses : list of lists of strings or empty list
        The list of statuses used to infer the resulting status (the result
        of execute_fetchall)

    Returns
    -------
    str
        The inferred status

    Notes
    -----
    The inference is done in the following priority (high to low):
        (1) public
        (2) private
        (3) awaiting_approval
        (4) sandbox
    """
    if statuses:
        statuses = set(s[0] for s in statuses)
        if 'public' in statuses:
            return 'public'
        if 'private' in statuses:
            return 'private'
        if 'awaiting_approval' in statuses:
            return 'awaiting_approval'
    # If there are no statuses, or any of the previous ones have been found
    # then the inferred status is 'sandbox'
    return 'sandbox'

def add_message(message, users):
    """Adds a message to the messages table, attaching it to given users

    Parameters
    ----------
    message : str
        Message to add
    users : list of User objects
        Users to connect the message to
    """
    with qdb.sql_connection.TRN:
        sql = """INSERT INTO qiita.message (message) VALUES (%s)
                 RETURNING message_id"""
        qdb.sql_connection.TRN.add(sql, [message])
        msg_id = qdb.sql_connection.TRN.execute_fetchlast()
        sql = """INSERT INTO qiita.message_user (email, message_id)
                 VALUES (%s, %s)"""
        sql_args = [[user.id, msg_id] for user in users]
        qdb.sql_connection.TRN.add(sql, sql_args, many=True)
        qdb.sql_connection.TRN.execute()

def add_system_message(message, expires):
    """Adds a system message to the messages table, attaching it to asl users

    Parameters
    ----------
    message : str
        Message to add
    expires : datetime object
        Expiration for the message
    """
    with qdb.sql_connection.TRN:
        sql = """INSERT INTO qiita.message (message, expiration)
                 VALUES (%s, %s)
                 RETURNING message_id"""
        qdb.sql_connection.TRN.add(sql, [message, expires])
        msg_id = qdb.sql_connection.TRN.execute_fetchlast()
        sql = """INSERT INTO qiita.message_user (email, message_id)
                 SELECT email, %s FROM qiita.qiita_user"""
        qdb.sql_connection.TRN.add(sql, [msg_id])
        qdb.sql_connection.TRN.execute()

def clear_system_messages():
    with qdb.sql_connection.TRN:
        sql = "SELECT message_id FROM qiita.message WHERE expiration < %s"
        qdb.sql_connection.TRN.add(sql, [datetime.now()])
        msg_ids = qdb.sql_connection.TRN.execute_fetchflatten()
        if msg_ids:
            msg_ids = tuple(msg_ids)
            sql = "DELETE FROM qiita.message_user WHERE message_id IN %s"
            qdb.sql_connection.TRN.add(sql, [msg_ids])
            sql = "DELETE FROM qiita.message WHERE message_id IN %s"
            qdb.sql_connection.TRN.add(sql, [msg_ids])
            qdb.sql_connection.TRN.execute()

def supported_filepath_types(artifact_type):
    """Returns the list of supported filepath types for the given artifact type

    Parameters
    ----------
    artifact_type : str
        The artifact type to check the supported filepath types

    Returns
    -------
    list of [str, bool]
        The list of supported filepath types and whether it is required by the
        artifact type or not
    """
    with qdb.sql_connection.TRN:
        sql = """SELECT DISTINCT filepath_type, required
                 FROM qiita.artifact_type_filepath_type
                    JOIN qiita.artifact_type USING (artifact_type_id)
                    JOIN qiita.filepath_type USING (filepath_type_id)
                 WHERE artifact_type = %s"""
        qdb.sql_connection.TRN.add(sql, [artifact_type])
        return qdb.sql_connection.TRN.execute_fetchindex()

def generate_study_list(study_ids, build_samples):
    """Get general study information

    Parameters
    ----------
    study_ids : list of ints
        The study ids to look for. Non-existing ids will be ignored
    build_samples : bool
        If true the sample information for each process artifact within each
        study will be included

    Returns
    -------
    list of dict
        The list of studies and their information

    Notes
    -----
    The main select might look scary but it's pretty simple:
    - We select the requiered fields from qiita.study and qiita.study_person
        SELECT metadata_complete, study_abstract, study_id,
            study_title, ebi_study_accession, ebi_submission_status,
            qiita.study_person.name AS pi_name,
            qiita.study_person.email AS pi_email,
    - the total number of samples collected by counting sample_ids
            (SELECT COUNT(sample_id) FROM qiita.study_sample
                WHERE study_id=qiita.study.study_id)
                AS number_samples_collected,
    - all the BIOM artifact_ids sorted by artifact_id that belong to the study
            (SELECT array_agg(artifact_id ORDER BY artifact_id)
                FROM qiita.study_artifact
                LEFT JOIN qiita.artifact USING (artifact_id)
                LEFT JOIN qiita.artifact_type USING (artifact_type_id)
                WHERE artifact_type='BIOM' AND
                study_id = qiita.study.study_id) AS artifact_biom_ids,
    - all the BIOM data_types sorted by artifact_id that belong to the study
            (SELECT array_agg(data_type ORDER BY artifact_id)
                FROM qiita.study_artifact
                LEFT JOIN qiita.artifact USING (artifact_id)
                LEFT JOIN qiita.data_type USING (data_type_id)
                LEFT JOIN qiita.artifact_type USING (artifact_type_id)
                WHERE artifact_type='BIOM' AND
                study_id = qiita.study.study_id) AS artifact_biom_dts,
    - all the BIOM parameters sorted by artifact_id that belong to the study
            (SELECT array_agg(command_parameters ORDER BY artifact_id)
                FROM qiita.study_artifact
                LEFT JOIN qiita.artifact USING (artifact_id)
                LEFT JOIN qiita.artifact_type USING (artifact_type_id)
                WHERE artifact_type='BIOM' AND
                    study_id = qiita.study.study_id)
                AS artifact_biom_params,
    - all the BIOM command_ids sorted by artifact_id that belong to the study,
            (SELECT array_agg(command_id ORDER BY artifact_id)
                FROM qiita.study_artifact
                LEFT JOIN qiita.artifact USING (artifact_id)
                LEFT JOIN qiita.artifact_type USING (artifact_type_id)
                WHERE artifact_type='BIOM' AND
                    study_id = qiita.study.study_id)
                AS artifact_biom_cmd,
    - all the BIOM timestamps sorted by artifact_id that belong to the study
            (SELECT array_agg(generated_timestamp ORDER BY artifact_id)
                FROM qiita.study_artifact
                LEFT JOIN qiita.artifact USING (artifact_id)
                LEFT JOIN qiita.artifact_type USING (artifact_type_id)
                WHERE artifact_type='BIOM' AND
                    study_id = qiita.study.study_id) AS artifact_biom_ts,
    - all the visibilities of all artifacts that belong to the study
            (SELECT array_agg(DISTINCT visibility) FROM qiita.artifact
                LEFT JOIN qiita.visibility USING (visibility_id)
                WHERE study_id=qiita.study.study_id)
                AS artifacts_visibility,
    - all the publication_doi that belong to the study
            (SELECT array_agg(publication_doi ORDER BY publication_doi)
                FROM qiita.study_publication
                WHERE study_id=qiita.study.study_id) AS publication_doi,
    - all names sorted by email of users that have access to the study
            (SELECT array_agg(name ORDER BY email) FROM qiita.study_users
                LEFT JOIN qiita.qiita_user USING (email)
                WHERE study_id=qiita.study.study_id) AS shared_with_name,
    - all emails sorted by email of users that have access to the study
            (SELECT array_agg(email ORDER BY email) FROM qiita.study_users
                LEFT JOIN qiita.qiita_user USING (email)
                WHERE study_id=qiita.study.study_id) AS shared_with_email
    """
    with qdb.sql_connection.TRN:
        sql = """
            SELECT metadata_complete, study_abstract, study_id,
                study_title, ebi_study_accession, ebi_submission_status,
                qiita.study_person.name AS pi_name,
                qiita.study_person.email AS pi_email,
                (SELECT COUNT(sample_id) FROM qiita.study_sample
                    WHERE study_id=qiita.study.study_id)
                    AS number_samples_collected,
                (SELECT array_agg(artifact_id ORDER BY artifact_id)
                    FROM qiita.study_artifact
                    LEFT JOIN qiita.artifact USING (artifact_id)
                    LEFT JOIN qiita.artifact_type USING (artifact_type_id)
                    WHERE artifact_type='BIOM' AND
                        study_id = qiita.study.study_id) AS artifact_biom_ids,
                (SELECT array_agg(data_type ORDER BY artifact_id)
                    FROM qiita.study_artifact
                    LEFT JOIN qiita.artifact USING (artifact_id)
                    LEFT JOIN qiita.data_type USING (data_type_id)
                    LEFT JOIN qiita.artifact_type USING (artifact_type_id)
                    WHERE artifact_type='BIOM' AND
                        study_id = qiita.study.study_id) AS artifact_biom_dts,
                (SELECT array_agg(command_parameters ORDER BY artifact_id)
                    FROM qiita.study_artifact
                    LEFT JOIN qiita.artifact USING (artifact_id)
                    LEFT JOIN qiita.artifact_type USING (artifact_type_id)
                    WHERE artifact_type='BIOM' AND
                        study_id = qiita.study.study_id)
                    AS artifact_biom_params,
                (SELECT array_agg(command_id ORDER BY artifact_id)
                    FROM qiita.study_artifact
                    LEFT JOIN qiita.artifact USING (artifact_id)
                    LEFT JOIN qiita.artifact_type USING (artifact_type_id)
                    WHERE artifact_type='BIOM' AND
                        study_id = qiita.study.study_id)
                    AS artifact_biom_cmd,
                (SELECT array_agg(generated_timestamp ORDER BY artifact_id)
                    FROM qiita.study_artifact
                    LEFT JOIN qiita.artifact USING (artifact_id)
                    LEFT JOIN qiita.artifact_type USING (artifact_type_id)
                    WHERE artifact_type='BIOM' AND
                        study_id = qiita.study.study_id) AS artifact_biom_ts,
                (SELECT array_agg(DISTINCT visibility) FROM qiita.artifact
                    LEFT JOIN qiita.visibility USING (visibility_id)
                    WHERE study_id=qiita.study.study_id)
                    AS artifacts_visibility,
                (SELECT array_agg(publication_doi ORDER BY publication_doi)
                    FROM qiita.study_publication
                    WHERE study_id=qiita.study.study_id) AS publication_doi,
                (SELECT array_agg(name ORDER BY email) FROM qiita.study_users
                    LEFT JOIN qiita.qiita_user USING (email)
                    WHERE study_id=qiita.study.study_id) AS shared_with_name,
                (SELECT array_agg(email ORDER BY email) FROM qiita.study_users
                    LEFT JOIN qiita.qiita_user USING (email)
                    WHERE study_id=qiita.study.study_id) AS shared_with_email
                FROM qiita.study
                LEFT JOIN qiita.study_person ON (
                    study_person_id=principal_investigator_id)
                WHERE study_id IN %s"""
        qdb.sql_connection.TRN.add(sql, [tuple(study_ids)])
        infolist = []
        refs = {}
        commands = {}
        for info in qdb.sql_connection.TRN.execute_fetchindex():
            info = dict(info)

            # publication info
            if info['publication_doi'] is not None:
                info['pmid'] = get_pubmed_ids_from_dois(
                    info['publication_doi']).values()
            else:
                info['publication_doi'] = []
                info['pmid'] = []

            # visibility
            info["status"] = infer_status([info['artifacts_visibility']])
            del info['artifacts_visibility']

            # pi info
            info["pi"] = (info['pi_email'], info['pi_name'])
            del info["pi_email"]
            del info["pi_name"]

            # shared with
            info['shared'] = []
            if info['shared_with_name'] and info['shared_with_email']:
                for name, email in zip(info['shared_with_name'],
                                       info['shared_with_email']):
                    if not name:
                        name = email
                    info['shared'].append((email, name))
            del info["shared_with_name"]
            del info["shared_with_email"]

            info['proc_data_info'] = []
            if build_samples and info['artifact_biom_ids']:
                to_loop = zip(
                    info['artifact_biom_ids'], info['artifact_biom_dts'],
                    info['artifact_biom_ts'], info['artifact_biom_params'],
                    info['artifact_biom_cmd'])
                for artifact_id, dt, ts, params, cmd in to_loop:
                    proc_info = {'processed_date': str(ts)}
                    proc_info['pid'] = artifact_id
                    proc_info['data_type'] = dt

                    # if cmd exists then we can get its parameters
                    if cmd is not None:
                        # making sure that the command is only queried once
                        if cmd not in commands:
                            commands[cmd] = [
                                k for k, v in viewitems(
                                    qdb.software.Command(
                                        cmd).parameters) if v[0] == 'artifact']
                        for k in commands[cmd]:
                            del params[k]

                        # making sure that the reference is only created once
                        rid = params.pop('reference')
                        if rid not in refs:
                            reference = qdb.reference.Reference(rid)
                            refs[rid] = {
                                'name': reference.name,
                                'taxonomy_fp': basename(reference.taxonomy_fp),
                                'sequence_fp': basename(reference.sequence_fp),
                                'tree_fp': basename(reference.tree_fp),
                                'version': reference.version
                            }
                        proc_info['reference_name'] = refs[rid]['name']
                        proc_info['taxonomy_filepath'] = refs[rid][
                            'taxonomy_fp']
                        proc_info['sequence_filepath'] = refs[rid][
                            'sequence_fp']
                        proc_info['tree_filepath'] = refs[rid]['tree_fp']
                        proc_info['reference_version'] = refs[rid]['version']
                        proc_info['algorithm'] = 'sortmerna'
                        proc_info.update(params)

                    # getting all samples
                    sql = """SELECT sample_id from qiita.prep_template_sample
                             WHERE prep_template_id = (
                                 SELECT prep_template_id
                                 FROM qiita.prep_template
                                 WHERE artifact_id IN (
                                     SELECT *
                                     FROM qiita.find_artifact_roots(%s)))"""
                    qdb.sql_connection.TRN.add(sql, [proc_info['pid']])
                    proc_info['samples'] = sorted(
                        qdb.sql_connection.TRN.execute_fetchflatten())

                    info["proc_data_info"].append(proc_info)

            del info["artifact_biom_ids"]
            del info["artifact_biom_dts"]
            del info["artifact_biom_ts"]
            del info["artifact_biom_params"]
            del info['artifact_biom_cmd']

            infolist.append(info)

    return infolist


text:
{% load i18n sekizai_tags cms_tags %}
{% load url from future %}
{% url 'logout' as logout_url %}
<form class="cms-toolbar-item cms-form-login" method="post" action="{% url 'pages-root' %}{% if request.path != logout_url %}?next={{ request.path }}&edit&cms-toolbar-login=1{% endif %}">
    {% csrf_token %}
    {% trans request.toolbar.login_form.username.label %} {{ request.toolbar.login_form.username }}
    {% trans request.toolbar.login_form.password.label %} {{ request.toolbar.login_form.password }}
    <input type="submit" value="Login" />
</form>

redpajama_set_name: RedPajamaGithub


text:
import chance = require('../../extlibs/wrap_chance');

export class Question {

    num: number;

    constructor() {
        this.num = chance.natural({ min: 20, max: 20000 }) * 2;
    }

    get avg() {
        return this.num + 1;
    }

    get cans() {
        return this.num - 2;
    }

    get qtxt() {
        return `Average of 4 consecutive even numbers is ${this.avg}, smallest number is `
    }
}


text:
// Copyright (c) Jupyter Development Team.
// Distributed under the terms of the Modified BSD License.

define(function(require){
    "use strict";

    var ActionHandler = function (env) {
        this.env = env || {};
        Object.seal(this);
    };

    /**
     *  A bunch of predefined `Simple Actions` used by Jupyter.
     *  `Simple Actions` have the following keys:
     *  help (optional): a short string the describe the action.
     *      will be used in various context, like as menu name, tool tips on buttons,
     *      and short description in help menu.
     *  help_index (optional): a string used to sort action in help menu.
     *  icon (optional): a short string that represent the icon that have to be used with this
     *  action. this should mainly correspond to a Font_awesome class.
     *  handler : a function which is called when the action is activated. It will receive at first parameter
     *      a dictionary containing various handle to element of the notebook.
     *
     *  action need to be registered with a **name** that can be use to refer to this action.
     *
     *
     *  if `help` is not provided it will be derived by replacing any dash by space
     *  in the **name** of the action. It is advised to provide a prefix to action name to
     *  avoid conflict the prefix should be all lowercase and end with a dot `.`
     *  in the absence of a prefix the behavior of the action is undefined.
     *
     *  All action provided by Jupyter are prefixed with `ipython.`.
     *
     *  One can register extra actions or replace an existing action with another one is possible
     *  but is considered undefined behavior.
     *
     **/
    var _actions = {
        'run-select-next': {
            icon: 'fa-step-forward',
            help    : 'run cell, select below',
            help_index : 'ba',
            handler : function (env) {
                env.notebook.execute_cell_and_select_below();
            }
        },
        'execute-in-place':{
            help    : 'run cell',
            help_index : 'bb',
            handler : function (env) {
                env.notebook.execute_cell();
            }
        },
        'execute-and-insert-after':{
            help    : 'run cell, insert below',
            help_index : 'bc',
            handler : function (env) {
                env.notebook.execute_cell_and_insert_below();
            }
        },
        'go-to-command-mode': {
            help    : 'command mode',
            help_index : 'aa',
            handler : function (env) {
                env.notebook.command_mode();
            }
        },
        'split-cell-at-cursor': {
            help    : 'split cell',
            help_index : 'ea',
            handler : function (env) {
                env.notebook.split_cell();
            }
        },
        'enter-edit-mode' : {
            help_index : 'aa',
            handler : function (env) {
                env.notebook.edit_mode();
            }
        },
        'select-previous-cell' : {
            help: 'select cell above',
            help_index : 'da',
            handler : function (env) {
                var index = env.notebook.get_selected_index();
                if (index !== 0 && index !== null) {
                    env.notebook.select_prev();
                    env.notebook.focus_cell();
                }
            }
        },
        'select-next-cell' : {
            help: 'select cell below',
            help_index : 'db',
            handler : function (env) {
                var index = env.notebook.get_selected_index();
                if (index !== (env.notebook.ncells()-1) && index !== null) {
                    env.notebook.select_next();
                    env.notebook.focus_cell();
                }
            }
        },
        'extend-selection-previous' : {
            help: 'extend selection above',
            help_index : 'dc',
            handler : function (env) {
                var index = env.notebook.get_selected_index();
                if (index !== 0 && index !== null) {
                    env.notebook.extend_selection('up');
                    env.notebook.focus_cell();
                }
            }
        },
        'extend-selection-next' : {
            help: 'extend selection below',
            help_index : 'dd',
            handler : function (env) {
                var index = env.notebook.get_selected_index();
                if (index !== (env.notebook.ncells()-1) && index !== null) {
                    env.notebook.extend_selection('down');
                    env.notebook.focus_cell();
                }
            }
        },
        'reset-selection': {
            help: 'clear selected cells',
            help_index: 'de',
            handler: function(env) {
                env.notebook.reset_selection();
            }
        },
        'cut-selected-cell' : {
            icon: 'fa-cut',
            help_index : 'ee',
            handler : function (env) {
                var index = env.notebook.get_selected_index();
                env.notebook.cut_cell();
                env.notebook.select(index);
            }
        },
        'copy-selected-cell' : {
            icon: 'fa-copy',
            help_index : 'ef',
            handler : function (env) {
                env.notebook.copy_cell();
            }
        },
        'paste-cell-before' : {
            help: 'paste cell above',
            help_index : 'eg',
            handler : function (env) {
                env.notebook.paste_cell_above();
            }
        },
        'paste-cell-after' : {
            help: 'paste cell below',
            icon: 'fa-paste',
            help_index : 'eh',
            handler : function (env) {
                env.notebook.paste_cell_below();
            }
        },
        'insert-cell-before' : {
            help: 'insert cell above',
            help_index : 'ec',
            handler : function (env) {
                env.notebook.insert_cell_above();
                env.notebook.select_prev();
                env.notebook.focus_cell();
            }
        },
        'insert-cell-after' : {
            help: 'insert cell below',
            icon : 'fa-plus',
            help_index : 'ed',
            handler : function (env) {
                env.notebook.insert_cell_below();
                env.notebook.select_next();
                env.notebook.focus_cell();
            }
        },
        'change-selected-cell-to-code-cell' : {
            help    : 'to code',
            help_index : 'ca',
            handler : function (env) {
                env.notebook.to_code();
            }
        },
        'change-selected-cell-to-markdown-cell' : {
            help    : 'to markdown',
            help_index : 'cb',
            handler : function (env) {
                env.notebook.to_markdown();
            }
        },
        'change-selected-cell-to-raw-cell' : {
            help    : 'to raw',
            help_index : 'cc',
            handler : function (env) {
                env.notebook.to_raw();
            }
        },
        'change-selected-cell-to-heading-1' : {
            help    : 'to heading 1',
            help_index : 'cd',
            handler : function (env) {
                env.notebook.to_heading(undefined, 1);
            }
        },
        'change-selected-cell-to-heading-2' : {
            help    : 'to heading 2',
            help_index : 'ce',
            handler : function (env) {
                env.notebook.to_heading(undefined, 2);
            }
        },
        'change-selected-cell-to-heading-3' : {
            help    : 'to heading 3',
            help_index : 'cf',
            handler : function (env) {
                env.notebook.to_heading(undefined, 3);
            }
        },
        'change-selected-cell-to-heading-4' : {
            help    : 'to heading 4',
            help_index : 'cg',
            handler : function (env) {
                env.notebook.to_heading(undefined, 4);
            }
        },
        'change-selected-cell-to-heading-5' : {
            help    : 'to heading 5',
            help_index : 'ch',
            handler : function (env) {
                env.notebook.to_heading(undefined, 5);
            }
        },
        'change-selected-cell-to-heading-6' : {
            help    : 'to heading 6',
            help_index : 'ci',
            handler : function (env) {
                env.notebook.to_heading(undefined, 6);
            }
        },
        'toggle-output-visibility-selected-cell' : {
            help    : 'toggle output',
            help_index : 'gb',
            handler : function (env) {
                env.notebook.toggle_output();
            }
        },
        'toggle-output-scrolling-selected-cell' : {
            help    : 'toggle output scrolling',
            help_index : 'gc',
            handler : function (env) {
                env.notebook.toggle_output_scroll();
            }
        },
        'move-selected-cell-down' : {
            icon: 'fa-arrow-down',
            help_index : 'eb',
            handler : function (env) {
                env.notebook.move_cell_down();
            }
        },
        'move-selected-cell-up' : {
            icon: 'fa-arrow-up',
            help_index : 'ea',
            handler : function (env) {
                env.notebook.move_cell_up();
            }
        },
        'toggle-line-number-selected-cell' : {
            help    : 'toggle line numbers',
            help_index : 'ga',
            handler : function (env) {
                env.notebook.cell_toggle_line_numbers();
            }
        },
        'show-keyboard-shortcut-help-dialog' : {
            help_index : 'ge',
            handler : function (env) {
                env.quick_help.show_keyboard_shortcuts();
            }
        },
        'delete-cell': {
            help: 'delete selected cell',
            help_index : 'ej',
            handler : function (env) {
                env.notebook.delete_cell();
            }
        },
        'interrupt-kernel':{
            icon: 'fa-stop',
            help_index : 'ha',
            handler : function (env) {
                env.notebook.kernel.interrupt();
            }
        },
        'restart-kernel':{
            icon: 'fa-repeat',
            help_index : 'hb',
            handler : function (env) {
                env.notebook.restart_kernel();
            }
        },
        'undo-last-cell-deletion' : {
            help_index : 'ei',
            handler : function (env) {
                env.notebook.undelete_cell();
            }
        },
        'merge-selected-cell-with-cell-after' : {
            help    : 'merge cell below',
            help_index : 'ek',
            handler : function (env) {
                env.notebook.merge_cell_below();
            }
        },
        'merge-selected-cells' : {
            help : 'merge selected cells',
            help_index: 'el',
            handler: function(env) {
                env.notebook.merge_selected_cells();
            }
        },
        'close-pager' : {
            help_index : 'gd',
            handler : function (env) {
                env.pager.collapse();
            }
        }

    };

    /**
     * A bunch of `Advance actions` for Jupyter.
     * Cf `Simple Action` plus the following properties.
     *
     * handler: first argument of the handler is the event that triggerd the action
     *      (typically keypress). The handler is responsible for any modification of the
     *      event and event propagation.
     *      Is also responsible for returning false if the event have to be further ignored,
     *      true, to tell keyboard manager that it ignored the event.
     *
     *      the second parameter of the handler is the environemnt passed to Simple Actions
     *
     **/
    var custom_ignore = {
        'ignore':{
            handler : function () {
                return true;
            }
        },
        'move-cursor-up-or-previous-cell':{
            handler : function (env, event) {
                var index = env.notebook.get_selected_index();
                var cell = env.notebook.get_cell(index);
                var cm = env.notebook.get_selected_cell().code_mirror;
                var cur = cm.getCursor();
                if (cell && cell.at_top() && index !== 0 && cur.ch === 0) {
                    if(event){
                        event.preventDefault();
                    }
                    env.notebook.command_mode();
                    env.notebook.select_prev();
                    env.notebook.edit_mode();
                    cm = env.notebook.get_selected_cell().code_mirror;
                    cm.setCursor(cm.lastLine(), 0);
                }
                return false;
            }
        },
        'move-cursor-down-or-next-cell':{
            handler : function (env, event) {
                var index = env.notebook.get_selected_index();
                var cell = env.notebook.get_cell(index);
                if (cell.at_bottom() && index !== (env.notebook.ncells()-1)) {
                    if(event){
                        event.preventDefault();
                    }
                    env.notebook.command_mode();
                    env.notebook.select_next();
                    env.notebook.edit_mode();
                    var cm = env.notebook.get_selected_cell().code_mirror;
                    cm.setCursor(0, 0);
                }
                return false;
            }
        },
        'scroll-down': {
            handler: function(env, event) {
                if(event){
                    event.preventDefault();
                }
                return env.notebook.scroll_manager.scroll(1);
            },
        },
        'scroll-up': {
            handler: function(env, event) {
                if(event){
                    event.preventDefault();
                }
                return env.notebook.scroll_manager.scroll(-1);
            },
        },
        'scroll-cell-center': {
            help: "Scroll the current cell to the center",
            handler: function (env, event) {
                if(event){
                    event.preventDefault();
                }
                var cell = env.notebook.get_selected_index();
                return env.notebook.scroll_cell_percent(cell, 50, 0);
            }
        },
        'scroll-cell-top': {
            help: "Scroll the current cell to the top",
            handler: function (env, event) {
                if(event){
                    event.preventDefault();
                }
                var cell = env.notebook.get_selected_index();
                return env.notebook.scroll_cell_percent(cell, 0, 0);
            }
        },
        'save-notebook':{
            help: "Save and Checkpoint",
            help_index : 'fb',
            icon: 'fa-save',
            handler : function (env, event) {
                env.notebook.save_checkpoint();
                if(event){
                    event.preventDefault();
                }
                return false;
            }
        },
    };

    // private stuff that prepend `.ipython` to actions names
    // and uniformize/fill in missing pieces in of an action.
    var _prepare_handler = function(registry, subkey, source){
        registry['ipython.'+subkey] = {};
        registry['ipython.'+subkey].help = source[subkey].help||subkey.replace(/-/g,' ');
        registry['ipython.'+subkey].help_index = source[subkey].help_index;
        registry['ipython.'+subkey].icon = source[subkey].icon;
        return source[subkey].handler;
    };

    // Will actually generate/register all the Jupyter actions
    var fun = function(){
        var final_actions = {};
        var k;
        for(k in _actions){
            if(_actions.hasOwnProperty(k)){
                // Js closure are function level not block level need to wrap in a IIFE
                // and append ipython to event name these things do intercept event so are wrapped
                // in a function that return false.
                var handler = _prepare_handler(final_actions, k, _actions);
                (function(key, handler){
                    final_actions['ipython.'+key].handler = function(env, event){
                        handler(env);
                        if(event){
                            event.preventDefault();
                        }
                        return false;
                    };
                })(k, handler);
            }
        }

        for(k in custom_ignore){
            // Js closure are function level not block level need to wrap in a IIFE
            // same as above, but decide for themselves wether or not they intercept events.
            if(custom_ignore.hasOwnProperty(k)){
                var handler = _prepare_handler(final_actions, k, custom_ignore);
                (function(key, handler){
                    final_actions['ipython.'+key].handler = function(env, event){
                        return handler(env, event);
                    };
                })(k, handler);
            }
        }

        return final_actions;
    };
    ActionHandler.prototype._actions = fun();

    /**
     *  extend the environment variable that will be pass to handlers
     **/
    ActionHandler.prototype.extend_env = function(env){
        for(var k in env){
            this.env[k] = env[k];
        }
    };

    ActionHandler.prototype.register = function(action, name, prefix){
        /**
         * Register an `action` with an optional name and prefix.
         *
         * if name and prefix are not given they will be determined automatically.
         * if action if just a `function` it will be wrapped in an anonymous action.
         *
         * @return the full name to access this action .
         **/
        action = this.normalise(action);
        if( !name ){
            name = 'autogenerated-'+String(action.handler);
        }
        prefix = prefix || 'auto';
        var full_name = prefix+'.'+name;
        this._actions[full_name] = action;
        return full_name;

    };

    ActionHandler.prototype.normalise = function(data){
        /**
         * given an `action` or `function`, return a normalised `action`
         * by setting all known attributes and removing unknown attributes;
         **/
        if(typeof(data) === 'function'){
            data = {handler:data};
        }
        if(typeof(data.handler) !== 'function'){
            throw('unknown datatype, cannot register');
        }
        var _data = data;
        data = {};
        data.handler = _data.handler;
        data.help = _data.help || '';
        data.icon = _data.icon || '';
        data.help_index = _data.help_index || '';
        return data;
    };

    ActionHandler.prototype.get_name = function(name_or_data){
        /**
         * given an `action` or `name` of a action, return the name attached to this action.
         * if given the name of and corresponding actions does not exist in registry, return `null`.
         **/

        if(typeof(name_or_data) === 'string'){
            if(this.exists(name_or_data)){
                return name_or_data;
            } else {
                return null;
            }
        } else {
            return this.register(name_or_data);
        }
    };

    ActionHandler.prototype.get = function(name){
        return this._actions[name];
    };

    ActionHandler.prototype.call = function(name, event, env){
        return this._actions[name].handler(env|| this.env, event);
    };

    ActionHandler.prototype.exists = function(name){
        return (typeof(this._actions[name]) !== 'undefined');
    };

    return {init:ActionHandler};

});


text:
class op_cx_scalar_times
  {
  public:

  template<typename T1>
  inline static void
  apply
    (
          Mat< typename std::complex<typename T1::pod_type> >& out,
    const mtOp<typename std::complex<typename T1::pod_type>, T1, op_cx_scalar_times>& X
    );

  template<typename T1>
  inline static void
  apply
    (
             Cube< typename std::complex<typename T1::pod_type> >& out,
    const mtOpCube<typename std::complex<typename T1::pod_type>, T1, op_cx_scalar_times>& X
    );

  };

class op_cx_scalar_plus
  {
  public:

  template<typename T1>
  inline static void
  apply
    (
          Mat< typename std::complex<typename T1::pod_type> >& out,
    const mtOp<typename std::complex<typename T1::pod_type>, T1, op_cx_scalar_plus>& X
    );

  template<typename T1>
  inline static void
  apply
    (
             Cube< typename std::complex<typename T1::pod_type> >& out,
    const mtOpCube<typename std::complex<typename T1::pod_type>, T1, op_cx_scalar_plus>& X
    );

  };

class op_cx_scalar_minus_pre
  {
  public:

  template<typename T1>
  inline static void
  apply
    (
          Mat< typename std::complex<typename T1::pod_type> >& out,
    const mtOp<typename std::complex<typename T1::pod_type>, T1, op_cx_scalar_minus_pre>& X
    );

  template<typename T1>
  inline static void
  apply
    (
             Cube< typename std::complex<typename T1::pod_type> >& out,
    const mtOpCube<typename std::complex<typename T1::pod_type>, T1, op_cx_scalar_minus_pre>& X
    );

  };

class op_cx_scalar_minus_post
  {
  public:

  template<typename T1>
  inline static void
  apply
    (
          Mat< typename std::complex<typename T1::pod_type> >& out,
    const mtOp<typename std::complex<typename T1::pod_type>, T1, op_cx_scalar_minus_post>& X
    );

  template<typename T1>
  inline static void
  apply
    (
             Cube< typename std::complex<typename T1::pod_type> >& out,
    const mtOpCube<typename std::complex<typename T1::pod_type>, T1, op_cx_scalar_minus_post>& X
    );

  };

class op_cx_scalar_div_pre
  {
  public:

  template<typename T1>
  inline static void
  apply
    (
          Mat< typename std::complex<typename T1::pod_type> >& out,
    const mtOp<typename std::complex<typename T1::pod_type>, T1, op_cx_scalar_div_pre>& X
    );

  template<typename T1>
  inline static void
  apply
    (
             Cube< typename std::complex<typename T1::pod_type> >& out,
    const mtOpCube<typename std::complex<typename T1::pod_type>, T1, op_cx_scalar_div_pre>& X
    );

  };

class op_cx_scalar_div_post
  {
  public:

  template<typename T1>
  inline static void
  apply
    (
          Mat< typename std::complex<typename T1::pod_type> >& out,
    const mtOp<typename std::complex<typename T1::pod_type>, T1, op_cx_scalar_div_post>& X
    );

  template<typename T1>
  inline static void
  apply
    (
             Cube< typename std::complex<typename T1::pod_type> >& out,
    const mtOpCube<typename std::complex<typename T1::pod_type>, T1, op_cx_scalar_div_post>& X
    );

  };

//! @}


text:
var grunt = require('grunt'),
  manifestUrl = require('../grunt-utils/manifest_url.js').manifestUrl;

/*
  Make substitutions for templated parts of files where substitutions
  need to happen after finger-printed keys generated.
 */

module.exports = function(grunt) {
  //can add additional files here
  var templateFiles = ['css/glazier.css', 'css/glazier_card.css'];

  grunt.registerTask("templateCSS:devOnly", "process templateCSS:devOnly", function() {
    if (process.env.GLAZIER_ENV === "prod") {
      return;
    }
    for (var i=0; i<templateFiles.length; i++) {
      var templateFile = templateFiles[i];
      templateDevFile(templateFile, devConfig);
    }
  });
  grunt.registerTask("templateCSS:prodOnly", "process templateCSS:prodOnly", function() {
    if (process.env.GLAZIER_ENV !== "prod") {
      return;
    }
    for (var i=0; i<templateFiles.length; i++) {
      var templateFile = templateFiles[i];
      templateProdFile(templateFile);
    }
  });
};

//processed dev file => tmp/public
var devConfig = {
  tempDir: "tmp",
  templateDir: "public",
  buildDir: "public",
  sourceFile: function(template){
    return [this.tempDir, this.templateDir, template].join("/");
  },
  writeFile: function(template){
    return [this.tempDir, this.buildDir, template].join("/");
  },
  getData: function(){ return data; },
  data: {
    manifestUrl: function(path) {
      return path;
    }
  }
}

function templateDevFile(templatePath, config) {
  var template = grunt.file.read(config.sourceFile(templatePath));
  var indexContents = grunt.template.process(template, { data: config.data});
  console.log(config.writeFile(templatePath));
  grunt.file.write(config.writeFile(templatePath), indexContents);
}

//processed prod file => tmp/md5

/*
  eventually might want to extract a config and
  unify the process for dev/prod generation but prod side
  needs a bit of unraveling
*/
function templateProdFile(templatePath) {
  var manifest = grunt.file.readJSON('tmp/manifest.json');
  var md5File = getMD5Filename(manifest, templatePath);
  var template = grunt.file.read(md5File);
  var assetHost = grunt.config.get('pkg').assetHost || '';
  console.log(md5File);
  var indexContents = grunt.template.process(template, {
    data: {
      manifestUrl: manifestUrl(manifest),
      manifest: JSON.stringify(manifest || {}),
      assetHost: assetHost
    }
  });
  grunt.file.write(md5File, indexContents);
}

function getMD5Filename(manifest, path) {
  var file =  manifest["/"+path];

  if (!file) {
    throw new Error("No file found in manifest for path: " + path);
  } else {
    var md5FileArray = file.split("/");
    var md5File =  md5FileArray[md5FileArray.length-1];
    var pathArray = path.split("/");
    pathArray.pop();
  }

  return "tmp/md5/" + pathArray.join("/") + "/" + md5File;
}


text:
{-# LANGUAGE FlexibleContexts #-}
module IRC.Commands where

import           Control.Applicative
import           Control.Monad
import           Data.Text (Text)
import qualified Data.Text as T
import qualified Data.Text.Encoding as T
import           Data.ByteString (ByteString)
import           IRC.Types
import           IRC.Raw.Monad
import           Data.String
import qualified IRC.Raw as Raw

commandMatchServerHost :: Cmd -> ([Text] -> Maybe a) -> Raw.Message -> Maybe (Host, a)
commandMatchServerHost cmd params_fn = cnd
    where cnd (Raw.Message
                    _
                    (Just (Raw.ServerName (Raw.Host host)))
                    cmd_input
                    (Raw.Params params))
                | case (cmd, cmd_input) of
                       (N x, Raw.CmdNumber y) -> x == y
                       (S x, Raw.Command   y) -> x == y
                       (_  , _              ) -> False
                , Just rest <- params_fn (map (\(Raw.Param x) -> T.decodeUtf8 x) params)
                = Just (T.decodeUtf8 host
                       ,rest)
          cnd _ = Nothing

commandMatchUser :: Cmd -> ([Text] -> Maybe a) -> Raw.Message -> Maybe (User , a)
commandMatchUser cmd params_fn = cnd
    where cnd (Raw.Message
                    _
                    (Just (Raw.Prefix (Raw.Nick nick) (Just (Raw.User ident)) (Just hostp)))
                    cmd_input
                    (Raw.Params params))
                | case (cmd, cmd_input) of
                       (N x, Raw.CmdNumber y) -> x == y
                       (S x, Raw.Command   y) -> x == y
                       (_  , _              ) -> False
                , let host = case hostp of
                                Raw.ValidHost (Raw.Host h) -> h
                                Raw.InvalidHost h      -> h
                , Just rest <- params_fn (map (\(Raw.Param x) -> T.decodeUtf8 x) params)
                = Just (User (Nick (T.decodeUtf8 nick)) (T.decodeUtf8 ident) (T.decodeUtf8 host)
                       ,rest)
          cnd _ = Nothing

commandMatchRaw :: (Cmd -> [Text] -> Maybe a) -> Raw.Message -> Maybe a
commandMatchRaw params_fn = cnd
    where cnd (Raw.Message
                    _
                    _
                    cmd_input
                    (Raw.Params params))
                | let cmd = case cmd_input of
                       Raw.CmdNumber y -> N y
                       Raw.Command   y -> S y
                = params_fn cmd (map (\(Raw.Param x) -> T.decodeUtf8 x) params)

join_pattern :: Raw.Message -> Maybe (User , (Channel, [Text]))
join_pattern = commandMatchUser (S "JOIN") f
    where f (channel:metadata) = Just (Channel channel, metadata)
          f  _                 = Nothing

pattern JOIN user channel meta <- (join_pattern -> Just (user,(channel,meta)))

part_pattern :: Raw.Message -> Maybe (User , (Channel, Maybe Text))
part_pattern = commandMatchUser (S "PART") f
    where f [channel]   = Just (Channel channel, Nothing)
          f [channel,x] = Just (Channel channel, Just x )
          f  _          = Nothing

pattern PART user channel msg <- (part_pattern -> Just (user,(channel,msg)))

quit_pattern :: Raw.Message -> Maybe (User, (Channel, Maybe Text))
quit_pattern = commandMatchUser (S "QUIT") f
    where f [channel]   = Just (Channel channel, Nothing)
          f [channel,x] = Just (Channel channel, Just x )
          f _           = Nothing

pattern QUIT user channel msg <- (quit_pattern -> Just (user,(channel,msg)))

kick_pattern :: Raw.Message -> Maybe (User, (Channel, Nick, Maybe Text))
kick_pattern = commandMatchUser (S "QUIT") f
    where f [ch,kicked]          = Just (Channel ch, Nick kicked, Nothing      )
          f [ch,kicked,partmsg]  = Just (Channel ch, Nick kicked, Just partmsg )
          f _                    = Nothing

pattern KICK user channel kicked msg <- (kick_pattern -> Just (user,(channel,kicked,msg)))

channelmsg_pattern :: Raw.Message -> Maybe (User, (Channel, Message))
channelmsg_pattern = commandMatchUser (S "PRIVMSG") f
    where f [channel,msg]  | Just ('#', _) <- T.uncons channel
                           = Just (Channel channel , Message msg)
          f _              = Nothing

pattern CHMSG user channel msg <- (channelmsg_pattern -> Just (user,(channel,msg)))

privmsg_pattern :: Raw.Message -> Maybe (User, (Target,Message))
privmsg_pattern= commandMatchUser (S "PRIVMSG") f
    where f [channel,msg]  = Just (Target channel , Message msg)
          f _              = Nothing

pattern PRIVMSG user target msg <- (privmsg_pattern -> Just (user, (target,msg)))

nick_pattern :: Raw.Message -> Maybe (User, Nick)
nick_pattern= commandMatchUser (S "PRIVMSG") f
    where f [new_nick]  = Just (Nick new_nick)
          f _           = Nothing

pattern NICK user nick <- (nick_pattern -> Just (user, nick))

account_pattern :: Raw.Message -> Maybe (User, Maybe Account)
account_pattern = commandMatchUser (S "ACCOUNT") f
    where f ["*"]  = Just Nothing
          f [acc]  = Just (Just (Account acc))
          f  _     = Nothing

pattern ACCOUNT user acc <- (account_pattern -> Just (user, acc))

raw_pattern :: Raw.Message -> Maybe (Cmd, [Text])
raw_pattern = commandMatchRaw f
    where f cmd params = Just (cmd, params)

pattern RAW cmd pms <- (raw_pattern -> Just (cmd,pms))

encode :: [Text] -> Raw.Params
encode ts = Raw.Params (map (Raw.Param . T.encodeUtf8) ts)

cmd :: MonadIRC m => Text -> [Text] -> m ()
cmd cmd params = Raw.irc_send (command cmd params)

command :: Text -> [Text] -> Raw.Message
command cmd params = Raw.Message Nothing Nothing (Raw.Command (T.encodeUtf8 cmd)) (encode params)

msg :: (IRCCommand target, MonadIRC m) => target -> Message -> m ()
msg = privmsg

class IRCCommand target where
    privmsg :: MonadIRC m => target -> Message -> m ()
    notice  :: MonadIRC m => target -> Message -> m ()

instance IRCCommand User where
    privmsg (User (Nick target) _ _) (Message msg) = cmd "PRIVMSG" [target, msg]
    notice  (User (Nick target) _ _) (Message msg) = cmd "NOTICE"  [target, msg]

instance IRCCommand Channel where
    privmsg (Channel ch) (Message msg) = cmd "PRIVMSG" [ch, msg]
    notice  (Channel ch) (Message msg) = cmd "NOTICE"  [ch, msg]

instance IRCCommand Nick where
    privmsg (Nick n) (Message msg) = cmd "PRIVMSG" [n,msg]
    notice  (Nick n) (Message msg) = cmd "NOTICE"  [n,msg]

instance IRCCommand Target where
    privmsg (Target t) (Message msg) = cmd "PRIVMSG" [t,msg]
    notice  (Target t) (Message msg) = cmd "NOTICE"  [t,msg]

instance IsString Message where
    fromString = Message . fromString


text:
package org.camunda.bpm.engine.test.api.externaltask;

import java.io.Serializable;

public class ExternalTaskCustomValue implements Serializable {

  private static final long serialVersionUID = 1L;

  protected String testValue;

  public String getTestValue() {
    return testValue;
  }

  public void setTestValue(String testValue) {
    this.testValue = testValue;
  }

}


text:
//=================================================================================================

//=================================================================================================

#ifndef _BLAZE_MATH_SHIMS_RESET_H_
#define _BLAZE_MATH_SHIMS_RESET_H_

//*************************************************************************************************
// Includes
//*************************************************************************************************

#include <blaze/system/Inline.h>

namespace blaze {

//=================================================================================================
//
//  RESET SHIM
//
//=================================================================================================

//*************************************************************************************************
/*!\brief Resetting the given value/object to the default value.
// \ingroup math_shims
//
// \param resettable The value/object to be resetted.
// \return void
//
// The \a reset shim represents an abstract interface for the resetting of a value/object of
// any given data type to its default value. Values of built-in data type are reset to zero.
*/
template< typename Type >
BLAZE_ALWAYS_INLINE constexpr void reset( Type& resettable )
{
   resettable = Type();
}
//*************************************************************************************************

} // namespace blaze

#endif


text:
        <form novalidate class="container css-form" name="login">

                Name:   <input class="form-control" type="text" ng-model="user.Name" required id="userName" />

                E-mail:   <input class="form-control" type="email" ng-model="user.Email" required id="userEMail" />

                Password:   <input class="form-control" type="password" ng-model="user.Password" required id="userPassword" />

                <button class="btn btn-primary" ng-disabled="login.$invalid" ng-click="executeSignup()">Signup</button>
                Reset
                Login
                Reset password

        </form>

            Error:
            {{message}}


text:
<!DOCTYPE html>
<html>
<head>
  <meta http-equiv="content-type" content="text/html; charset=UTF-8">
  <title>T.js example 3 - a TODO list that uses data-binding</title>
  <script type="text/javascript" src="coffee-script.js"></script>
  <script type="text/javascript" src="watch.js"></script>
  <script type="text/javascript" src="../javascripts/jquery-1.8.2.min.js"></script>
  <script type="text/javascript" src="../javascripts/t.js"></script>
  <script type="text/coffeescript">

templates =
  main: (todos) ->
    [ "div#container"
      style:
        width: 300
        padding: 3
        border: "1px solid grey"

      [ "div.header"
        style:
          padding: 3
          background: "#fbb"
          font_weight: "bold"

        "My TODO List"
      ]
      [ "div.todos"
        style: width: "100%"

        templates.todos todos
      ]
      [ "input"
        type: "text"
        placeholder: "Type content and hit Enter"
        style: width: "98%"
        keyup: (event) ->
          return if event.which isnt 13

          event.preventDefault()
          todos.push $(this).val()
          $(this).val('')
      ]
    ]

  todo: (todos, index) ->
    [ "div.content"
      style: padding: 3

      T.escape(todos[index])
      "  "
      [ "a"
        href: "javascript:void(0)"
        click: -> todos.splice(index, 1)

        "X"
      ]
    ]

  todos: (todos) ->
    if todos.length > 0
      templates.todo(todos, i) for i in [0..todos.length - 1]

# Model
window.todos = ['Do this', 'Do that']

# Model change listener
watch window, 'todos', ->
  T(templates.todos(window.todos)).render inside: '.todos'

T(templates.main(todos)).render inside: 'body'

</script>
</head>
<body>
T.js example 2

This example shows a simple TODO list.
</body>
</html>


text:
@interface RTDataSourceAdapter ()
{
    RTselectHandler selectedHandler;
    RTwillDisplayCellHandler willDisplayHandler;
    RTdidEndDisplayingCellHandler endDisplayingHandler;
    RTvoidHandler refreshHandler;
    RTvoidHandler loadHandler;
    UITableView *table;
    UIView *footerView;
    UIImageView *footerActInd;
}

@property (assign, nonatomic) BOOL isPageLoading;

@end

@implementation RTDataSourceAdapter

- (instancetype)init
{
    self = [super init];

    if (self) {
        self.content = [[RTDataSourceAdapterContent alloc] init];
        self.pullToRefreshWidth = [[UIScreen mainScreen] bounds].size.width;
    }

    return self;
}

- (instancetype)initWithRTAdapterItems:(NSArray *)items
{
    self = [super init];

    if (self) {
        self.content = [[RTDataSourceAdapterContent alloc] initWithRTAdapterItems:items];
        self.pullToRefreshWidth = [[UIScreen mainScreen] bounds].size.width;
    }

    return self;
}

#pragma mark - Register TableView

- (void)registerTableView:(UITableView *)tableView
      cellIdentifier:(NSString *)identifier
    addPullToRefresh:(BOOL)value
  pageLoadingEnabled:(BOOL)loading
{
    table = tableView;

    tableView.delegate = self;
    tableView.dataSource = self;
    self.pageLoadingEnabled = loading;

    [self registerNibForTable:tableView cellIdentifier:identifier];

    if (value) [self addPullToRefreshControl];
    if (loading) [self addFooter];
}

- (void)registerTableView:(UITableView *)tableView
     cellIdentifiers:(NSArray *)identifiers
    addPullToRefresh:(BOOL)value
  pageLoadingEnabled:(BOOL)loading
{
    table = tableView;
    tableView.delegate = self;
    tableView.dataSource = self;
    self.pageLoadingEnabled = loading;

    for (NSString *identifier in identifiers) {
        [self registerNibForTable:tableView cellIdentifier:identifier];
    }

    if (value) [self addPullToRefreshControl];
    if (loading) [self addFooter];
}

- (void)registerNibForTable:(UITableView *)tableView cellIdentifier:(NSString *)identifier
{
    [tableView registerNib:[UINib nibWithNibName:identifier bundle:[NSBundle mainBundle]] forCellReuseIdentifier:identifier];
}

- (void)addPullToRefreshControl
{
    refreshControl = [[RTRefreshControl alloc] init];
    [table addSubview:refreshControl];
    [refreshControl addTarget:self
                       action:@selector(refresh)
             forControlEvents:UIControlEventValueChanged];

    [refreshControl setup:self.pullToRefreshWidth color:table.tintColor];
}

- (void)addFooter
{
    CGRect screenRect = [[UIScreen mainScreen] bounds];
    CGRect footerFrame = CGRectMake(0, 0, screenRect.size.width, 44);
    footerView = [[UIView alloc] initWithFrame:footerFrame];
    footerView.backgroundColor = [UIColor clearColor];
    footerView.hidden = false;
    footerActInd = [[UIImageView alloc] initWithImage:[UIImage imageNamed:@"rt_loader"]];
    footerActInd.image = [footerActInd.image imageWithRenderingMode:UIImageRenderingModeAlwaysTemplate];
    footerActInd.tintColor = table.tintColor;
    footerActInd.center = CGPointMake(screenRect.size.width/2.0, 30);
    [footerView addSubview:footerActInd];
}

#pragma mark - Handlers

- (void)setPullToRefreshHandler:(RTvoidHandler)refresh
{
    refreshHandler = refresh;
}

- (void)setLoadMoreHandler:(RTvoidHandler)load
{
    loadHandler = load;
}

- (void)setSelectHandler:(RTselectHandler)selected
{
    selectedHandler = selected;
}

- (void)setWillDisplayCellHandler:(RTwillDisplayCellHandler)willDisplay
{
    willDisplayHandler = willDisplay;
}

- (void)setDidEndDisplayingCellHandler:(RTdidEndDisplayingCellHandler)endDisplaying
{
    endDisplayingHandler = endDisplaying;
}

#pragma mark - SupportMethods

- (void)reloadData
{
    [table reloadData];
    [self hideLoading];
    self.isPageLoading = NO;
}

- (void)deleteRow:(RTAdapterItem *)item
{
    [table deleteRowsAtIndexPaths:@[item.index] withRowAnimation:UITableViewRowAnimationFade];
}

- (void)insertRow:(int)position
{
    [table insertRowsAtIndexPaths:@[[NSIndexPath indexPathForRow:position inSection:0]] withRowAnimation:UITableViewRowAnimationFade];
}

- (void)selectRow:(RTAdapterItem *)item
{
    if (item.index)
    [table selectRowAtIndexPath:item.index animated:NO  scrollPosition:UITableViewScrollPositionMiddle];
}

- (void)selectRowPosition:(int)position
{
    [table selectRowAtIndexPath:[NSIndexPath indexPathForRow:position inSection:0] animated:NO  scrollPosition:UITableViewScrollPositionMiddle];
}

- (void)scrollToRowAtPosition:(int)position atScrollPosition:(UITableViewScrollPosition)scroll animated:(BOOL)anim
{
    [table scrollToRowAtIndexPath:[NSIndexPath indexPathForRow:position inSection:0]
                     atScrollPosition:scroll animated:anim];
}

- (void)scrollToRowAtItem:(RTAdapterItem *)item atScrollPosition:(UITableViewScrollPosition)scroll animated:(BOOL)anim
{
    [table scrollToRowAtIndexPath:item.index atScrollPosition:scroll animated:anim];
}

- (void)refresh
{
    if (refreshHandler) refreshHandler();

    [refreshControl endRefreshing];
}

- (void)scrollViewDidScroll:(UIScrollView *)scrollView
{
    [refreshControl pull];
}

- (void)loadMore
{
    if (loadHandler) loadHandler();
}

- (void)showLoading
{
    footerView.hidden = NO;
    table.tableFooterView = footerView;
    [footerActInd rt_startAnimating];
}

- (void)hideLoading
{
    footerView.hidden = YES;
    table.tableFooterView = nil;
    [footerActInd rt_stopAnimating];
}

#pragma mark - table methods

- (NSInteger)numberOfSectionsInTableView:(UITableView *)tableView
{
    return 1;
}

- (NSInteger)tableView:(UITableView *)tableView numberOfRowsInSection:(NSInteger)section
{
    if (self.content) return [self.content getCount];
    return 0;
}

- (UITableViewCell *)tableView:(UITableView *)tableView cellForRowAtIndexPath:(NSIndexPath *)indexPath
{
    RTAdapterItem *item = [self.content getItem:indexPath.row];
    if (!item) return [UITableViewCell new];

    item.index = indexPath;

    RTBaseAdapterTableViewCell *cell = [tableView dequeueReusableCellWithIdentifier:item.itemID forIndexPath:indexPath];

    if (indexPath.row == 0) {
        cell.isFirst = YES;
    } else if ([self.content getCount] - 1 == indexPath.row){
        cell.isLast = YES;
    }

    if (item.isSelected) [self selectRow:item];

    [cell cellContent:item.itemValue];

    return cell;
}

- (CGFloat)tableView:(UITableView *)tableView heightForRowAtIndexPath:(NSIndexPath *)indexPath
{
    RTAdapterItem *item = [self.content getItem:indexPath.row];

    Class<RTCellHeight> cellClass = NSClassFromString(item.itemID);

    return [cellClass heightFromCellContent:item.itemValue];
}

- (void)tableView:(UITableView *)tableView didSelectRowAtIndexPath:(NSIndexPath *)indexPath
{
    if (self.deselectable) [tableView deselectRowAtIndexPath:indexPath animated:NO];
    if (selectedHandler) selectedHandler([self.content getItem:indexPath.row], indexPath);
}

-(void)tableView:(UITableView *)tableView didDeselectRowAtIndexPath:(NSIndexPath *)indexPath
{
    RTAdapterItem *item = [self.content getItem:indexPath.row];

    item.isSelected = NO;
}

- (void)tableView:(UITableView *)tableView willDisplayCell:(UITableViewCell *)cell forRowAtIndexPath:(NSIndexPath *)indexPath
{
     if (willDisplayHandler) willDisplayHandler([self.content getItem:indexPath.row], indexPath);

    if (self.isPageLoading || !self.pageLoadingEnabled) {
        return;
    }

    if ((indexPath.row == [self.content getCount] - 1) && self.isPageLoading == false) {
        [self showLoading];
        self.isPageLoading = YES;

        __weak typeof(self)weakSelf = self;

        dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.5 * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{
            [weakSelf loadMore];
        });

    }
}

- (void)tableView:(UITableView *)tableView didEndDisplayingCell:(UITableViewCell *)cell forRowAtIndexPath:(NSIndexPath *)indexPath
{
    if (endDisplayingHandler) endDisplayingHandler([self.content getItem:indexPath.row], indexPath);
}

@end


text:
using System;
using System.IO;
using System.Net;

namespace Servya
{
	public interface IHttpResponse : IDisposable
	{
		HttpStatusCode Status { get; set; }
		Stream Stream { get; }
		bool Chunked { get; set; }
		long ContentLength { get; set; }
		string ContentType { get; set; }
		bool KeepAlive { get; set; }
	}
}