text: Flask Project Structure

redpajama_set_name: RedPajamaGithub


text:
using System;
using QuantConnect.Data;
using System.Collections;
using QuantConnect.Interfaces;
using QuantConnect.Securities;
using System.Collections.Generic;
using QuantConnect.Data.Auxiliary;
using QuantConnect.Util;

namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
{
    /// <summary>
    /// A live trading delisting event provider which uses a <see cref="DelistingEventProvider"/> internally to emit events
    /// based on the current frontier time
    /// </summary>
    /// <remarks>We create the events dynamically so that we can set the current security price in the event</remarks>
    public class LiveDelistingEventProviderEnumerator : IEnumerator<BaseData>
    {
        private DateTime _lastTime;
        private readonly ITimeProvider _timeProvider;
        private readonly Queue<BaseData> _dataToEmit;
        private readonly SecurityCache _securityCache;
        private readonly SubscriptionDataConfig _dataConfig;
        private readonly DelistingEventProvider _delistingEventProvider;

        /// <summary>
        /// Gets the element in the collection at the current position of the enumerator.
        /// </summary>
        /// <returns>The element in the collection at the current position of the enumerator.</returns>
        public BaseData Current { get; private set; }

        /// <summary>
        /// Gets the current element in the collection.
        /// </summary>
        /// <returns>The current element in the collection.</returns>
        object IEnumerator.Current => Current;

        /// <summary>
        /// Creates a new instance
        /// </summary>
        private LiveDelistingEventProviderEnumerator(ITimeProvider timeProvider, SubscriptionDataConfig dataConfig,
            SecurityCache securityCache, DelistingEventProvider delistingEventProvider, MapFile mapFile)
        {
            _dataConfig = dataConfig;
            _timeProvider = timeProvider;
            _securityCache = securityCache;
            _dataToEmit = new Queue<BaseData>();
            _delistingEventProvider = delistingEventProvider;
            _delistingEventProvider.Initialize(dataConfig, null, mapFile, DateTime.UtcNow);
        }

        /// <summary>
        /// Advances the enumerator to the next element of the collection.
        /// </summary>
        /// <returns> true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.</returns>
        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created.</exception>
        public bool MoveNext()
        {
            var currentDate = _timeProvider.GetUtcNow().ConvertFromUtc(_dataConfig.ExchangeTimeZone).Date;
            if (currentDate != _lastTime)
            {
                // when the date changes for the security we trigger a new tradable date event
                var newDayEvent = new NewTradableDateEventArgs(currentDate, _securityCache.GetData(), _dataConfig.Symbol, null);
                foreach (var delistingEvent in _delistingEventProvider.GetEvents(newDayEvent))
                {
                    _dataToEmit.Enqueue(delistingEvent);
                }

                // update last time
                _lastTime = currentDate;
            }

            if (_dataToEmit.Count > 0)
            {
                // emit event if any
                Current = _dataToEmit.Dequeue();
                return true;
            }

            Current = null;
            return false;
        }

        /// <summary>
        /// Sets the enumerator to its initial position, which is before the first element in the collection.
        /// </summary>
        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created.</exception>
        public void Reset()
        {
            _dataToEmit.Clear();
        }

        /// <summary>
        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
        /// </summary>
        public void Dispose()
        {
            var liveProvider = _delistingEventProvider as LiveDataBasedDelistingEventProvider;
            liveProvider.DisposeSafely();
        }

        /// <summary>
        /// Helper method to create a new instance.
        /// Knows which security types should create one and determines the appropriate delisting event provider to use
        /// </summary>
        public static bool TryCreate(SubscriptionDataConfig dataConfig, ITimeProvider timeProvider, IDataQueueHandler dataQueueHandler,
            SecurityCache securityCache, IMapFileProvider mapFileProvider, out IEnumerator<BaseData> enumerator)
        {
            enumerator = null;
            var securityType = dataConfig.SecurityType;
            if (securityType.IsOption() || securityType == SecurityType.Future || securityType == SecurityType.Equity)
            {
                var delistingEventProvider = new DelistingEventProvider();
                MapFile mapFile = null;
                if (securityType == SecurityType.Equity)
                {
                    delistingEventProvider = new LiveDataBasedDelistingEventProvider(dataConfig, dataQueueHandler);
                    mapFile = mapFileProvider.Get(dataConfig.Symbol.ID.Market).ResolveMapFile(dataConfig.Symbol, dataConfig.Type);
                }
                enumerator = new LiveDelistingEventProviderEnumerator(timeProvider, dataConfig, securityCache, delistingEventProvider, mapFile);
                return true;
            }
            return false;
        }
    }
}


text:
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">

<TITLE>
FTFont (sdljava api)
</TITLE>

<META NAME="keywords" CONTENT="org.gljava.opengl.ftgl.FTFont class">

<LINK REL ="stylesheet" TYPE="text/css" HREF="../../../../stylesheet.css" TITLE="Style">

      Overview
      Package
    Class
      Use
      Tree
      Deprecated
      Index
      Help

 PREV CLASS
 NEXT CLASS

  FRAMES
 NO FRAMES

  All Classes

  SUMMARY: NESTED | FIELD | CONSTR | METHOD

DETAIL: FIELD | CONSTR | METHOD

org.gljava.opengl.ftgl

Class FTFont

java.lang.Object
  org.gljava.opengl.ftgl.FTFont

Direct Known Subclasses:
FTGLBitmapFont, FTGLExtrdFont, FTGLOutlineFont, FTGLPixmapFont, FTGLPolygonFont, FTGLTextureFont

public class FTFont
extends java.lang.Object

Field Summary

protected  boolean
swigCMemOwn

private  long
swigCPtr

Constructor Summary

protected
FTFont()

protected
FTFont(long cPtr,
       boolean cMemoryOwn)

Method Summary

 float
advance(java.lang.String string)

 float
advance(SWIGTYPE_p_wchar_t string)

 float
ascender()

 boolean
attach(java.lang.String fontFilePath)

 boolean
attach(SWIGTYPE_p_unsigned_char pBufferBytes,
       int bufferSizeInBytes)

 void
bBox(java.lang.String string,
     float[] arg1,
     float[] arg2,
     float[] arg3,
     float[] arg4,
     float[] arg5,
     float[] arg6)

 boolean
charMap(SWIGTYPE_p_FT_Encoding encoding)

 long
charMapCount()

 SWIGTYPE_p_FT_Encoding
charMapList()

 void
delete()

 void
depth(float depth)

 float
descender()

 int
error()

 long
faceSize()

 boolean
faceSize(long size,
         long res)

protected  void
finalize()

 BoundingBox
getBoundingBox(java.lang.String string)

protected static long
getCPtr(FTFont obj)

 float
lineHeight()

 void
render(java.lang.String string)

 void
useDisplayList(boolean useList)

Methods inherited from class java.lang.Object

clone, equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

swigCPtr

private long swigCPtr

swigCMemOwn

protected boolean swigCMemOwn

Constructor Detail

FTFont

protected FTFont(long cPtr,
                 boolean cMemoryOwn)

FTFont

protected FTFont()

Method Detail

getCPtr

protected static long getCPtr(FTFont obj)

finalize

protected void finalize()

Overrides:
finalize in class java.lang.Object

delete

public void delete()

attach

public boolean attach(java.lang.String fontFilePath)

attach

public boolean attach(SWIGTYPE_p_unsigned_char pBufferBytes,
                      int bufferSizeInBytes)

charMap

public boolean charMap(SWIGTYPE_p_FT_Encoding encoding)

charMapCount

public long charMapCount()

charMapList

public SWIGTYPE_p_FT_Encoding charMapList()

faceSize

public boolean faceSize(long size,
                        long res)

faceSize

public long faceSize()

depth

public void depth(float depth)

useDisplayList

public void useDisplayList(boolean useList)

ascender

public float ascender()

descender

public float descender()

lineHeight

public float lineHeight()

bBox

public void bBox(java.lang.String string,
                 float[] arg1,
                 float[] arg2,
                 float[] arg3,
                 float[] arg4,
                 float[] arg5,
                 float[] arg6)

advance

public float advance(SWIGTYPE_p_wchar_t string)

advance

public float advance(java.lang.String string)

render

public void render(java.lang.String string)

error

public int error()

getBoundingBox

public BoundingBox getBoundingBox(java.lang.String string)

      Overview
      Package
    Class
      Use
      Tree
      Deprecated
      Index
      Help

 PREV CLASS
 NEXT CLASS

  FRAMES
 NO FRAMES

  All Classes

  SUMMARY: NESTED | FIELD | CONSTR | METHOD

DETAIL: FIELD | CONSTR | METHOD


text: window.isWebApp = true;


text:
@font-face {
  font-family: 'icomoon';
  font-weight: normal;
  src: url("../fonts/icomoon.eot");
  src: url("../fonts/icomoon.eot?#iefix") format('embedded-opentype'),url("../fonts/icomoon.woff") format('woff'),url("../fonts/icomoon.ttf") format('truetype'),url("../fonts/icomoon.svg#icomoonn") format('svg');
}

/* normalize.css v3.0.0 | MIT License | git.io/normalize */

html {
  font-family: sans-serif;
  -ms-text-size-adjust: 100%;
  -webkit-text-size-adjust: 100%;
}

body {
  margin: 0;
}

article,
figure,
footer,
header {
  display: block;
}

a {
  background: transparent;
}

a:active,
a:hover {
  outline: 0;
}

img {
  border: 0;
}

figure {
  margin: 1em 40px;
}

input {
  color: inherit;
  font: inherit;
  margin: 0;
}

input[type="submit"] {
  -webkit-appearance: button;
  cursor: pointer;
}

button::-moz-focus-inner,
input::-moz-focus-inner {
  border: 0;
  padding: 0;
}

input {
  line-height: normal;
}

[class^="icon-"] {
  font-family: 'icomoon';
  speak: none;
  font-style: normal;
  font-weight: normal;
  font-variant: normal;
  text-transform: none;
  line-height: 1;
  -webkit-font-smoothing: antialiased;
  -moz-osx-font-smoothing: grayscale;
}

.icon-mail-transp:before {
  content: "\f003";
}

.icon-user:before {
  content: "\f007";
}

.icon-check:before {
  content: "\f00c";
}

.icon-clock:before {
  content: "\f017";
}

.icon-map:before {
  content: "\f041";
}

.icon-mail-color:before {
  content: "\f0e0";
}

.icon-telefono:before {
  content: "\e602";
}

a {
  text-decoration: none;
}

ul,
p,
figure,
h2 {
  margin: 0;
  padding: 0;
}

ul {
  list-style: none;
}

body {
  font-family: 'Open Sans',arial;
  font-size: 16px;
  font-weight: 400;
}

header {
  background-color: #00359c;
  padding: 5px 1rem;
}

header figure {
  text-align: center;
}

header figure img {
  height: auto;
  max-width: 100%;
}

header .info {
  text-align: right;
  color: #fff;
  font-size: 1.1rem;
}

header .info ul {
  margin: 10px 0;
}

header .info ul li {
  display: inline-block;
  vertical-align: top;
}

header .info ul li:first-child {
  margin-right: 7px;
}

header .info ul li a {
  display: block;
  height: 25px;
  width: 25px;
  background-image: url("../img/sprite-social.png");
}

header .info ul li a:hover {
  opacity: .8;
  -ms-filter: "progid:DXImageTransform.Microsoft.Alpha(Opacity=80)";
  filter: alpha(opacity=80);
}

header .info ul li:first-child a {
  background-position: 0 0;
}

header .info ul li:last-child a {
  background-position: 25px 0;
}

.slider {
  font-size: 0;
}

.slider img {
  max-width: 100%;
  height: auto;
}

.newsletter {
  background-color: #00359c;
}

.newsletter .ancho {
  padding: .5rem 1rem;
}

.newsletter .ancho h2 {
  color: #fff;
  font-size: 18px;
  font-weight: 300;
  text-align: center;
}

.newsletter .ancho .formulario .nombre,
.newsletter .ancho .formulario .user,
.newsletter .ancho .formulario .mail,
.newsletter .ancho .formulario .boton {
  position: relative;
  color: #acacac;
  text-align: center;
  margin: 15px auto;
  width: 244px;
}

.newsletter .ancho .formulario .nombre span,
.newsletter .ancho .formulario .user span,
.newsletter .ancho .formulario .mail span,
.newsletter .ancho .formulario .boton span {
  position: absolute;
  top: 10px;
  left: 8px;
  font-size: 1.2rem;
}

.newsletter .ancho .formulario .nombre input[type="text"],
.newsletter .ancho .formulario .user input[type="email"],
.newsletter .ancho .formulario .mail input[type="email"] {
  border: 0;
  -webkit-box-shadow: inset 0 0 5px 0 rgba(10,11,2,0.5);
  box-shadow: inset 0 0 5px 0 rgba(10,11,2,0.5);
  padding: 8px 30px;
}

.newsletter .ancho .formulario .boton {
  margin-left: auto;
  margin-right: auto;
  width: 150px;
}

.newsletter .ancho .formulario .boton input[type="submit"] {
  background-color: #ffe829;
  border: 0;
  border-bottom: 4px solid #897c10;
  -webkit-border-radius: 5px;
  border-radius: 5px;
  color: #897c10;
  font-weight: 700;
  padding: 12px 15px 12px 35px;
  text-transform: uppercase;
}

.newsletter .ancho .formulario .boton span {
  color: #897c10;
  left: 12px;
  top: 13px;
}

.sucursales {
  background-color: #0d2d69;
  -webkit-box-sizing: border-box;
  -moz-box-sizing: border-box;
  box-sizing: border-box;
  color: #fff;
  padding: 10px 1rem 25px 1rem;
}

.sucursales .ancho h2 {
  padding: 15px 0;
  text-align: center;
  text-transform: uppercase;
}

.sucursales .ancho .sucursal {
  text-align: center;
}

.sucursales .ancho .sucursal .lugar {
  font-size: 18px;
  font-weight: 700;
  padding: 10px 0;
  text-transform: capitalize;
}

.sucursales .ancho .sucursal .telefono,
.sucursales .ancho .sucursal .horario,
.sucursales .ancho .sucursal .direccion {
  font-size: 14px;
  font-weight: 300;
}

.mapa {
  min-height: 300px;
  width: auto;
}

footer {
  background-color: #0d2d69;
  color: #fff;
  padding: 3rem 1rem;
  text-align: center;
  width: auto;
}

footer .copy {
  font-size: 14px;
  font-weight: 300;
}

footer figure img {
  height: auto;
  max-width: 100%;
}

@media screen and (min-width:468px) {
  header.small figure {
    text-align: left;
  }
}

@media screen and (min-width:768px) {
  header.medium figure,
  header.medium .info {
    display: inline-block;
    vertical-align: top;
    width: 49%;
  }

  .newsletter.medium .formulario {
    text-align: center;
  }

  .newsletter.medium .formulario h2 {
    text-align: left;
  }

  .newsletter.medium .formulario .nombre,
  .newsletter.medium .formulario .user,
  .newsletter.medium .formulario .mail,
  .newsletter.medium .formulario .boton {
    display: inline-block;
  }

  .newsletter.medium .formulario .nombre,
  .newsletter.medium .formulario .user,
  .newsletter.medium .formulario .mail {
    margin-right: 3.9rem;
    width: 179px;
  }

  .newsletter.medium .formulario .nombre input,
  .newsletter.medium .formulario .user input,
  .newsletter.medium .formulario .mail input {
    padding-right: 5px;
    width: 100%;
  }

  .sucursales.medium .ancho {
    margin: 0 auto;
    width: 70%;
    zoom: 1;
  }

  .sucursales.medium .ancho:before,
  .sucursales.medium .ancho:after {
    content: "";
    display: table;
  }

  .sucursales.medium .ancho:after {
    clear: both;
  }

  .sucursales.medium .sucursal {
    text-align: left;
    float: left;
    width: 250px;
  }

  .sucursales.medium .sucursal:nth-child(2n + 1) {
    float: right;
  }

  .sucursales.medium .sucursal .lugar {
    padding-left: .3rem;
  }

  footer.medium {
    zoom: 1;
  }

  footer.medium:before,
  footer.medium:after {
    content: "";
    display: table;
  }

  footer.medium:after {
    clear: both;
  }

  footer.medium .copy {
    float: left;
  }

  footer.medium figure {
    float: right;
  }
}

@media screen and (min-width:1000px) {
  header.large .ancho {
    margin: 0 auto;
    width: 58.75rem;
  }

  header.large .ancho .info {
    font-size: 1rem;
  }

  .newsletter.large .ancho {
    -webkit-box-sizing: border-box;
    -moz-box-sizing: border-box;
    box-sizing: border-box;
    margin: 0 auto;
    padding-left: 5rem;
    width: 58.75rem;
  }

  .newsletter.large .ancho .formulario {
    text-align: left;
  }

  .newsletter.large .ancho .formulario .nombre span,
  .newsletter.large .ancho .formulario .user span,
  .newsletter.large .ancho .formulario .mail span {
    left: 4px;
    top: 6px;
  }

  .newsletter.large .ancho .formulario .boton {
    width: auto;
  }

  .newsletter.large .ancho .formulario .boton input[type="submit"] {
    font-size: 18px;
    letter-spacing: 1.5px;
    padding: 22px 15px 20px 45px;
    position: absolute;
    top: -40px;
  }

  .newsletter.large .ancho .formulario .boton span {
    top: -18px;
  }

  .sucursales.large .ancho {
    text-align: center;
  }

  .sucursales.large .ancho .sucursal {
    display: inline-block;
    float: none;
    margin-bottom: 1.5rem;
    margin-right: 2rem;
    vertical-align: top;
  }

  .sucursales.large .ancho .sucursal:nth-child(3n + 4) {
    margin-right: 0;
  }

  footer.large .ancho {
    margin: 0 auto;
    width: 58.75rem;
  }
}


text:
import mongoose from '../stores/mongoose.js'
import { Schema } from 'mongoose'

const postSchema = new Schema({
  title: {
    type: String,
    required: true
  },
  content: {
    type: String,
    required: true
  },
  cover: String,
  creatorAvatarUrl: String,
  summary: String,
  tag: {
    type: Schema.Types.ObjectId,
    ref: 'Tag'
  }
}, {
  timestamps: true
})

const Post = mongoose.model('Post', postSchema)

export default Post


text:
"""Tools for inspecting Python objects.

Uses syntax highlighting for presenting the various information elements.

Similar in spirit to the inspect module, but all calls take a name argument to
reference the name under which an object is being read.
"""

# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.

from __future__ import print_function

__all__ = ['Inspector','InspectColors']

# stdlib modules
import inspect
import linecache
import os
from textwrap import dedent
import types
import io as stdlib_io

try:
    from itertools import izip_longest
except ImportError:
    from itertools import zip_longest as izip_longest

# IPython's own
from IPython.core import page
from IPython.lib.pretty import pretty
from IPython.testing.skipdoctest import skip_doctest_py3
from IPython.utils import PyColorize
from IPython.utils import io
from IPython.utils import openpy
from IPython.utils import py3compat
from IPython.utils.dir2 import safe_hasattr
from IPython.utils.path import compress_user
from IPython.utils.text import indent
from IPython.utils.wildcard import list_namespace
from IPython.utils.coloransi import TermColors, ColorScheme, ColorSchemeTable
from IPython.utils.py3compat import cast_unicode, string_types, PY3
from IPython.utils.signatures import signature

# builtin docstrings to ignore
_func_call_docstring = types.FunctionType.__call__.__doc__
_object_init_docstring = object.__init__.__doc__
_builtin_type_docstrings = {
    inspect.getdoc(t) for t in (types.ModuleType, types.MethodType,
                                types.FunctionType, property)
}

_builtin_func_type = type(all)
_builtin_meth_type = type(str.upper)  # Bound methods have the same type as builtin functions
#****************************************************************************
# Builtin color schemes

Colors = TermColors  # just a shorthand

# Build a few color schemes
NoColor = ColorScheme(
    'NoColor',{
    'header' : Colors.NoColor,
    'normal' : Colors.NoColor  # color off (usu. Colors.Normal)
    }  )

LinuxColors = ColorScheme(
    'Linux',{
    'header' : Colors.LightRed,
    'normal' : Colors.Normal  # color off (usu. Colors.Normal)
    } )

LightBGColors = ColorScheme(
    'LightBG',{
    'header' : Colors.Red,
    'normal' : Colors.Normal  # color off (usu. Colors.Normal)
    }  )

# Build table of color schemes (needed by the parser)
InspectColors = ColorSchemeTable([NoColor,LinuxColors,LightBGColors],
                                 'Linux')

#****************************************************************************
# Auxiliary functions and objects

# See the messaging spec for the definition of all these fields.  This list
# effectively defines the order of display
info_fields = ['type_name', 'base_class', 'string_form', 'namespace',
               'length', 'file', 'definition', 'docstring', 'source',
               'init_definition', 'class_docstring', 'init_docstring',
               'call_def', 'call_docstring',
               # These won't be printed but will be used to determine how to
               # format the object
               'ismagic', 'isalias', 'isclass', 'argspec', 'found', 'name'
               ]

def object_info(**kw):
    """Make an object info dict with all fields present."""
    infodict = dict(izip_longest(info_fields, [None]))
    infodict.update(kw)
    return infodict

def get_encoding(obj):
    """Get encoding for python source file defining obj

    Returns None if obj is not defined in a sourcefile.
    """
    ofile = find_file(obj)
    # run contents of file through pager starting at line where the object
    # is defined, as long as the file isn't binary and is actually on the
    # filesystem.
    if ofile is None:
        return None
    elif ofile.endswith(('.so', '.dll', '.pyd')):
        return None
    elif not os.path.isfile(ofile):
        return None
    else:
        # Print only text files, not extension binaries.  Note that
        # getsourcelines returns lineno with 1-offset and page() uses
        # 0-offset, so we must adjust.
        with stdlib_io.open(ofile, 'rb') as buffer:   # Tweaked to use io.open for Python 2
            encoding, lines = openpy.detect_encoding(buffer.readline)
        return encoding

def getdoc(obj):
    """Stable wrapper around inspect.getdoc.

    This can't crash because of attribute problems.

    It also attempts to call a getdoc() method on the given object.  This
    allows objects which provide their docstrings via non-standard mechanisms
    (like Pyro proxies) to still be inspected by ipython's ? system."""
    # Allow objects to offer customized documentation via a getdoc method:
    try:
        ds = obj.getdoc()
    except Exception:
        pass
    else:
        # if we get extra info, we add it to the normal docstring.
        if isinstance(ds, string_types):
            return inspect.cleandoc(ds)

    try:
        docstr = inspect.getdoc(obj)
        encoding = get_encoding(obj)
        return py3compat.cast_unicode(docstr, encoding=encoding)
    except Exception:
        # Harden against an inspect failure, which can occur with
        # SWIG-wrapped extensions.
        raise
        return None

def getsource(obj, oname=''):
    """Wrapper around inspect.getsource.

    This can be modified by other projects to provide customized source
    extraction.

    Parameters
    ----------
    obj : object
        an object whose source code we will attempt to extract
    oname : str
        (optional) a name under which the object is known

    Returns
    -------
    src : unicode or None

    """

    if isinstance(obj, property):
        sources = []
        for attrname in ['fget', 'fset', 'fdel']:
            fn = getattr(obj, attrname)
            if fn is not None:
                encoding = get_encoding(fn)
                oname_prefix = ('%s.' % oname) if oname else ''
                sources.append(cast_unicode(
                    ''.join(('# ', oname_prefix, attrname)),
                    encoding=encoding))
                if inspect.isfunction(fn):
                    sources.append(dedent(getsource(fn)))
                else:
                    # Default str/repr only prints function name,
                    # pretty.pretty prints module name too.
                    sources.append(cast_unicode(
                        '%s%s = %s\n' % (
                            oname_prefix, attrname, pretty(fn)),
                        encoding=encoding))
        if sources:
            return '\n'.join(sources)
        else:
            return None

    else:
        # Get source for non-property objects.

        obj = _get_wrapped(obj)

        try:
            src = inspect.getsource(obj)
        except TypeError:
            # The object itself provided no meaningful source, try looking for
            # its class definition instead.
            if hasattr(obj, '__class__'):
                try:
                    src = inspect.getsource(obj.__class__)
                except TypeError:
                    return None

        encoding = get_encoding(obj)
        return cast_unicode(src, encoding=encoding)

def is_simple_callable(obj):
    """True if obj is a function ()"""
    return (inspect.isfunction(obj) or inspect.ismethod(obj) or \
            isinstance(obj, _builtin_func_type) or isinstance(obj, _builtin_meth_type))

def getargspec(obj):
    """Wrapper around :func:`inspect.getfullargspec` on Python 3, and
    :func:inspect.getargspec` on Python 2.

    In addition to functions and methods, this can also handle objects with a
    ``__call__`` attribute.
    """
    if safe_hasattr(obj, '__call__') and not is_simple_callable(obj):
        obj = obj.__call__

    return inspect.getfullargspec(obj) if PY3 else inspect.getargspec(obj)

def format_argspec(argspec):
    """Format argspect, convenience wrapper around inspect's.

    This takes a dict instead of ordered arguments and calls
    inspect.format_argspec with the arguments in the necessary order.
    """
    return inspect.formatargspec(argspec['args'], argspec['varargs'],
                                 argspec['varkw'], argspec['defaults'])

def call_tip(oinfo, format_call=True):
    """Extract call tip data from an oinfo dict.

    Parameters
    ----------
    oinfo : dict

    format_call : bool, optional
      If True, the call line is formatted and returned as a string.  If not, a
      tuple of (name, argspec) is returned.

    Returns
    -------
    call_info : None, str or (str, dict) tuple.
      When format_call is True, the whole call information is formattted as a
      single string.  Otherwise, the object's name and its argspec dict are
      returned.  If no call information is available, None is returned.

    docstring : str or None
      The most relevant docstring for calling purposes is returned, if
      available.  The priority is: call docstring for callable instances, then
      constructor docstring for classes, then main object's docstring otherwise
      (regular functions).
    """
    # Get call definition
    argspec = oinfo.get('argspec')
    if argspec is None:
        call_line = None
    else:
        # Callable objects will have 'self' as their first argument, prune
        # it out if it's there for clarity (since users do *not* pass an
        # extra first argument explicitly).
        try:
            has_self = argspec['args'][0] == 'self'
        except (KeyError, IndexError):
            pass
        else:
            if has_self:
                argspec['args'] = argspec['args'][1:]

        call_line = oinfo['name']+format_argspec(argspec)

    # Now get docstring.
    # The priority is: call docstring, constructor docstring, main one.
    doc = oinfo.get('call_docstring')
    if doc is None:
        doc = oinfo.get('init_docstring')
    if doc is None:
        doc = oinfo.get('docstring','')

    return call_line, doc

def _get_wrapped(obj):
    """Get the original object if wrapped in one or more @decorators"""
    while safe_hasattr(obj, '__wrapped__'):
        obj = obj.__wrapped__
    return obj

def find_file(obj):
    """Find the absolute path to the file where an object was defined.

    This is essentially a robust wrapper around `inspect.getabsfile`.

    Returns None if no file can be found.

    Parameters
    ----------
    obj : any Python object

    Returns
    -------
    fname : str
      The absolute path to the file where the object was defined.
    """
    obj = _get_wrapped(obj)

    fname = None
    try:
        fname = inspect.getabsfile(obj)
    except TypeError:
        # For an instance, the file that matters is where its class was
        # declared.
        if hasattr(obj, '__class__'):
            try:
                fname = inspect.getabsfile(obj.__class__)
            except TypeError:
                # Can happen for builtins
                pass
    except:
        pass
    return cast_unicode(fname)

def find_source_lines(obj):
    """Find the line number in a file where an object was defined.

    This is essentially a robust wrapper around `inspect.getsourcelines`.

    Returns None if no file can be found.

    Parameters
    ----------
    obj : any Python object

    Returns
    -------
    lineno : int
      The line number where the object definition starts.
    """
    obj = _get_wrapped(obj)

    try:
        try:
            lineno = inspect.getsourcelines(obj)[1]
        except TypeError:
            # For instances, try the class object like getsource() does
            if hasattr(obj, '__class__'):
                lineno = inspect.getsourcelines(obj.__class__)[1]
            else:
                lineno = None
    except:
        return None

    return lineno

class Inspector:
    def __init__(self, color_table=InspectColors,
                 code_color_table=PyColorize.ANSICodeColors,
                 scheme='NoColor',
                 str_detail_level=0):
        self.color_table = color_table
        self.parser = PyColorize.Parser(code_color_table,out='str')
        self.format = self.parser.format
        self.str_detail_level = str_detail_level
        self.set_active_scheme(scheme)

    def _getdef(self,obj,oname=''):
        """Return the call signature for any callable object.

        If any exception is generated, None is returned instead and the
        exception is suppressed."""
        try:
            hdef = oname + str(signature(obj))
            return cast_unicode(hdef)
        except:
            return None

    def __head(self,h):
        """Return a header string with proper colors."""
        return '%s%s%s' % (self.color_table.active_colors.header,h,
                           self.color_table.active_colors.normal)

    def set_active_scheme(self, scheme):
        self.color_table.set_active_scheme(scheme)
        self.parser.color_table.set_active_scheme(scheme)

    def noinfo(self, msg, oname):
        """Generic message when no information is found."""
        print('No %s found' % msg, end=' ')
        if oname:
            print('for %s' % oname)
        else:
            print()

    def pdef(self, obj, oname=''):
        """Print the call signature for any callable object.

        If the object is a class, print the constructor information."""

        if not callable(obj):
            print('Object is not callable.')
            return

        header = ''

        if inspect.isclass(obj):
            header = self.__head('Class constructor information:\n')
            obj = obj.__init__
        elif (not py3compat.PY3) and type(obj) is types.InstanceType:
            obj = obj.__call__

        output = self._getdef(obj,oname)
        if output is None:
            self.noinfo('definition header',oname)
        else:
            print(header,self.format(output), end=' ', file=io.stdout)

    # In Python 3, all classes are new-style, so they all have __init__.
    @skip_doctest_py3
    def pdoc(self,obj,oname='',formatter = None):
        """Print the docstring for any object.

        Optional:
        -formatter: a function to run the docstring through for specially
        formatted docstrings.

        Examples
        --------

        In [1]: class NoInit:
           ...:     pass

        In [2]: class NoDoc:
           ...:     def __init__(self):
           ...:         pass

        In [3]: %pdoc NoDoc
        No documentation found for NoDoc

        In [4]: %pdoc NoInit
        No documentation found for NoInit

        In [5]: obj = NoInit()

        In [6]: %pdoc obj
        No documentation found for obj

        In [5]: obj2 = NoDoc()

        In [6]: %pdoc obj2
        No documentation found for obj2
        """

        head = self.__head  # For convenience
        lines = []
        ds = getdoc(obj)
        if formatter:
            ds = formatter(ds)
        if ds:
            lines.append(head("Class docstring:"))
            lines.append(indent(ds))
        if inspect.isclass(obj) and hasattr(obj, '__init__'):
            init_ds = getdoc(obj.__init__)
            if init_ds is not None:
                lines.append(head("Init docstring:"))
                lines.append(indent(init_ds))
        elif hasattr(obj,'__call__'):
            call_ds = getdoc(obj.__call__)
            if call_ds:
                lines.append(head("Call docstring:"))
                lines.append(indent(call_ds))

        if not lines:
            self.noinfo('documentation',oname)
        else:
            page.page('\n'.join(lines))

    def psource(self, obj, oname=''):
        """Print the source code for an object."""

        # Flush the source cache because inspect can return out-of-date source
        linecache.checkcache()
        try:
            src = getsource(obj, oname=oname)
        except Exception:
            src = None

        if src is None:
            self.noinfo('source', oname)
        else:
            page.page(self.format(src))

    def pfile(self, obj, oname=''):
        """Show the whole file where an object was defined."""

        lineno = find_source_lines(obj)
        if lineno is None:
            self.noinfo('file', oname)
            return

        ofile = find_file(obj)
        # run contents of file through pager starting at line where the object
        # is defined, as long as the file isn't binary and is actually on the
        # filesystem.
        if ofile.endswith(('.so', '.dll', '.pyd')):
            print('File %r is binary, not printing.' % ofile)
        elif not os.path.isfile(ofile):
            print('File %r does not exist, not printing.' % ofile)
        else:
            # Print only text files, not extension binaries.  Note that
            # getsourcelines returns lineno with 1-offset and page() uses
            # 0-offset, so we must adjust.
            page.page(self.format(openpy.read_py_file(ofile, skip_encoding_cookie=False)), lineno - 1)

    def _format_fields(self, fields, title_width=0):
        """Formats a list of fields for display.

        Parameters
        ----------
        fields : list
          A list of 2-tuples: (field_title, field_content)
        title_width : int
          How many characters to pad titles to. Default to longest title.
        """
        out = []
        header = self.__head
        if title_width == 0:
            title_width = max(len(title) + 2 for title, _ in fields)
        for title, content in fields:
            if len(content.splitlines()) > 1:
                title = header(title + ":") + "\n"
            else:
                title = header((title+":").ljust(title_width))
            out.append(cast_unicode(title) + cast_unicode(content))
        return "\n".join(out)

    def _format_info(self, obj, oname='', formatter=None, info=None, detail_level=0):
        """Format an info dict as text"""
        info = self.info(obj, oname=oname, formatter=formatter,
                            info=info, detail_level=detail_level)
        displayfields = []
        def add_fields(fields):
            for title, key in fields:
                field = info[key]
                if field is not None:
                    displayfields.append((title, field.rstrip()))

        if info['isalias']:
            add_fields([('Repr', "string_form")])

        elif info['ismagic']:
            if detail_level > 0 and info['source'] is not None:
                add_fields([("Source", "source")])
            else:
                add_fields([("Docstring", "docstring")])

            add_fields([("File", "file"),
                       ])

        elif info['isclass'] or is_simple_callable(obj):
            # Functions, methods, classes
            add_fields([("Signature", "definition"),
                        ("Init signature", "init_definition"),
                       ])
            if detail_level > 0 and info['source'] is not None:
                add_fields([("Source", "source")])
            else:
                add_fields([("Docstring", "docstring"),
                            ("Init docstring", "init_docstring"),
                           ])

            add_fields([('File', 'file'),
                        ('Type', 'type_name'),
                       ])

        else:
            # General Python objects
            add_fields([("Type", "type_name")])

            # Base class for old-style instances
            if (not py3compat.PY3) and isinstance(obj, types.InstanceType) and info['base_class']:
                displayfields.append(("Base Class", info['base_class'].rstrip()))

            add_fields([("String form", "string_form")])

            # Namespace
            if info['namespace'] != 'Interactive':
                displayfields.append(("Namespace", info['namespace'].rstrip()))

            add_fields([("Length", "length"),
                        ("File", "file"),
                        ("Signature", "definition"),
                       ])

            # Source or docstring, depending on detail level and whether
            # source found.
            if detail_level > 0 and info['source'] is not None:
                displayfields.append(("Source",
                                      self.format(cast_unicode(info['source']))))
            elif info['docstring'] is not None:
                displayfields.append(("Docstring", info["docstring"]))

            add_fields([("Class docstring", "class_docstring"),
                        ("Init docstring", "init_docstring"),
                        ("Call signature", "call_def"),
                        ("Call docstring", "call_docstring")])

        if displayfields:
            return self._format_fields(displayfields)
        else:
            return u''

    def pinfo(self, obj, oname='', formatter=None, info=None, detail_level=0):
        """Show detailed information about an object.

        Optional arguments:

        - oname: name of the variable pointing to the object.

        - formatter: special formatter for docstrings (see pdoc)

        - info: a structure with some information fields which may have been
          precomputed already.

        - detail_level: if set to 1, more information is given.
        """
        text = self._format_info(obj, oname, formatter, info, detail_level)
        if text:
            page.page(text)

    def info(self, obj, oname='', formatter=None, info=None, detail_level=0):
        """Compute a dict with detailed information about an object.

        Optional arguments:

        - oname: name of the variable pointing to the object.

        - formatter: special formatter for docstrings (see pdoc)

        - info: a structure with some information fields which may have been
          precomputed already.

        - detail_level: if set to 1, more information is given.
        """

        obj_type = type(obj)

        if info is None:
            ismagic = 0
            isalias = 0
            ospace = ''
        else:
            ismagic = info.ismagic
            isalias = info.isalias
            ospace = info.namespace

        # Get docstring, special-casing aliases:
        if isalias:
            if not callable(obj):
                try:
                    ds = "Alias to the system command:\n  %s" % obj[1]
                except:
                    ds = "Alias: " + str(obj)
            else:
                ds = "Alias to " + str(obj)
                if obj.__doc__:
                    ds += "\nDocstring:\n" + obj.__doc__
        else:
            ds = getdoc(obj)
            if ds is None:
                ds = '<no docstring>'
        if formatter is not None:
            ds = formatter(ds)

        # store output in a dict, we initialize it here and fill it as we go
        out = dict(name=oname, found=True, isalias=isalias, ismagic=ismagic)

        string_max = 200 # max size of strings to show (snipped if longer)
        shalf = int((string_max -5)/2)

        if ismagic:
            obj_type_name = 'Magic function'
        elif isalias:
            obj_type_name = 'System alias'
        else:
            obj_type_name = obj_type.__name__
        out['type_name'] = obj_type_name

        try:
            bclass = obj.__class__
            out['base_class'] = str(bclass)
        except: pass

        # String form, but snip if too long in ? form (full in ??)
        if detail_level >= self.str_detail_level:
            try:
                ostr = str(obj)
                str_head = 'string_form'
                if not detail_level and len(ostr)>string_max:
                    ostr = ostr[:shalf] + ' <...> ' + ostr[-shalf:]
                    ostr = ("\n" + " " * len(str_head.expandtabs())).\
                            join(q.strip() for q in ostr.split("\n"))
                out[str_head] = ostr
            except:
                pass

        if ospace:
            out['namespace'] = ospace

        # Length (for strings and lists)
        try:
            out['length'] = str(len(obj))
        except: pass

        # Filename where object was defined
        binary_file = False
        fname = find_file(obj)
        if fname is None:
            # if anything goes wrong, we don't want to show source, so it's as
            # if the file was binary
            binary_file = True
        else:
            if fname.endswith(('.so', '.dll', '.pyd')):
                binary_file = True
            elif fname.endswith('<string>'):
                fname = 'Dynamically generated function. No source code available.'
            out['file'] = compress_user(fname)

        # Original source code for a callable, class or property.
        if detail_level:
            # Flush the source cache because inspect can return out-of-date
            # source
            linecache.checkcache()
            try:
                if isinstance(obj, property) or not binary_file:
                    src = getsource(obj, oname)
                    if src is not None:
                        src = src.rstrip()
                    out['source'] = src

            except Exception:
                pass

        # Add docstring only if no source is to be shown (avoid repetitions).
        if ds and out.get('source', None) is None:
            out['docstring'] = ds

        # Constructor docstring for classes
        if inspect.isclass(obj):
            out['isclass'] = True
            # reconstruct the function definition and print it:
            try:
                obj_init =  obj.__init__
            except AttributeError:
                init_def = init_ds = None
            else:
                init_def = self._getdef(obj_init,oname)
                init_ds  = getdoc(obj_init)
                # Skip Python's auto-generated docstrings
                if init_ds == _object_init_docstring:
                    init_ds = None

            if init_def or init_ds:
                if init_def:
                    out['init_definition'] = self.format(init_def)
                if init_ds:
                    out['init_docstring'] = init_ds

        # and class docstring for instances:
        else:
            # reconstruct the function definition and print it:
            defln = self._getdef(obj, oname)
            if defln:
                out['definition'] = self.format(defln)

            # First, check whether the instance docstring is identical to the
            # class one, and print it separately if they don't coincide.  In
            # most cases they will, but it's nice to print all the info for
            # objects which use instance-customized docstrings.
            if ds:
                try:
                    cls = getattr(obj,'__class__')
                except:
                    class_ds = None
                else:
                    class_ds = getdoc(cls)
                # Skip Python's auto-generated docstrings
                if class_ds in _builtin_type_docstrings:
                    class_ds = None
                if class_ds and ds != class_ds:
                    out['class_docstring'] = class_ds

            # Next, try to show constructor docstrings
            try:
                init_ds = getdoc(obj.__init__)
                # Skip Python's auto-generated docstrings
                if init_ds == _object_init_docstring:
                    init_ds = None
            except AttributeError:
                init_ds = None
            if init_ds:
                out['init_docstring'] = init_ds

            # Call form docstring for callable instances
            if safe_hasattr(obj, '__call__') and not is_simple_callable(obj):
                call_def = self._getdef(obj.__call__, oname)
                if call_def:
                    call_def = self.format(call_def)
                    # it may never be the case that call def and definition differ,
                    # but don't include the same signature twice
                    if call_def != out.get('definition'):
                        out['call_def'] = call_def
                call_ds = getdoc(obj.__call__)
                # Skip Python's auto-generated docstrings
                if call_ds == _func_call_docstring:
                    call_ds = None
                if call_ds:
                    out['call_docstring'] = call_ds

        # Compute the object's argspec as a callable.  The key is to decide
        # whether to pull it from the object itself, from its __init__ or
        # from its __call__ method.

        if inspect.isclass(obj):
            # Old-style classes need not have an __init__
            callable_obj = getattr(obj, "__init__", None)
        elif callable(obj):
            callable_obj = obj
        else:
            callable_obj = None

        if callable_obj:
            try:
                argspec = getargspec(callable_obj)
            except (TypeError, AttributeError):
                # For extensions/builtins we can't retrieve the argspec
                pass
            else:
                # named tuples' _asdict() method returns an OrderedDict, but we
                # we want a normal
                out['argspec'] = argspec_dict = dict(argspec._asdict())
                # We called this varkw before argspec became a named tuple.
                # With getfullargspec it's also called varkw.
                if 'varkw' not in argspec_dict:
                    argspec_dict['varkw'] = argspec_dict.pop('keywords')

        return object_info(**out)

    def psearch(self,pattern,ns_table,ns_search=[],
                ignore_case=False,show_all=False):
        """Search namespaces with wildcards for objects.

        Arguments:

        - pattern: string containing shell-like wildcards to use in namespace
          searches and optionally a type specification to narrow the search to
          objects of that type.

        - ns_table: dict of name->namespaces for search.

        Optional arguments:

          - ns_search: list of namespace names to include in search.

          - ignore_case(False): make the search case-insensitive.

          - show_all(False): show all names, including those starting with
            underscores.
        """
        #print 'ps pattern:<%r>' % pattern # dbg

        # defaults
        type_pattern = 'all'
        filter = ''

        cmds = pattern.split()
        len_cmds  =  len(cmds)
        if len_cmds == 1:
            # Only filter pattern given
            filter = cmds[0]
        elif len_cmds == 2:
            # Both filter and type specified
            filter,type_pattern = cmds
        else:
            raise ValueError('invalid argument string for psearch: <%s>' %
                             pattern)

        # filter search namespaces
        for name in ns_search:
            if name not in ns_table:
                raise ValueError('invalid namespace <%s>. Valid names: %s' %
                                 (name,ns_table.keys()))

        #print 'type_pattern:',type_pattern # dbg
        search_result, namespaces_seen = set(), set()
        for ns_name in ns_search:
            ns = ns_table[ns_name]
            # Normally, locals and globals are the same, so we just check one.
            if id(ns) in namespaces_seen:
                continue
            namespaces_seen.add(id(ns))
            tmp_res = list_namespace(ns, type_pattern, filter,
                                    ignore_case=ignore_case, show_all=show_all)
            search_result.update(tmp_res)

        page.page('\n'.join(sorted(search_result)))


text:
<?php

namespace OpisColibri\Manager\Controller;

use OpisColibri\Manager\View\ModuleList;
use OpisColibri\System\Helper as System;
use function Opis\Colibri\Functions\app;
use function Opis\Colibri\Functions\getURL;
use function Opis\Colibri\Functions\redirect;
use function Opis\Colibri\Functions\request;
use function Opis\Colibri\Functions\t;
use function Opis\Colibri\Functions\validator;
use function Opis\Colibri\Functions\view;

class Modules
{
    public function get()
    {
        return [
            'title' => 'Module manager',
            'content' => view('manager.module.manage', [
                'list' => new ModuleList(),
                'alerts' => System::alerts()->dismissable(),
            ]),
        ];
    }

    public function post()
    {
        $validator = validator();
        $validator->csrf()->validate('csrf', request()->post('csrf_token'));

        if($validator->hasErrors()){
            System::alerts()->fromValidator($validator);
            redirect(getURL('/admin/modules'));
        }

        if(app()->getCollector()->recollect()){
            System::alerts()->success(t('All items have been collected'));
        } else {
            System::alerts()->success(t('Collecting items has failed'));
        }

        redirect(getURL('/admin/modules'));
    }
}


text:
// Copyright 2015 Tosho Toshev
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

using ElmCommunicatorPortable.Commands;
using UnitsNet;

namespace ElmCommunicatorPortable.Responses.ObdIIResponses.ShowCurrentData
{
    public class IntakeAirTemperatureResponse : ResponseMessage
    {
        public override string ExpectedCommand
        {
            get
            {
                return "0F";
            }
        }

        public override IReceiveMessage Parse(string message)
        {
            this.Command = this.GetCommand(ref message);
            if(!this.IsValid())
            {
                return null;
            }

            this.Data = message.Substring(4);
            this.Temperature = Temperature.FromDegreesCelsius(this.HexToDec(this.Data) - 40);
            return this;
        }

        public Temperature Temperature { get; set; }
    }
}


text:
import * as Linter from 'tslint';
import { Program } from 'typescript';
export declare function getLinter(filePath: string, fileContent: string, program: Program): Linter;
export declare function createProgram(configFilePath: string, sourceDir: string): Program;
export declare function getFileNames(program: Program): string[];


text:
//
//  STPImages.h
//  Stripe
//
//  Created by Jack Flintermann on 6/30/16.
//  Copyright © 2016 Stripe, Inc. All rights reserved.
//

#import <Foundation/Foundation.h>
#import <UIKit/UIKit.h>
#import "STPCardBrand.h"

NS_ASSUME_NONNULL_BEGIN

/**
 *  This class lets you access card icons used by the Stripe SDK. All icons are 32 x 20 points.
 */
@interface STPImageLibrary : NSObject

/**
 *  An icon representing Apple Pay.
 */
+ (UIImage *)applePayCardImage;

/**
 *  An icon representing American Express.
 */
+ (UIImage *)amexCardImage;

/**
 *  An icon representing Diners Club.
 */
+ (UIImage *)dinersClubCardImage;

/**
 *  An icon representing Discover.
 */
+ (UIImage *)discoverCardImage;

/**
 *  An icon representing JCB.
 */
+ (UIImage *)jcbCardImage;

/**
 *  An icon representing MasterCard.
 */
+ (UIImage *)masterCardCardImage;

/**
 *  An icon representing Visa.
 */
+ (UIImage *)visaCardImage;

/**
 *  An icon to use when the type of the card is unknown.
 */
+ (UIImage *)unknownCardCardImage;

/**
 *  This returns the appropriate icon for the specified card brand.
 */
+ (UIImage *)brandImageForCardBrand:(STPCardBrand)brand;

/**
 *  This returns the appropriate icon for the specified card brand as a
 *  single color template that can be tinted
 */
+ (UIImage *)templatedBrandImageForCardBrand:(STPCardBrand)brand;

/**
 *  This returns a small icon indicating the CVC location for the given card brand.
 */
+ (UIImage *)cvcImageForCardBrand:(STPCardBrand)brand;

@end

NS_ASSUME_NONNULL_END


text:
class CreateProducts < ActiveRecord::Migration[5.0]
  def change
    create_table :products do |t|
      t.integer :sport_id
      t.integer :user_id
      t.string :product_name
      t.text :product_description

      t.timestamps
    end
  end
end


text:
import {SIGN_IN, LOG_OUT} from '../constants/ActionTypes';

export default function current_user(state = null, action) {

	switch (action.type) {

		case SIGN_IN:
			state = action.login;
			break;

		case LOG_OUT:
			state = null;
			break;

		default:
			break;
	}

	return state;
}


text:
<?xml version="1.0" ?>

<container xmlns="http://symfony.com/schema/dic/services"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd">

    <services>
        <defaults public="false" autowire="true" autoconfigure="true" />

        <service id="Wisembly\AmqpBundle\Command\ConsumerCommand">
            <tag name="console.command" command="amqp:consume" />
        </service>

        <service id="Wisembly\AmqpBundle\Command\PublishCommand">
            <tag name="console.command" command="amqp:publish" />
        </service>
    </services>
</container>


text:
namespace impala {

class QuerySchedule;
class ExecEnv;

/// The AdmissionController is used to throttle requests (e.g. queries, DML) based
/// on available cluster resources, which are configured in one or more resource pools. A
/// request will either be admitted for immediate execution, queued for later execution,
/// or rejected.  Resource pools can be configured to have maximum number of concurrent
/// queries, maximum memory, and a maximum queue size. Queries will be queued if there
/// are already too many queries executing or there isn't enough available memory. Once
/// the queue reaches the maximum queue size, incoming queries will be rejected. Requests
/// in the queue will time out after a configurable timeout.
///
/// Any impalad can act as a coordinator and thus also an admission controller, so some
/// cluster state must be shared between impalads in order to make admission decisions on
/// any node. Every impalad maintains some per-pool and per-host statistics related to
/// the requests it itself is servicing as the admission controller. Some of these
/// local admission statistics in addition to some backend-specific statistics (i.e.
/// the backend executor associated with the same impalad process) are disseminated
/// across the cluster via the statestore using the IMPALA_REQUEST_QUEUE_TOPIC topic.
/// For example, coordinators will end up sending statestore updates where the admission
/// statistics reflect the load and all participating backends will have statestore
/// updates reflecting load they're executing.
///
/// Every <impalad, pool> pair is sent as a topic update at the statestore heartbeat
/// interval when pool statistics change, and the topic updates from other impalads are
/// used to re-compute the aggregate per-pool stats. Because the pool statistics are only
/// updated on statestore heartbeats and all decisions are made with the cached state,
/// the aggregate pool statistics are only estimates. As a result, more requests may be
/// admitted or queued than the configured thresholds, which are really soft limits.
///
/// Memory resources:
/// A pool may be configured to allow a maximum amount of memory resources to be
/// 'reserved' by requests admitted to that pool. While Impala does not yet truly
/// 'reserve' the memory at admission (i.e. Impala does not yet guarantee the memory for
/// a request, it is still possible to overadmit such that multiple queries think they
/// have reserved the same memory), the admission controller uses several metrics to
/// estimate the available memory and admit only when it thinks the necessary memory is
/// available. Future work will enable real reservations, but this is a much larger
/// effort and will involve changes outside of the admission controller.
///
/// The memory required for admission for a request is specified as the query option
/// MEM_LIMIT (either explicitly or via a default value). This is a per-node value. If
/// there is no memory limit, the per-node estimate from planning is used instead. The
/// following two conditions must hold in order for the request to be admitted:
///  1) There must be enough memory resources available in this resource pool for the
///     request. The max memory resources configured for the resource pool specifies the
///     aggregate, cluster-wide memory that may be reserved by all executing queries in
///     this pool. Thus the aggregate memory to be reserved across all participating
///     backends for this request, *plus* that of already admitted requests must be less
///     than or equal to the max resources specified.
///  2) All participating backends must have enough memory available. Each impalad has a
///     per-process mem limit, and that is the max memory that can be reserved on that
///     backend.
///
/// In order to admit based on these conditions, the admission controller accounts for
/// the following on both a per-host and per-pool basis:
///  a) Mem Reserved: the amount of memory that has been reported as reserved by all
///     backends, which come from the statestore topic updates. The values that are sent
///     come from the pool mem trackers in UpdateMemTrackerStats(), which reflects the
///     memory reserved by fragments that have begun execution. For queries that have mem
///     limits, the limit is considered to be its reserved memory, otherwise the current
///     consumption is used (see MemTracker::GetPoolMemReserved()). The per-pool and
///     per-host aggregates are computed in UpdateClusterAggregates(). This state, once
///     all updates are fully distributed and aggregated, provides enough information to
///     make admission decisions by any impalad. However, this requires waiting for both
///     admitted requests to start all remote fragments and then for the updated state to
///     be distributed via the statestore.
///  b) Mem Admitted: the amount of memory required (i.e. the value used in admission,
///     either the mem limit or estimate) for the requests that this impalad's admission
///     controller has admitted. Both the per-pool and per-host accounting is updated
///     when requests are admitted and released (and note: not via the statestore, so
///     there is no latency, but this does not account for memory from requests admitted
///     by other impalads).
///
/// As described, both the 'reserved' and 'admitted' mem accounting mechanisms have
/// different advantages and disadvantages. The 'reserved' mem accounting works well in
/// the steady state, i.e. given enough time to distribute updates. The 'admitted'
/// mem accounting works perfectly when there is a single coordinator (and perhaps works
/// reasonably with just a few). The maximum of the reserved and admitted mem is used in
/// making admission decisions, which works well when either relatively few coordinators
/// are used or, if there is a wide distribution of requests across impalads, the rate of
/// submission is low enough that new state is able to be updated by the statestore.
///
/// Example:
/// Consider a 10-node cluster with 100gb/node and a resource pool 'q1' configured with
/// 500gb of memory. An incoming request with a 40gb MEM_LIMIT and schedule to execute on
/// all backends is received by AdmitQuery() on an otherwise quiet cluster.
/// CanAdmitRequest() checks the number of running queries and then calls
/// HasAvailableMemResources() to check for memory resources. It first checks whether
/// there is enough memory for the request using PoolStats::EffectiveMemReserved() (which
/// is the max of the pool's agg_mem_reserved_ and local_mem_admitted_, see #1 above),
/// and then checks for enough memory on each individual host via the max of the values
/// in the host_mem_reserved_ and host_mem_admitted_ maps (see #2 above). In this case,
/// ample resources are available so CanAdmitRequest() returns true.  PoolStats::Admit()
/// is called to update q1's PoolStats: it first updates agg_num_running_ and
/// local_mem_admitted_ which are able to be used immediately for incoming admission
/// requests, then it updates num_admitted_running in the struct sent to the statestore
/// (local_stats_). UpdateHostMemAdmitted() is called to update the per-host admitted mem
/// (stored in the map host_mem_admitted_) for all participating hosts. Then AdmitQuery()
/// returns to the Scheduler. If another identical admission request is received by the
/// same coordinator immediately, it will be rejected because q1's local_mem_admitted_ is
/// already 400gb. If that request were sent to another impalad at the same time, it
/// would have been admitted because not all updates have been disseminated yet. The next
/// statestore update will contain the updated value of num_admitted_running for q1 on
/// this backend. As remote fragments begin execution on remote impalads, their pool mem
/// trackers will reflect the updated amount of memory reserved (set in
/// local_stats_.backend_mem_reserved by UpdateMemTrackerStats()) and the next statestore
/// updates coming from those impalads will send the updated value.  As the statestore
/// updates are received (in the subscriber callback fn UpdatePoolStats()), the incoming
/// per-backend, per-pool mem_reserved values are aggregated to
/// PoolStats::agg_mem_reserved_ (pool aggregate over all hosts) and
/// backend_mem_reserved_ (per-host aggregates over all pools). Once this has happened,
/// any incoming admission request now has the updated state required to make correct
/// admission decisions.
///
/// Queuing Behavior:
/// Once the resources in a pool are consumed each coordinator receiving requests will
/// begin queuing. While each individual queue is FIFO, there is no total ordering on the
/// queued requests between admission controllers and no FIFO behavior is guaranteed for
/// requests submitted to different coordinators. When resources become available, there
/// is no synchronous coordination between nodes used to determine which get to dequeue and
/// admit requests. Instead, we use a simple heuristic to try to dequeue a number of
/// requests proportional to the number of requests that are waiting in each individual
/// admission controller to the total number of requests queued across all admission
/// controllers (i.e. impalads). This limits the amount of overadmission that may result
/// from a large amount of resources becoming available at the same time.
/// When there are requests queued in multiple pools on the same host, the admission
/// controller simply iterates over the pools in pool_stats_ and attempts to dequeue from
/// each. This is fine for the max_requests limit, but is unfair for memory-based
/// admission because the iteration order of pools effectively gives priority to the
/// queues at the beginning. Requests across queues may be competing for the same
/// resources on particular hosts, i.e. #2 in the description of memory-based admission
/// above. Note the pool's max_mem_resources (#1) is not contented.
/// TODO: Improve the dequeuing policy. IMPALA-2968.
///
/// TODO: Assumes all impalads have the same proc mem limit. Should send proc mem limit
///       via statestore (e.g. ideally in TBackendDescriptor) and check per-node
///       reservations against this value.
/// TODO: Remove less important debug logging after more cluster testing. Should have a
///       better idea of what is perhaps unnecessary.
class AdmissionController {
 public:
  AdmissionController(RequestPoolService* request_pool_service, MetricGroup* metrics,
      const TNetworkAddress& host_addr);
  ~AdmissionController();

  /// Submits the request for admission. Returns immediately if rejected, but
  /// otherwise blocks until the request is admitted. When this method returns,
  /// schedule->is_admitted() is true if and only if the request was admitted.
  /// For all calls to AdmitQuery(), ReleaseQuery() should also be called after
  /// the query completes to ensure that the pool statistics are updated.
  Status AdmitQuery(QuerySchedule* schedule);

  /// Updates the pool statistics when a query completes (either successfully,
  /// is cancelled or failed). This should be called for all requests that have
  /// been submitted via AdmitQuery(). (If the request was not admitted, this is
  /// a no-op.)
  /// This does not block.
  Status ReleaseQuery(QuerySchedule* schedule);

  /// Registers with the subscription manager.
  Status Init(StatestoreSubscriber* subscriber);

 private:
  class PoolStats;
  friend class PoolStats;

  /// Statestore topic name.
  static const std::string IMPALA_REQUEST_QUEUE_TOPIC;

  /// Used for user-to-pool resolution and looking up pool configurations. Not owned by
  /// the AdmissionController.
  RequestPoolService* request_pool_service_;

  /// Metrics subsystem access
  MetricGroup* metrics_group_;

  /// Thread dequeuing and admitting queries.
  boost::scoped_ptr<Thread> dequeue_thread_;

  // The local impalad's host/port id, used to construct topic keys.
  const std::string host_id_;

  /// Serializes/deserializes TPoolStats when sending and receiving topic updates.
  ThriftSerializer thrift_serializer_;

  /// Protects all access to all variables below.
  /// Coordinates access to the results of the promise QueueNode::is_admitted,
  /// but the lock is not required to wait on the promise.
  boost::mutex admission_ctrl_lock_;

  /// Maps from host id to memory reserved and memory admitted, both aggregates over all
  /// pools. See the class doc for a definition of reserved and admitted. Protected by
  /// admission_ctrl_lock_.
  typedef boost::unordered_map<std::string, int64_t> HostMemMap;
  HostMemMap host_mem_reserved_;
  HostMemMap host_mem_admitted_;

  /// Contains all per-pool statistics and metrics. Accessed via GetPoolStats().
  class PoolStats {
   public:
    struct PoolMetrics {
      /// Monotonically increasing counters (since process start) referring to this
      /// host's admission controller.
      IntCounter* total_admitted;
      IntCounter* total_rejected;
      IntCounter* total_queued;
      IntCounter* total_dequeued; // Does not include those in total_timed_out
      IntCounter* total_timed_out;
      IntCounter* total_released;
      IntCounter* time_in_queue_ms;

      /// The following mirror the current values in PoolStats.
      /// TODO: Avoid duplication: replace the int64_t fields on PoolStats with these.
      IntGauge* agg_num_running;
      IntGauge* agg_num_queued;
      IntGauge* agg_mem_reserved;
      IntGauge* local_mem_admitted;

      /// The following mirror the current values of local_stats_.
      /// TODO: As above, consolidate the metrics and local_stats_.
      IntGauge* local_num_admitted_running;
      IntGauge* local_num_queued;
      IntGauge* local_backend_mem_reserved;
      IntGauge* local_backend_mem_usage;

      /// Metrics exposing the pool settings.
      IntGauge* pool_max_mem_resources;
      IntGauge* pool_max_requests;
      IntGauge* pool_max_queued;
    };

    PoolStats(AdmissionController* parent, const std::string& name)
      : name_(name), parent_(parent), agg_num_running_(0), agg_num_queued_(0),
        agg_mem_reserved_(0), local_mem_admitted_(0) {
      InitMetrics();
    }

    int64_t agg_num_running() const { return agg_num_running_; }
    int64_t agg_num_queued() const { return agg_num_queued_; }
    int64_t EffectiveMemReserved() const {
      return std::max(agg_mem_reserved_, local_mem_admitted_);
    }

    /// ADMISSION LIFECYCLE METHODS
    /// The following methods update the pool stats when the request represented by
    /// schedule is admitted, released, queued, or dequeued.
    void Admit(const QuerySchedule& schedule);
    void Release(const QuerySchedule& schedule);
    void Queue(const QuerySchedule& schedule);
    void Dequeue(const QuerySchedule& schedule, bool timed_out);

    /// STATESTORE CALLBACK METHODS
    /// Updates the local_stats_.mem_reserved with the pool mem tracker. Called
    /// before sending local_stats().
    void UpdateMemTrackerStats();

    /// Called on a full topic update to clear all stats before processing the update.
    void ClearRemoteStats() { remote_stats_.clear(); }

    /// Called to update remote host TPoolStats with the new host_stats for the
    /// specified host. If host_stats is NULL the stats for the specified remote host
    /// are removed (i.e. topic deletion).
    void UpdateRemoteStats(const std::string& backend_id, TPoolStats* host_stats);

    /// Called after updating local_stats_ and remote_stats_ to update the aggregate
    /// values of agg_num_running_, agg_num_queued_, and agg_mem_reserved_. The in/out
    /// parameter host_mem_reserved is a map from host id to memory reserved used to
    /// aggregate the mem reserved values across all pools for each host. Used by
    /// UpdateClusterAggregates() to update host_mem_reserved_; it provides the host
    /// aggregates when called over all pools.
    void UpdateAggregates(HostMemMap* host_mem_reserved);

    const TPoolStats& local_stats() { return local_stats_; }

    /// Updates the metrics exposing the pool configuration to those in pool_cfg.
    void UpdateConfigMetrics(const TPoolConfig& pool_cfg);

    PoolMetrics* metrics() { return &metrics_; }
    std::string DebugString() const;
   private:
    const std::string name_;
    AdmissionController* parent_;

    /// Aggregate (across all hosts) number of running queries in this pool. Updated
    /// by Admit(), Release(), and after processing statestore updates by
    /// UpdateAggregates().
    int64_t agg_num_running_;

    /// Aggregate (across all hosts) number of queued requests. Updated by Queue(),
    /// Dequeue(), and after processing statestore updates by UpdateAggregates().
    int64_t agg_num_queued_;

    /// Aggregate memory reported as reserved for fragments executing in this pool by
    /// every host, i.e. the sum of all local_stats_.mem_reserved from all
    /// other hosts. Updated only by UpdateAggregates().
    int64_t agg_mem_reserved_;

    /// Memory in this pool (across all nodes) that is needed for requests that have been
    /// admitted by this local coordinator. Updated only on Admit() and Release(). Stored
    /// separately from the other 'local' stats in local_stats_ because it is not sent
    /// to the statestore (no 'aggregated' value is needed).
    int64_t local_mem_admitted_;

    /// This pool's TPoolStats for this host. Sent to the statestore (and thus not stored
    /// in remote_stats_ with the remote hosts). Most fields are updated eagerly and used
    /// for local admission decisions. local_stats_.backend_mem_reserved is the
    /// exception: it is not used in local admission decisions so it can be updated
    /// lazily before sending a statestore update.
    TPoolStats local_stats_;

    /// Map of host_ids to the latest TPoolStats. Entirely generated by incoming
    /// statestore updates; updated by UpdateRemoteStats() and used by UpdateAggregates().
    typedef boost::unordered_map<std::string, TPoolStats> RemoteStatsMap;
    RemoteStatsMap remote_stats_;

    /// Per-pool metrics, created by InitMetrics().
    PoolMetrics metrics_;
    void InitMetrics();
  };

  /// Map of pool names to pool stats. Accessed via GetPoolStats().
  /// Protected by admission_ctrl_lock_.
  typedef boost::unordered_map<std::string, PoolStats> PoolStatsMap;
  PoolStatsMap pool_stats_;

  /// The set of pools that have changed between topic updates that need stats to be sent
  /// to the statestore. The key is the pool name.
  typedef boost::unordered_set<std::string> PoolSet;
  PoolSet pools_for_updates_;

  /// Structure stored in a QueryQueue representing a request. This struct lives only
  /// during the call to AdmitQuery().
  struct QueueNode : public InternalQueue<QueueNode>::Node {
    QueueNode(const QuerySchedule& query_schedule) : schedule(query_schedule) { }

    /// Set when the request is admitted or rejected by the dequeuing thread. Used
    /// by AdmitQuery() to wait for admission or until the timeout is reached.
    /// The admission_ctrl_lock_ is not held while waiting on this promise, but
    /// the lock should be held when checking the result because the dequeuing
    /// thread holds it to Set().
    Promise<bool> is_admitted;

    /// The query schedule of the queued request. The schedule lives longer than the
    /// duration of the QueueNode, which only lives the duration of the call to
    /// AdmitQuery.
    const QuerySchedule& schedule;
  };

  /// Queue for the queries waiting to be admitted for execution. Once the
  /// maximum number of concurrently executing queries has been reached,
  /// incoming queries are queued and admitted FCFS.
  typedef InternalQueue<QueueNode> RequestQueue;

  /// Map of pool names to request queues.
  typedef boost::unordered_map<std::string, RequestQueue> RequestQueueMap;
  RequestQueueMap request_queue_map_;

  /// Map of pool names to the pool configs returned by request_pool_service_. Stored so
  /// that the dequeue thread does not need to access the configs via the request pool
  /// service again (which involves a JNI call and error checking).
  typedef boost::unordered_map<std::string, TPoolConfig> PoolConfigMap;
  PoolConfigMap pool_config_map_;

  /// Notifies the dequeuing thread that pool stats have changed and it may be
  /// possible to dequeue and admit queries.
  boost::condition_variable dequeue_cv_;

  /// If true, tear down the dequeuing thread. This only happens in unit tests.
  bool done_;

  /// Statestore subscriber callback that sends outgoing topic deltas (see
  /// AddPoolUpdates()) and processes incoming topic deltas, updating the PoolStats
  /// state.
  void UpdatePoolStats(
      const StatestoreSubscriber::TopicDeltaMap& incoming_topic_deltas,
      std::vector<TTopicDelta>* subscriber_topic_updates);

  /// Adds outgoing topic updates to subscriber_topic_updates for pools that have changed
  /// since the last call to AddPoolUpdates(). Called by UpdatePoolStats() before
  /// UpdateClusterAggregates(). Must hold admission_ctrl_lock_.
  void AddPoolUpdates(std::vector<TTopicDelta>* subscriber_topic_updates);

  /// Updates the remote stats with per-host topic_updates coming from the statestore.
  /// Called by UpdatePoolStats(). Must hold admission_ctrl_lock_.
  void HandleTopicUpdates(const std::vector<TTopicItem>& topic_updates);

  /// Removes remote stats identified by the topic_deletions coming from the statestore.
  /// Called by UpdatePoolStats(). Must hold admission_ctrl_lock_.
  void HandleTopicDeletions(const std::vector<std::string>& topic_deletions);

  /// Re-computes the per-pool aggregate stats and the per-host aggregates in
  /// host_mem_reserved_ using each pool's remote_stats_ and local_stats_.
  /// Called by UpdatePoolStats() after handling updates and deletions.
  /// Must hold admission_ctrl_lock_.
  void UpdateClusterAggregates();

  /// Dequeues and admits queued queries when notified by dequeue_cv_.
  void DequeueLoop();

  /// Returns true if schedule can be admitted to the pool with pool_cfg.
  /// admit_from_queue is true if attempting to admit from the queue. Otherwise, returns
  /// false and not_admitted_reason specifies why the request can not be admitted
  /// immediately. Caller owns not_admitted_reason.  Must hold admission_ctrl_lock_.
  bool CanAdmitRequest(const QuerySchedule& schedule, const TPoolConfig& pool_cfg,
      bool admit_from_queue, std::string* not_admitted_reason);

  /// Returns true if there is enough memory available to admit the query based on the
  /// schedule, the aggregate pool memory, and the per-host memory. If not, this returns
  /// false and returns the reason in mem_unavailable_reason. Caller owns
  /// mem_unavailable_reason. Must hold admission_ctrl_lock_.
  bool HasAvailableMemResources(const QuerySchedule& schedule,
      const TPoolConfig& pool_cfg, std::string* mem_unavailable_reason);

  /// Adds per_node_mem to host_mem_admitted_ for each host in schedule. Must hold
  /// admission_ctrl_lock_.
  void UpdateHostMemAdmitted(const QuerySchedule& schedule, int64_t per_node_mem);

  /// Returns an error status if this request must be rejected immediately, e.g. requires
  /// more memory than possible to reserve or the queue is already full.
  /// Must hold admission_ctrl_lock_.
  Status RejectImmediately(QuerySchedule* schedule, const TPoolConfig& pool_cfg);

  /// Gets or creates the PoolStats for pool_name. Must hold admission_ctrl_lock_.
  PoolStats* GetPoolStats(const std::string& pool_name);
};

}

#endif // SCHEDULING_ADMISSION_CONTROLLER_H