text:
# -*- coding: utf-8 -*-
"""
Objects used with ProofreadPage Extension.

The extension is supported by MW 1.21+.

This module includes objects:

* ProofreadPage(Page)
* FullHeader
* IndexPage(Page)

OCR support of page scans via:
- https://tools.wmflabs.org/phetools/hocr_cgi.py
- https://tools.wmflabs.org/phetools/ocr.php
inspired by https://en.wikisource.org/wiki/MediaWiki:Gadget-ocr.js

"""
#
# (C) Pywikibot team, 2015-2017
#
# Distributed under the terms of the MIT license.
#
from __future__ import absolute_import, unicode_literals

__version__ = '$Id$'

from functools import partial
import json
import re

try:
    from bs4 import BeautifulSoup, FeatureNotFound
except ImportError as e:
    BeautifulSoup = e
else:
    try:
        BeautifulSoup('', 'lxml')
    except FeatureNotFound:
        Soup = partial(BeautifulSoup, features='html.parser')
    else:
        Soup = partial(BeautifulSoup, features='lxml')

import pywikibot
from pywikibot.comms import http
from pywikibot.data.api import Request

_logger = 'proofreadpage'

class FullHeader(object):

    """Header of a ProofreadPage object."""

    p_header = re.compile(
        r'<pagequality level="(?P<ql>\d)" user="(?P<user>.*?)" />'
        r'(?P<has_div>
)?(?P.*)',
        re.DOTALL)

    TEMPLATE_V1 = ('<pagequality level="{0.ql}" user="{0.user}" />'
                   '
{0.header}\n\n\n')
    TEMPLATE_V2 = ('<pagequality level="{0.ql}" user="{0.user}" />'
                   '{0.header}')

    def __init__(self, text=None):
        """Constructor."""
        self._text = text or ''
        self._has_div = True

        m = self.p_header.search(self._text)
        if m:
            self.ql = int(m.group('ql'))
            self.user = m.group('user')
            self.header = m.group('header')
            if not m.group('has_div'):
                self._has_div = False
        else:
            self.ql = ProofreadPage.NOT_PROOFREAD
            self.user = ''
            self.header = ''

    def __str__(self):
        """Return a string representation."""
        if self._has_div:
            return FullHeader.TEMPLATE_V1.format(self)
        else:
            return FullHeader.TEMPLATE_V2.format(self)

class ProofreadPage(pywikibot.Page):

    """ProofreadPage page used in Mediawiki ProofreadPage extension."""

    WITHOUT_TEXT = 0
    NOT_PROOFREAD = 1
    PROBLEMATIC = 2
    PROOFREAD = 3
    VALIDATED = 4
    PROOFREAD_LEVELS = [WITHOUT_TEXT,
                        NOT_PROOFREAD,
                        PROBLEMATIC,
                        PROOFREAD,
                        VALIDATED,
                        ]

    _FMT = ('{0.open_tag}{0._full_header}{0.close_tag}'
            '{0._body}'
            '{0.open_tag}{0._footer}%s{0.close_tag}')

    open_tag = '<noinclude>'
    close_tag = '</noinclude>'
    p_open = re.compile(r'<noinclude>')
    p_close = re.compile(r'(
|\n\n\n)?</noinclude>')

    # phe-tools ocr utility
    HOCR_CMD = ('https://tools.wmflabs.org/phetools/hocr_cgi.py?'
                'cmd=hocr&book={book}&lang={lang}&user={user}')

    OCR_CMD = ('https://tools.wmflabs.org/phetools/ocr.php?'
               'cmd=ocr&url={url_image}&lang={lang}&user={user}')

    MULTI_PAGE_EXT = ['djvu', 'pdf']

    def __init__(self, source, title=''):
        """Instantiate a ProofreadPage object.

        @raise UnknownExtension: source Site has no ProofreadPage Extension.
        """
        if not isinstance(source, pywikibot.site.BaseSite):
            site = source.site
        else:
            site = source
        super(ProofreadPage, self).__init__(source, title)
        if self.namespace() != site.proofread_page_ns:
            raise ValueError('Page %s must belong to %s namespace'
                             % (self.title(), site.proofread_page_ns))
        # Ensure that constants are in line with Extension values.
        if list(self.site.proofread_levels.keys()) != self.PROOFREAD_LEVELS:
            raise ValueError('QLs do not match site values: %s != %s'
                             % (self.site.proofread_levels.keys(),
                                self.PROOFREAD_LEVELS))

        self._base, self._base_ext, self._num = self._parse_title()
        self._multi_page = self._base_ext in self.MULTI_PAGE_EXT

    @property
    def _fmt(self):
        if self._full_header._has_div:
            return self._FMT % '
'
        else:
            return self._FMT % ''

    def _parse_title(self):
        """Get ProofreadPage base title, base extension and page number.

        Base title is the part of title before the last '/', if any,
        or the whole title if no '/' is present.

        Extension is the extension of the base title.

        Page number is the part of title after the last '/', if any,
        or None if no '/' is present.

        E.g. for title 'Page:Popular Science Monthly Volume 1.djvu/12':
        - base = 'Popular Science Monthly Volume 1.djvu'
        - extenstion = 'djvu'
        - number = 12

        E.g. for title 'Page:Original Waltzing Matilda manuscript.jpg':
        - base = 'Original Waltzing Matilda manuscript.jpg'
        - extenstion = 'jpg'
        - number = None

        @return: (base, ext, num).
        @rtype: tuple
        """
        left, sep, right = self.title(withNamespace=False).rpartition('/')
        if sep:
            base = left
            num = int(right)
        else:
            base = right
            num = None

        left, sep, right = base.rpartition('.')
        if sep:
            ext = right
        else:
            ext = ''

        return (base, ext, num)

    @property
    def index(self):
        """Get the Index page which contains ProofreadPage.

        If there are many Index pages link to this ProofreadPage, and
        the ProofreadPage is titled Page:<index title>/<page number>,
        the Index page with the same title will be returned.
        Otherwise None is returned in the case of multiple linked Index pages.

        To force reload, delete index and call it again.

        @return: the Index page for this ProofreadPage
        @rtype: IndexPage or None
        """
        if not hasattr(self, '_index'):
            index_ns = self.site.proofread_index_ns
            what_links_here = [IndexPage(page) for page in
                               set(self.getReferences(namespaces=index_ns))]

            if not what_links_here:
                self._index = (None, [])
            elif len(what_links_here) == 1:
                self._index = (what_links_here.pop(), [])
            else:
                self._index = (None, what_links_here)
                # Try to infer names from page titles.
                if self._num is not None:
                    for page in what_links_here:
                        if page.title(withNamespace=False) == self._base:
                            what_links_here.remove(page)
                            self._index = (page, what_links_here)
                            break

        page, others = self._index
        if others:
            pywikibot.warning('%s linked to several Index pages.' % self)
            pywikibot.output('{0}{1!s}'.format(' ' * 9, [page] + others))

            if page:
                pywikibot.output('{0}Selected Index: {1}'.format(' ' * 9, page))
                pywikibot.output('{0}remaining: {1!s}'.format(' ' * 9, others))

        if not page:
            pywikibot.warning('Page %s is not linked to any Index page.'
                              % self)

        return page

    @index.setter
    def index(self, value):
        if not isinstance(value, IndexPage):
            raise TypeError('value %s must be a IndexPage object.'
                            % value)
        self._index = (value, None)

    @index.deleter
    def index(self):
        if hasattr(self, "_index"):
            del self._index

    @property
    def quality_level(self):
        """Return the quality level of this page when it is retrieved from API.

        This is only applicable if contentmodel equals 'proofread-page'.
        None is returned otherwise.

        This property is read-only and is applicable only when page is loaded.
        If quality level is overwritten during page processing, this property
        is no longer necessarily aligned with the new value.

        In this way, no text parsing is necessary to check quality level when
        fetching a page.
        # TODO: align this value with ProofreadPage.ql

        """
        if self.content_model == 'proofread-page' and hasattr(self, '_quality'):
            return self._quality
        return self.ql

    def decompose(fn):  # flake8: disable=N805
        """Decorator.

        Decompose text if needed and recompose text.
        """
        def wrapper(obj, *args, **kwargs):
            if not hasattr(obj, '_full_header'):
                obj._decompose_page()
            _res = fn(obj, *args, **kwargs)
            obj._compose_page()
            return _res
        return wrapper

    @property
    @decompose
    def ql(self):
        """Return page quality level."""
        return self._full_header.ql

    @ql.setter
    @decompose
    def ql(self, value):
        """Set page quality level."""
        if value not in self.site.proofread_levels:
            raise ValueError('Not valid QL value: %s (legal values: %s)'
                             % (value, self.site.proofread_levels))
        # TODO: add logic to validate ql value change, considering
        # site.proofread_levels.
        self._full_header.ql = value

    @property
    @decompose
    def user(self):
        """Return user in page header."""
        return self._full_header.user

    @user.setter
    @decompose
    def user(self, value):
        """Set user in page header."""
        self._full_header.user = value

    @property
    @decompose
    def status(self):
        """Return Proofread Page status."""
        try:
            return self.site.proofread_levels[self.ql]
        except KeyError:
            pywikibot.warning('Not valid status set for %s: quality level = %s'
                              % (self.title(asLink=True), self.ql))
            return None

    def without_text(self):
        """Set Page QL to "Without text"."""
        self.ql = self.WITHOUT_TEXT

    def problematic(self):
        """Set Page QL to "Problematic"."""
        self.ql = self.PROBLEMATIC

    def not_proofread(self):
        """Set Page QL to "Not Proofread"."""
        self.ql = self.NOT_PROOFREAD

    def proofread(self):
        """Set Page QL to "Proofread"."""
        # TODO: check should be made to be consistent with Proofread Extension
        self.ql = self.PROOFREAD

    def validate(self):
        """Set Page QL to "Validated"."""
        # TODO: check should be made to be consistent with Proofread Extension
        self.ql = self.VALIDATED

    @property
    @decompose
    def header(self):
        """Return editable part of Page header."""
        return self._full_header.header

    @header.setter
    @decompose
    def header(self, value):
        """Set editable part of Page header."""
        self._full_header.header = value

    @property
    @decompose
    def body(self):
        """Return Page body."""
        return self._body

    @body.setter
    @decompose
    def body(self, value):
        """Set Page body."""
        self._body = value

    @property
    @decompose
    def footer(self):
        """Return Page footer."""
        return self._footer

    @footer.setter
    @decompose
    def footer(self, value):
        """Set Page footer."""
        self._footer = value

    def _create_empty_page(self):
        """Create empty page."""
        self._full_header = FullHeader()
        self._body = ''
        self._footer = ''
        self.user = self.site.username()  # Fill user field in empty header.
        self._compose_page()

    @property
    def text(self):
        """Override text property.

        Preload text returned by EditFormPreloadText to preload non-existing
        pages.
        """
        # Text is already cached.
        if hasattr(self, '_text'):
            return self._text
        # If page does not exist, preload it.
        if self.exists():
            # If page exists, load it.
            super(ProofreadPage, self).text
        else:
            self._text = self.preloadText()
            self.user = self.site.username()  # Fill user field in empty header.
        return self._text

    @text.setter
    def text(self, value):
        """Update current text.

        Mainly for use within the class, called by other methods.
        Use self.header, self.body and self.footer to set page content,

        @param value: New value or None
        @param value: basestring

        @raise Error: the page is not formatted according to ProofreadPage
            extension.
        """
        self._text = value
        if self._text:
            self._decompose_page()
        else:
            self._create_empty_page()

    @text.deleter
    def text(self):
        """Delete current text."""
        if hasattr(self, '_text'):
            del self._text

    def _decompose_page(self):
        """Split Proofread Page text in header, body and footer.

        @raise Error: the page is not formatted according to ProofreadPage
            extension.
        """
        # Property force page text loading.
        if not (hasattr(self, '_text') or self.text):
            self._create_empty_page()
            return

        open_queue = list(self.p_open.finditer(self._text))
        close_queue = list(self.p_close.finditer(self._text))

        len_oq = len(open_queue)
        len_cq = len(close_queue)
        if (len_oq != len_cq) or (len_oq < 2 or len_cq < 2):
            raise pywikibot.Error('ProofreadPage %s: invalid format'
                                  % self.title(asLink=True))

        f_open, f_close = open_queue[0], close_queue[0]
        self._full_header = FullHeader(self._text[f_open.end():f_close.start()])

        l_open, l_close = open_queue[-1], close_queue[-1]
        self._footer = self._text[l_open.end():l_close.start()]

        self._body = self._text[f_close.end():l_open.start()]

    def _compose_page(self):
        """Compose Proofread Page text from header, body and footer."""
        self._text = self._fmt.format(self)
        return self._text

    def _page_to_json(self):
        """Convert page text to json format.

        This is the format accepted by action=edit specifying
        contentformat=application/json. This format is recommended to save the
        page, as it is not subject to possible errors done in composing the
        wikitext header and footer of the page or changes in the ProofreadPage
        extension format.
        """
        page_dict = {'header': self.header,
                     'body': self.body,
                     'footer': self.footer,
                     'level': {'level': self.ql, 'user': self.user},
                     }
        # Ensure_ascii=False returns a unicode.
        return json.dumps(page_dict, ensure_ascii=False)

    def save(self, *args, **kwargs):  # See Page.save().
        """Save page content after recomposing the page."""
        summary = kwargs.pop('summary', '')
        summary = self.pre_summary + summary
        # Save using contentformat='application/json'.
        kwargs['contentformat'] = 'application/json'
        kwargs['contentmodel'] = 'proofread-page'
        text = self._page_to_json()
        super(ProofreadPage, self).save(*args, text=text, summary=summary,
                                        **kwargs)

    @property
    def pre_summary(self):
        """Return trailing part of edit summary.

        The edit summary shall be appended to pre_summary to highlight
        Status in the edit summary on wiki.
        """
        return '/* {0.status} */ '.format(self)

    @property
    def url_image(self):
        """Get the file url of the scan of ProofreadPage.

        @return: file url of the scan ProofreadPage or None.
        @rtype: str/unicode

        @raises:
        - Exception in case of http errors.
        """
        # wrong link fail with various possible Exceptions.
        if not hasattr(self, '_url_image'):

            if self.exists():
                url = self.full_url()
            else:
                path = 'w/index.php?title={0}&action=edit&redlink=1'
                url = self.site.base_url(path.format(self.title(asUrl=True)))

            try:
                response = http.fetch(url, charset='utf-8')
            except Exception:
                pywikibot.error('Error fetching HTML for %s.' % self)
                raise

            soup = Soup(response.content)

            try:
                # None if nothing is found by .find()
                self._url_image = soup.find(class_='prp-page-image')
                self._url_image = self._url_image.find('img')
                # if None raises TypeError.
                self._url_image = self._url_image['src']
            except TypeError:
                raise ValueError('No prp-page-image src found for %s.' % self)
            else:
                self._url_image = 'https:' + self._url_image

        return self._url_image

    def _ocr_callback(self, cmd_uri, parser_func=None):
        """OCR callback function.

        @return: tuple (error, text [error description in case of error]).
        """
        def id(x):
            return x

        if not cmd_uri:
            raise ValueError('Parameter cmd_uri is mandatory.')

        if parser_func is None:
            parser_func = id

        if not callable(parser_func):
            raise TypeError('Keyword parser_func must be callable.')

        # wrong link fail with Exceptions
        try:
            response = http.fetch(cmd_uri, charset='utf-8')
        except Exception as e:
            pywikibot.error('Querying %s: %s' % (cmd_uri, e))
            return (True, e)

        data = json.loads(response.content)

        assert 'error' in data, 'Error from phe-tools: %s' % data
        assert data['error'] in [0, 1], 'Error from phe-tools: %s' % data

        error = bool(data['error'])
        if error:
            pywikibot.error('Querying %s: %s' % (cmd_uri, data['text']))
            return (error, data['text'])
        else:
            return (error, parser_func(data['text']))

    def _do_hocr(self):
        """Do hocr using //tools.wmflabs.org/phetools/hocr_cgi.py?cmd=hocr."""
        def parse_hocr_text(txt):
            """Parse hocr text."""
            soup = Soup(txt)

            res = []
            for ocr_page in soup.find_all(class_='ocr_page'):
                for area in soup.find_all(class_='ocr_carea'):
                    for par in area.find_all(class_='ocr_par'):
                        for line in par.find_all(class_='ocr_line'):
                            res.append(line.get_text())
                        res.append('\n')
            return ''.join(res)

        params = {'book': self.title(asUrl=True, withNamespace=False),
                  'lang': self.site.lang,
                  'user': self.site.user(),
                  }

        cmd_uri = self.HOCR_CMD.format(**params)

        return self._ocr_callback(cmd_uri, parser_func=parse_hocr_text)

    def _do_ocr(self):
        """Do ocr using //tools.wmflabs.org/phetools/ocr.pmp?cmd=ocr."""
        try:
            url_image = self.url_image
        except ValueError:
            error_text = 'No prp-page-image src found for %s.' % self
            pywikibot.error(error_text)
            return (True, error_text)

        params = {'url_image': url_image,
                  'lang': self.site.lang,
                  'user': self.site.user(),
                  }

        cmd_uri = self.OCR_CMD.format(**params)

        return self._ocr_callback(cmd_uri)

    def ocr(self):
        """Do OCR of Proofreadpage scan.

        The text returned by this function shalle be assign to self.body,
        otherwise the ProofreadPage format will not be maintained.

        It is the user's responsibility to reset quality level accordingly.
        """
        if self._multi_page:
            error, text = self._do_hocr()
            if not error:
                return text

        error, text = self._do_ocr()
        if not error:
            return text
        else:
            raise ValueError('Not possible to perform HOCR/OCR on %s.' % self)

class PurgeRequest(Request):

    """Subclass of Request which skips the check on write rights.

    Workaround for T128994.
    # TODO: remove once bug is fixed.
    """

    def __init__(self, **kwargs):
        """Monkeypatch action in Request constructor."""
        action = kwargs['parameters']['action']
        kwargs['parameters']['action'] = 'dummy'
        super(PurgeRequest, self).__init__(**kwargs)
        self.action = action
        self.update({'action': action})

class IndexPage(pywikibot.Page):

    """Index Page page used in Mediawiki ProofreadPage extension."""

    def __init__(self, source, title=''):
        """Instantiate a IndexPage object.

        In this class:
        page number is the number in the page title in the Page namespace, if
            the wikisource site adopts this convention (e.g. page_number is 12
            for Page:Popular Science Monthly Volume 1.djvu/12) or the sequential
            number of the pages linked from the index section in the Index page
            if the index is built via transclusion of a list of pages (e.g. like
            on de wikisource).
        page label is the label associated with a page in the Index page.

        This class provides methods to get pages contained in Index page,
        and relative page numbers and labels by means of several helper
        functions.

        It also provides a generator to pages contained in Index page, with
        possibility to define range, filter by quality levels and page
        existence.

        @raise UnknownExtension: source Site has no ProofreadPage Extension.
        @raise ImportError: bs4 is not installed.
        """
        # Check if BeautifulSoup is imported.
        if isinstance(BeautifulSoup, ImportError):
            raise BeautifulSoup

        if not isinstance(source, pywikibot.site.BaseSite):
            site = source.site
        else:
            site = source
        super(IndexPage, self).__init__(source, title)
        if self.namespace() != site.proofread_index_ns:
            raise ValueError('Page %s must belong to %s namespace'
                             % (self.title(), site.proofread_index_ns))

        self._all_page_links = set(
            self.site.pagelinks(self, namespaces=site.proofread_page_ns))

        self._cached = False

    def check_if_cached(fn):  # flake8: disable=N805
        """Decorator to check if data are cached and cache them if needed."""
        def wrapper(self, *args, **kwargs):
            if self._cached is False:
                self._get_page_mappings()
            return fn(self, *args, **kwargs)
        return wrapper

    def _parse_redlink(self, href):
        """Parse page title when link in Index is a redlink."""
        p_href = re.compile(r'/w/index\.php\?title=(.+?)&action=edit&redlink=1')
        title = p_href.search(href)
        if title:
            return title.group(1)
        else:
            return None

    def purge(self):
        """Overwrite purge method.

        Workaround for T128994.
        # TODO: remove once bug is fixed.

        Instead of a proper purge action, use PurgeRequest, which
        skips the check on write rights.
        """
        params = {'action': 'purge', 'titles': [self.title()]}
        request = PurgeRequest(site=self.site, parameters=params)
        rawdata = request.submit()
        error_message = 'Purge action failed for %s' % self
        assert 'purge' in rawdata, error_message
        assert 'purged' in rawdata['purge'][0], error_message

    def _get_page_mappings(self):
        """Associate label and number for each page linked to the index."""
        # Clean cache, if any.
        self._page_from_numbers = {}
        self._numbers_from_page = {}
        self._page_numbers_from_label = {}
        self._pages_from_label = {}
        self._labels_from_page_number = {}
        self._labels_from_page = {}
        if hasattr(self, '_parsed_text'):
            del self._parsed_text

        self._parsed_text = self._get_parsed_page()
        self._soup = Soup(self._parsed_text)
        # Do not search for "new" here, to avoid to skip purging if links
        # to non-existing pages are present.
        attrs = {'class': re.compile('prp-pagequality')}

        # Search for attribute "prp-pagequality" in tags:
        # Existing pages:
        # m
        #    class="quality1 prp-pagequality-1"
        #
        # Non-existing pages:
        # m
        #

        # Try to purge or raise ValueError.
        if not self._soup.find_all('a', attrs=attrs):
            self.purge()
            del self._parsed_text
            self._parsed_text = self._get_parsed_page()
            self._soup = Soup(self._parsed_text)
            if not self._soup.find_all('a', attrs=attrs):
                raise ValueError(
                    'Missing class="qualityN prp-pagequality-N" or '
                    'class="new" in: %s.'
                    % self)

        # Search for attribute "prp-pagequality" or "new" in tags:
        attrs = {'class': re.compile('prp-pagequality|new')}
        page_cnt = 0
        for a_tag in self._soup.find_all('a', attrs=attrs):
            label = a_tag.text.lstrip('0')  # Label is not converted to int.
            class_ = a_tag.get('class')
            href = a_tag.get('href')

            if 'new' in class_:
                title = self._parse_redlink(href)  # non-existing page
                if title is None:  # title not conforming to required format
                    continue
            else:
                title = a_tag.get('title')   # existing page
            try:
                page = ProofreadPage(self.site, title)
                page.index = self  # set index property for page
                page_cnt += 1
            except ValueError:
                # title is not in site.proofread_page_ns; do not consider it
                continue

            if page not in self._all_page_links:
                raise pywikibot.Error('Page %s not recognised.' % page)

            # In order to avoid to fetch other Page:title links outside
            # the Pages section of the Index page; these should hopefully be
            # the first ones, so if they start repeating, we are done.
            if page in self._labels_from_page:
                break

            # Sanity check if WS site use page convention name/number.
            if page._num is not None:
                assert page_cnt == int(page._num), (
                    'Page number %s not recognised as page %s.'
                    % (page_cnt, title))

            # Mapping: numbers <-> pages.
            self._page_from_numbers[page_cnt] = page
            self._numbers_from_page[page] = page_cnt
            # Mapping: numbers/pages as keys, labels as values.
            self._labels_from_page_number[page_cnt] = label
            self._labels_from_page[page] = label
            # Reverse mapping: labels as keys, numbers/pages as values.
            self._page_numbers_from_label.setdefault(label, set()).add(page_cnt)
            self._pages_from_label.setdefault(label, set()).add(page)

        # Sanity check: all links to Page: ns must have been considered.
        assert set(self._labels_from_page) == set(self._all_page_links)

        # Info cached.
        self._cached = True

    @property
    @check_if_cached
    def num_pages(self):
        """Return total number of pages in Index.

        @return: total number of pages in Index
        @rtype: int
        """
        return len(self._page_from_numbers)

    def page_gen(self, start=1, end=None, filter_ql=None,
                 only_existing=False, content=True):
        """Return a page generator which yields pages contained in Index page.

        Range is [start ... end], extremes included.

        @param start: first page, defaults to 1
        @type start: int
        @param end: num_pages if end is None
        @type end: int
        @param filter_ql: filters quality levels
                          if None: all but 'Without Text'.
        @type filter_ql: list of ints (corresponding to ql constants
                         defined in ProofreadPage).
        @param only_existing: yields only existing pages.
        @type only_existing: bool
        @param content: preload content.
        @type content: bool
        """
        if end is None:
            end = self.num_pages

        if not ((1 <= start <= self.num_pages) and
                (1 <= end <= self.num_pages) and
                (start <= end)):
            raise ValueError('start=%s, end=%s are not in valid range (%s, %s)'
                             % (start, end, 1, self.num_pages))

        # All but 'Without Text'
        if filter_ql is None:
            filter_ql = list(self.site.proofread_levels.keys())
            filter_ql.remove(ProofreadPage.WITHOUT_TEXT)

        gen = (self.get_page(i) for i in range(start, end + 1))
        if content:
            gen = self.site.preloadpages(gen)
        # Decorate and sort by page number because preloadpages does not
        # guarantee order.
        # TODO: remove if preloadpages will guarantee order.
        gen = ((p, self.get_number(p)) for p in gen)
        gen = (p[0] for p in sorted(gen, key=lambda x: x[1]))
        # Filter by QL.
        gen = (p for p in gen if p.ql in filter_ql)
        # Yield only existing.
        if only_existing:
            gen = (p for p in gen if p.exists())

        return gen

    @check_if_cached
    def get_label_from_page(self, page):
        """Return 'page label' for page.

        There is a 1-to-1 correspondence (each page has a label).

        @param page: Page instance
        @return: page label
        @rtype: unicode string
        """
        try:
            return self._labels_from_page[page]
        except KeyError:
            raise KeyError('Invalid Page: %s.' % page)

    @check_if_cached
    def get_label_from_page_number(self, page_number):
        """Return page label from page number.

        There is a 1-to-1 correspondence (each page has a label).

        @param page_number: int
        @return: page label
        @rtype: unicode string
        """
        try:
            return self._labels_from_page_number[page_number]
        except KeyError:
            raise KeyError('Page number ".../%s" not in range.'
                           % page_number)

    def _get_from_label(self, mapping_dict, label):
        """Helper function to get info from label."""
        # Convert label to string if an integer is passed.
        if isinstance(label, int):
            label = str(label)

        try:
            return mapping_dict[label]
        except KeyError:
            raise KeyError('No page has label: "%s".' % label)

    @check_if_cached
    def get_page_number_from_label(self, label='1'):
        """Return page number from page label.

        There is a 1-to-many correspondence (a label can be the same for
        several pages).

        @return: set containing page numbers corresponding to page label.
        """
        return self._get_from_label(self._page_numbers_from_label, label)

    @check_if_cached
    def get_page_from_label(self, label='1'):
        """Return page number from page label.

        There is a 1-to-many correspondence (a label can be the same for
        several pages).

        @return: set containing pages corresponding to page label.
        """
        return self._get_from_label(self._pages_from_label, label)

    @check_if_cached
    def get_page(self, page_number):
        """Return a page object from page number."""
        try:
            return self._page_from_numbers[page_number]
        except KeyError:
            raise KeyError('Invalid page number: %s.' % page_number)

    @check_if_cached
    def pages(self):
        """Return the list of pages in Index, sorted by page number.

        @return: list of pages
        @rtype: list
        """
        return [self._page_from_numbers[i] for i in range(1, self.num_pages + 1)]

    @check_if_cached
    def get_number(self, page):
        """Return a page number from page object."""
        try:
            return self._numbers_from_page[page]
        except KeyError:
            raise KeyError('Invalid page: %s.' % page)


text:
package com.gs.fw.finder.attribute;

import com.gs.fw.finder.Attribute;
import com.gs.fw.finder.AggregateAttribute;

public interface NumericAttribute<Owner> extends Attribute<Owner>
{
    NumericAttribute<Owner> plus(ByteAttribute addend);
    NumericAttribute<Owner> plus(ShortAttribute addend);
    NumericAttribute<Owner> plus(IntegerAttribute addend);
    NumericAttribute<Owner> plus(LongAttribute addend);
    NumericAttribute<Owner> plus(FloatAttribute addend);
    NumericAttribute<Owner> plus(DoubleAttribute addend);

    NumericAttribute<Owner> minus(ByteAttribute subtrahend);
    NumericAttribute<Owner> minus(ShortAttribute subtrahend);
    NumericAttribute<Owner> minus(IntegerAttribute subtrahend);
    NumericAttribute<Owner> minus(LongAttribute subtrahend);
    NumericAttribute<Owner> minus(FloatAttribute subtrahend);
    NumericAttribute<Owner> minus(DoubleAttribute subtrahend);

    NumericAttribute<Owner> times(ByteAttribute multiplicand);
    NumericAttribute<Owner> times(ShortAttribute multiplicand);
    NumericAttribute<Owner> times(IntegerAttribute multiplicand);
    NumericAttribute<Owner> times(LongAttribute multiplicand);
    NumericAttribute<Owner> times(FloatAttribute multiplicand);
    NumericAttribute<Owner> times(DoubleAttribute multiplicand);

    NumericAttribute<Owner> dividedBy(ByteAttribute divisor);
    NumericAttribute<Owner> dividedBy(ShortAttribute divisor);
    NumericAttribute<Owner> dividedBy(IntegerAttribute divisor);
    NumericAttribute<Owner> dividedBy(LongAttribute divisor);
    NumericAttribute<Owner> dividedBy(FloatAttribute divisor);
    NumericAttribute<Owner> dividedBy(DoubleAttribute divisor);

    NumericAttribute<Owner> absoluteValue();

    AggregateAttribute<Owner> sum();

    AggregateAttribute<Owner> avg();

    AggregateAttribute<Owner> standardDeviationSample();
}

redpajama_set_name: RedPajamaGithub


text:
package com.jlfex.hermes.common.support.spring;
import java.util.Locale;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.springframework.context.ApplicationContext;
import org.springframework.web.context.support.WebApplicationContextUtils;
import org.springframework.web.servlet.i18n.SessionLocaleResolver;
import org.springframework.web.util.WebUtils;
import com.jlfex.hermes.common.App;
import com.jlfex.hermes.common.Logger;
import com.jlfex.hermes.common.web.WebApp;

/**
 * 采用 Spring 容器的网络应用
 */
public class SpringWebApp extends WebApp {

	/** 容器 */
	private static ApplicationContext applicationContext;

	/**
	 * 构造函数
	 *
	 * @param request
	 * @param response
	 */
	public SpringWebApp(HttpServletRequest request, HttpServletResponse response) {
		super(request, response);
	}

	/*
	 * (non-Javadoc)
	 *
	 * @see com.jlfex.hermes.common.web.WebApp#getLocale()
	 */
	@Override
	public Locale getLocale() {
		Locale locale = Locale.class.cast(WebUtils.getSessionAttribute(request,
				SessionLocaleResolver.LOCALE_SESSION_ATTRIBUTE_NAME));
		if (locale == null) {
			locale = request.getLocale();
		}
		return locale;
	}

	/*
	 * (non-Javadoc)
	 *
	 * @see com.jlfex.hermes.common.web.WebApp#getMessage(java.lang.String, java.lang.Object[])
	 */
	@Override
	public String getMessage(String key, Object... args) {
		try {
			return String.format(applicationContext.getMessage(key, null, getLocale()), args);
		} catch (Exception e) {
			Logger.warn(e.getMessage());
			return key;
		}
	}

	/**
	 * 读取实例
	 *
	 * @param name
	 * @return
	 */
	public static Object getBean(String name) {
		return applicationContext.getBean(name);
	}

	/**
	 * 读取实例
	 *
	 * @param name
	 * @param args
	 * @return
	 */
	public static Object getBean(String name, Object... args) {
		return applicationContext.getBean(name, args);
	}

	/**
	 * 读取实例
	 *
	 * @param name
	 * @param requiredType
	 * @return
	 */
	public static <T> T getBean(String name, Class<T> requiredType) {
		return applicationContext.getBean(name, requiredType);
	}

	/**
	 * 读取实例
	 *
	 * @param requiredType
	 * @return
	 */
	public static <T> T getBean(Class<T> requiredType) {
		return applicationContext.getBean(requiredType);
	}

	/**
	 * 初始化
	 *
	 * @param context
	 */
	public static void initialize(ServletContext context) {
		WebApp.initialize(context);
		applicationContext = WebApplicationContextUtils.getWebApplicationContext(context);
		Logger.info("initialize spring application context: %s", applicationContext);
	}

	/**
	 * 创建
	 *
	 * @param request
	 * @param response
	 * @return
	 */
	public static App create(HttpServletRequest request, HttpServletResponse response) {
		set(new SpringWebApp(request, response));
		return current();
	}
}


text:
<?php
/*
This program shows how to assign a group or groups to
asset factories in all sites.
Also see 08_associate_af_with_group.php
*/
require_once( 'auth_chanw.php' );

use cascade_ws_AOHS      as aohs;
use cascade_ws_constants as c;
use cascade_ws_asset     as a;
use cascade_ws_property  as p;
use cascade_ws_utility   as u;
use cascade_ws_exception as e;

try
{
    /*
    // A map mapping site names to group names. It assumes that a site is mapped onto
    // a single group. Possible to have a site name mapped onto an array of group names.

    $site_group_map = array(
        "medicine" => "medicine",
    );
    */

    /*
    // An array for names to be processed.
    $site_to_process = array(
        "medicine"
    );
    */

    /*
    // Or an array for sites to be skipped.
    $site_to_skip = array(
        "medicine"
    );
    */

    $sites = $cascade->getSites();

    foreach( $sites as $site )
    {
        // in an array
        if( in_array( $site->getPathPath(), $site_to_process ) )
        // or not in an array
        // if( !in_array( $site->getPathPath(), $site_to_skip ) )
        {
            // process the site

            // first, get the group; replace $site->getPathPath() with a group name
            // the group name can be retrieved from $site_group_map
            $group = $cascade->getAsset( a\Group::TYPE, $site->getPathPath() );

            // second, create the asset tree and parameters
            $at = $cascade->getSite(
                $site->getPathPath() )->getRootAssetFactoryContainerAssetTree();

            $params = array();
            // if there are more than one group, then pass in an array instead
            $params[ "group" ] = $group; // a Group object

            // traverse the tree in every site
            $at->traverse(
                array( a\AssetFactory::TYPE =>
                    array( "assetTreeAssignGroupToAssetFactory" ) ),
                $params
            );
        }
    }
}
catch( \Exception $e )
{
    echo S_PRE . $e . E_PRE;
}
catch( \Error $er )
{
    echo S_PRE . $er . E_PRE;
}

function assetTreeAssignGroupToAssetFactory(
    aohs\AssetOperationHandlerService $service,
    p\Child $child, $params=NULL, &$results=NULL )
{
    // get the child type
    $type = $child->getType();

    // get out if not asset factory
    if( $type != a\AssetFactory::TYPE )
        return;

    // the group must be supplied
    if( !is_array( $params ) || !isset( $params[ 'group' ] ) )
        throw new e\NullAssetException(
            S_SPAN . "The group must be supplied" . E_SPAN );

    // retrieve the group; this can in fact be an array
    // if so, loop through each group
    $group = $params[ 'group' ];

    // associate group with asset factory
    //$child->getAsset( $service )->addGroup( $group )->edit();

    // for the moment, just output some info
    echo $child->getAsset( $service )->getName(), BR;
    echo $group->getName(), BR;
}
?>


text:
/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.facebook.presto.benchmark;

import com.facebook.presto.operator.OperatorFactory;
import com.facebook.presto.util.LocalQueryRunner;
import com.google.common.collect.ImmutableList;

import java.util.List;
import java.util.concurrent.ExecutorService;

import static com.facebook.presto.benchmark.BenchmarkQueryRunner.createLocalQueryRunner;
import static io.airlift.concurrent.Threads.daemonThreadsNamed;
import static java.util.concurrent.Executors.newCachedThreadPool;

public class RawStreamingBenchmark
        extends AbstractSimpleOperatorBenchmark
{
    public RawStreamingBenchmark(LocalQueryRunner localQueryRunner)
    {
        super(localQueryRunner, "raw_stream", 10, 100);
    }

    @Override
    protected List<? extends OperatorFactory> createOperatorFactories()
    {
        OperatorFactory tableScanOperator = createTableScanOperator(0, "orders", "totalprice");

        return ImmutableList.of(tableScanOperator);
    }

    public static void main(String[] args)
    {
        ExecutorService executor = newCachedThreadPool(daemonThreadsNamed("test"));
        new RawStreamingBenchmark(createLocalQueryRunner(executor)).runBenchmark(
                new SimpleLineBenchmarkResultWriter(System.out)
        );
    }
}


text:
package com.fsryan.forsuredb.annotationprocessor.generator;

import com.fsryan.forsuredb.annotationprocessor.util.PropertyRetriever;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;

import javax.annotation.processing.ProcessingEnvironment;
import javax.tools.FileObject;

import java.lang.reflect.Field;
import java.util.HashMap;
import java.util.Map;

import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

public abstract class GeneratorTest<F extends FileObject> {

    private final String classCode;
    protected ProcessingEnvironment mockProcessingEnv;

    private final String resultParameter;
    private final boolean generatedAnnotation;

    public GeneratorTest(String classCode, String resultParameter, boolean generatedAnnotation) {
        this.classCode = classCode;
        mockProcessingEnv = mock(ProcessingEnvironment.class);

        this.resultParameter = resultParameter;
        this.generatedAnnotation = generatedAnnotation;
    }

    @Before
    public void setUpOptions() {
        Map<String, String> options = new HashMap<>();
        options.put("forsuredb.resultParameter", resultParameter);
        options.put("forsuredb.addGeneratedAnnotation", Boolean.toString(generatedAnnotation));
        resetPropertyRetrieverOptions(options);
    }

    @Test
    public void shouldMatchExpectedClassCode() {
        assertEquals(classCode, generatorUnderTest().getCode());
    }

    protected abstract BaseGenerator<F> generatorUnderTest();

    private static void resetPropertyRetrieverOptions(Map<String, String> options) {
        try {
            PropertyRetriever pr = PropertyRetriever.properties();
            Field optionsField = PropertyRetriever.class.getDeclaredField("options");
            optionsField.setAccessible(true);
            Map<String, String> prOptions = (Map<String, String>) optionsField.get(pr);
            prOptions.clear();
            prOptions.putAll(options);
        } catch (Exception e) {
            // in this case, the PropertyRetriever has not yet been initialized
            ProcessingEnvironment mockProcessingEnv = mock(ProcessingEnvironment.class);
            when(mockProcessingEnv.getOptions()).thenReturn(options);
            PropertyRetriever.init(mockProcessingEnv);
        }
    }
}


text:
<!DOCTYPE html>
<title>Tests that the media player will send the relevant cookies when requesting the media file.</title>
<script src="../resources/testharness.js"></script>
<script src="../resources/testharnessreport.js"></script>
<script src="../../media-resources/media-file.js"></script>
<body>
<video></video>
<script>
async_test(function(t) {
    var movie = findMediaFile("video", "resources/test");
    var frame = document.createElement("iframe");
    document.body.appendChild(frame);

    frame.onload = t.step_func(function() {
        var video = document.querySelector("video");
        video.src="http://127.0.0.1:8000/media/resources/video-cookie-check-cookie.php";
        video.oncanplay = t.step_func_done();
        video.play();
    });

    frame.src = "http://127.0.0.1:8000/media/resources/setCookie.cgi?name=" + movie;
});
</script>


text:
package python

import (
	"testing"

	"github.com/anchore/syft/syft/artifact"
	"github.com/anchore/syft/syft/pkg"
	"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
	"github.com/anchore/syft/syft/source"
)

func TestParsePipFileLock(t *testing.T) {

	fixture := "test-fixtures/pipfile-lock/Pipfile.lock"
	locations := source.NewLocationSet(source.NewLocation(fixture))
	expectedPkgs := []pkg.Package{
		{
			Name:      "aio-pika",
			Version:   "6.8.0",
			PURL:      "pkg:pypi/aio-pika@6.8.0",
			Locations: locations,
			Language:  pkg.Python,
			Type:      pkg.PythonPkg,
		},
		{
			Name:      "aiodns",
			Version:   "2.0.0",
			PURL:      "pkg:pypi/aiodns@2.0.0",
			Locations: locations,
			Language:  pkg.Python,
			Type:      pkg.PythonPkg,
		},
		{
			Name:      "aiohttp",
			Version:   "3.7.4.post0",
			PURL:      "pkg:pypi/aiohttp@3.7.4.post0",
			Locations: locations,
			Language:  pkg.Python,
			Type:      pkg.PythonPkg,
		},
		{
			Name:      "aiohttp-jinja2",
			Version:   "1.4.2",
			PURL:      "pkg:pypi/aiohttp-jinja2@1.4.2",
			Locations: locations,
			Language:  pkg.Python,
			Type:      pkg.PythonPkg,
		},
	}

	// TODO: relationships are not under test
	var expectedRelationships []artifact.Relationship

	pkgtest.TestFileParser(t, fixture, parsePipfileLock, expectedPkgs, expectedRelationships)
}


text:
<?xml version="1.0" encoding="utf-8"?>
<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    android:background="@color/off_white_background">

    <android.support.v7.widget.Toolbar
        android:id="@+id/ctodo_tlb"
        android:layout_width="match_parent"
        android:layout_height="?actionBarSize"
        android:background="@color/primaryColor"
        android:elevation="4dp"
        android:theme="@style/ThemeOverlay.AppCompat.Dark.ActionBar">

    </android.support.v7.widget.Toolbar>

    <FrameLayout
        android:id="@+id/ctodo_fl_fragment"
        android:layout_below="@id/ctodo_tlb"
        android:layout_width="match_parent"
        android:layout_height="match_parent">
    </FrameLayout>

</RelativeLayout>


text:
function [ filt_segm_image] = majorityFilterSegmentation( segm_image, window_size)
%% majorityFilterSegmentation  denoises a segmentation using majority filter
%   segm_image - the noisy segmented image
%   window_size - vector for 2 elements- the window size of rows and columns
%               to define each pixels neighbourhood
%   filt_segm_image- the filtered segmented image
% For Testing use test_majorityFilterSegmentation

%% input control
if nargin < 2
    error('majorityFilterSegmentation: not enough input arguments!');
end

%% processing
fun = @(block_struct) mode(block_struct.data(:)) * ones(size(block_struct.data));
filt_segm_image = blockproc(segm_image, window_size,fun);


text:
package org.killbill.billing.subscription.events.user;

public class ApiEventTransfer extends ApiEventBase {
    public ApiEventTransfer(final ApiEventBuilder builder) {
        super(builder.setApiEventType(ApiEventType.TRANSFER));
    }

}


text:
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright (c) 2001-2004 World Wide Web Consortium,
(Massachusetts Institute of Technology, Institut National de
Recherche en Informatique et en Automatique, Keio University). All
Rights Reserved. This program is distributed under the W3C's Software
Intellectual Property License. This program is distributed in the
hope that it will be useful, but WITHOUT ANY WARRANTY; without even
the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
PURPOSE.
See W3C License http://www.w3.org/Consortium/Legal/ for more details.
--><!DOCTYPE test SYSTEM "dom1.dtd">

<test xmlns="http://www.w3.org/2001/DOM-Test-Suite/Level-1" name="hc_attrinsertbefore6">
<metadata>
<title>hc_attrinsertbefore6</title>
<creator>Curt Arnold</creator>
<description>
Attempt to append a text node from another document to an attribute which should result
in a WRONG_DOCUMENT_ERR.
</description>

<date qualifier="created">2004-01-01</date>
<!--  Attr interface  -->
<subject resource="http://www.w3.org/TR/1998/REC-DOM-Level-1-19981001/level-one-core#ID-637646024"/>
<!--  Node.insertBefore   -->
<subject resource="http://www.w3.org/TR/1998/REC-DOM-Level-1-19981001/level-one-core#ID-952280727"/>
</metadata>

<var name="doc" type="Document"/>
<var name="acronymList" type="NodeList"/>
<var name="testNode" type="Node"/>
<var name="attributes" type="NamedNodeMap"/>
<var name="titleAttr" type="Attr"/>
<var name="value" type="DOMString"/>
<var name="textNode" type="Node"/>
<var name="retval" type="Node"/>
<var name="refChild" type="Node" isNull="true"/>
<var name="otherDoc" type="Document"/>
<load var="doc" href="hc_staff" willBeModified="true"/>
<load var="otherDoc" href="hc_staff" willBeModified="true"/>
<getElementsByTagName interface="Document" obj="doc" var="acronymList" tagname='"acronym"'/>
<item interface="NodeList" obj="acronymList" var="testNode" index="3"/>
<attributes obj="testNode" var="attributes"/>
<getNamedItem obj="attributes" var="titleAttr" name='"title"'/>
<createTextNode var="textNode" obj="otherDoc" data='"terday"'/>

<assertDOMException id="throw_WRONG_DOCUMENT_ERR">
	<WRONG_DOCUMENT_ERR>
		<insertBefore var="retval" obj="titleAttr" newChild="textNode" refChild="refChild"/>
	</WRONG_DOCUMENT_ERR>
</assertDOMException>
</test>


text:
package com.cobber.fta.plugins;

import java.util.Set;

import com.cobber.fta.AnalysisConfig;
import com.cobber.fta.AnalyzerContext;
import com.cobber.fta.Facts;
import com.cobber.fta.FiniteMap;
import com.cobber.fta.KnownTypes;
import com.cobber.fta.LogicalTypeInfinite;
import com.cobber.fta.PluginAnalysis;
import com.cobber.fta.PluginDefinition;
import com.cobber.fta.SingletonSet;
import com.cobber.fta.core.FTAPluginException;
import com.cobber.fta.core.FTAType;
import com.cobber.fta.dates.DateTimeParser;
import com.cobber.fta.token.TokenStreams;

/**
 * Plugin to detect valid US Zip codes.
 * Note: we used an Infinite :-) Semantic Type since the domains is so large.
 */
public class USZip5 extends LogicalTypeInfinite {
	/** The Semantic type for this Plugin. */
	public static final String SEMANTIC_TYPE = "POSTAL_CODE.ZIP5_US";
	public static final String REGEXP_ZIP5 = "\\d{5}";
	public static final String REGEXP_VARIABLE = "\\d{3,5}";
	private int minLength = 5;
	private SingletonSet zipsRef;
	private Set<String> zips;

	/**
	 * Construct a US ZIP (See also @link USZipPlus4) plugin based on the Plugin Definition.{@link DateTimeParser}
	 * @param plugin The definition of this plugin.
	 */
	public USZip5(final PluginDefinition plugin) {
		super(plugin);
	}

	@Override
	public boolean isCandidate(final String trimmed, final StringBuilder compressed, final int[] charCounts, final int[] lastIndex) {
		final int len = trimmed.length();
		if (len != 5 && len != 4 && len != 3)
			return false;

		final int digits = charCounts['0'] + charCounts['1'] + charCounts['2'] + charCounts['3'] + charCounts['4'] +
				charCounts['5'] + charCounts['6'] + charCounts['7'] + charCounts['8'] + charCounts['9'];
		if (len == 5)
			return digits == 5;

		return zips.contains(len == 3 ? "00" + trimmed : "0" + trimmed);
	}

	@Override
	public boolean initialize(final AnalysisConfig analysisConfig) throws FTAPluginException {
		super.initialize(analysisConfig);

		zipsRef = new SingletonSet("resource", "/reference/us_zips.csv");
		zips = zipsRef.getMembers();

		return true;
	}

	@Override
	public String nextRandom() {
		return zipsRef.getRandom(random);
	}

	@Override
	public String getSemanticType() {
		return SEMANTIC_TYPE;
	}

	@Override
	public String getRegExp() {
		return minLength == 3 ? REGEXP_VARIABLE : REGEXP_ZIP5;
	}

	@Override
	public FTAType getBaseType() {
		return FTAType.LONG;
	}

	@Override
	public boolean isValid(String input, final boolean detectMode) {
		final int len = input.length();

		if (len != 5 && len != 4 && len != 3)
			return false;

		if (len < 5) {
			input = (len == 3 ? "00" : "0") + input;
			minLength = 3;
		}

		return zips.contains(input);
	}

	private String backout() {
		return KnownTypes.PATTERN_NUMERIC_VARIABLE;
	}

	@Override
	public PluginAnalysis analyzeSet(final AnalyzerContext context, final long matchCount, final long realSamples, final String currentRegExp, final Facts facts, final FiniteMap cardinality, final FiniteMap outliers, final TokenStreams tokenStreams, final AnalysisConfig analysisConfig) {
		final int headerConfidence = getHeaderConfidence(context.getStreamName());
		if (headerConfidence == 0 && cardinality.size() < 5)
			return new PluginAnalysis(backout());

		if (getConfidence(matchCount, realSamples, context) >= getThreshold()/100.0)
			return PluginAnalysis.OK;

		return new PluginAnalysis(backout());
	}

	@Override
	public double getConfidence(final long matchCount, final long realSamples, final AnalyzerContext context) {
		double confidence = (double)matchCount/realSamples;

		// Boost by up to 20% if we like the header
		if (getHeaderConfidence(context.getStreamName()) != 0)
			confidence = Math.min(confidence + Math.min((1.0 - confidence)/2, 0.20), 1.0);

		return confidence;
	}
}


text:
package org.hsqldb;

import org.hsqldb.error.Error;
import org.hsqldb.error.ErrorCode;
import org.hsqldb.lib.ArrayUtil;
import org.hsqldb.lib.HsqlList;
import org.hsqldb.map.ValuePool;
import org.hsqldb.types.BinaryData;
import org.hsqldb.types.BinaryType;
import org.hsqldb.types.CharacterType;
import org.hsqldb.types.DateTimeType;
import org.hsqldb.types.IntervalType;
import org.hsqldb.types.Type;
import org.hsqldb.types.Types;

/**
 * Implementation of CAST, CASE, LIMIT and ZONE operations.
 *
 * @author Campbell Boucher-Burnet (boucherb@users dot sourceforge.net)
 * @author Fred Toussi (fredt@users dot sourceforge.net)
 * @version 2.3.0
 * @since 1.9.0
 */
public class ExpressionOp extends Expression {

    static final ExpressionOp limitOneExpression = new ExpressionOp(
        OpTypes.LIMIT,
        new ExpressionValue(ValuePool.INTEGER_0, Type.SQL_INTEGER),
        new ExpressionValue(ValuePool.INTEGER_1, Type.SQL_INTEGER));

    /**
     * Creates a multiple arg operation expression
     */
    ExpressionOp(int type, Expression[] exprArray) {

        super(type);

        switch (opType) {

            case OpTypes.CONCAT_WS :
                nodes = exprArray;

                return;

            default :
                throw Error.runtimeError(ErrorCode.U_S0500, "ExpressionOp");
        }
    }

    /**
     * Creates a special binary operation expression
     */
    ExpressionOp(int type, Expression left, Expression right) {

        super(type);

        nodes        = new Expression[BINARY];
        nodes[LEFT]  = left;
        nodes[RIGHT] = right;

        switch (opType) {

            case OpTypes.LIKE_ARG :
            case OpTypes.ALTERNATIVE :
            case OpTypes.CASEWHEN :
            case OpTypes.LIMIT :
            case OpTypes.ZONE_MODIFIER :
                return;

            case OpTypes.PREFIX :
                dataType = left.dataType;

                return;

            default :
                throw Error.runtimeError(ErrorCode.U_S0500, "ExpressionOp");
        }
    }

    /**
     * creates a CAST expression
     */
    ExpressionOp(Expression e, Type dataType) {

        super(OpTypes.CAST);

        nodes         = new Expression[UNARY];
        nodes[LEFT]   = e;
        this.dataType = dataType;
        this.alias    = e.alias;
    }

    /**
     * creates a special conversion for time / timestamp comparison
     */
    ExpressionOp(Expression e) {

        super(e.dataType.isDateTimeTypeWithZone() ? OpTypes.CAST
                                                  : OpTypes.ZONE_MODIFIER);

        switch (e.dataType.typeCode) {

            case Types.SQL_TIME_WITH_TIME_ZONE :
                nodes                = new Expression[UNARY];
                nodes[LEFT] = new ExpressionOp(OpTypes.ZONE_MODIFIER, e, null);
                nodes[LEFT].dataType = e.dataType;
                dataType = DateTimeType.getDateTimeType(Types.SQL_TIME,
                        e.dataType.scale);
                break;

            case Types.SQL_TIMESTAMP_WITH_TIME_ZONE :
                nodes                = new Expression[UNARY];
                nodes[LEFT] = new ExpressionOp(OpTypes.ZONE_MODIFIER, e, null);
                nodes[LEFT].dataType = e.dataType;
                dataType = DateTimeType.getDateTimeType(Types.SQL_TIMESTAMP,
                        e.dataType.scale);
                break;

            case Types.SQL_TIME :
                nodes                = new Expression[BINARY];
                nodes[LEFT]          = e;
                nodes[LEFT].dataType = e.dataType;
                dataType =
                    DateTimeType.getDateTimeType(Types.SQL_TIME_WITH_TIME_ZONE,
                                                 e.dataType.scale);
                break;

            case Types.SQL_TIMESTAMP :
                nodes                = new Expression[BINARY];
                nodes[LEFT]          = e;
                nodes[LEFT].dataType = e.dataType;
                dataType = DateTimeType.getDateTimeType(
                    Types.SQL_TIMESTAMP_WITH_TIME_ZONE, e.dataType.scale);
                break;

            default :
                throw Error.runtimeError(ErrorCode.U_S0500, "ExpressionOp");
        }

        this.alias = e.alias;
    }

    public static Expression getCastExpression(Session session, Expression e,
            Type dataType) {

        if (e.getType() == OpTypes.VALUE) {
            Object value = dataType.castToType(session, e.getValue(session),
                                               e.getDataType());

            return new ExpressionValue(value, dataType);
        }

        return new ExpressionOp(e, dataType);
    }

    public String getSQL() {

        StringBuffer sb    = new StringBuffer(64);
        String       left  = getContextSQL(nodes.length > 0 ? nodes[LEFT]
                                                            : null);
        String       right = getContextSQL(nodes.length > 1 ? nodes[RIGHT]
                                                            : null);

        switch (opType) {

            case OpTypes.VALUE :
                if (valueData == null) {
                    return Tokens.T_NULL;
                }

                if (dataType == null) {
                    throw Error.runtimeError(ErrorCode.U_S0500,
                                             "ExpressionOp");
                }

                return dataType.convertToSQLString(valueData);

            case OpTypes.LIKE_ARG :
                sb.append(' ').append(Tokens.T_LIKE).append(' ');
                sb.append(left).append(' ').append(right).append(' ');
            case OpTypes.CAST :
                sb.append(' ').append(Tokens.T_CAST).append('(');
                sb.append(left).append(' ').append(Tokens.T_AS).append(' ');
                sb.append(dataType.getTypeDefinition());
                sb.append(')');

                return sb.toString();

            case OpTypes.CASEWHEN :
                sb.append(' ').append(Tokens.T_CASEWHEN).append('(');
                sb.append(left).append(',').append(right).append(')');

                return sb.toString();

            case OpTypes.ALTERNATIVE :
                sb.append(left).append(',').append(right);

                return sb.toString();

            case OpTypes.LIMIT :
                if (left != null) {
                    sb.append(' ').append(Tokens.T_OFFSET).append(' ');
                    sb.append(left).append(' ');
                }

                if (right != null) {
                    sb.append(' ').append(Tokens.T_FETCH).append(' ');
                    sb.append(Tokens.T_FIRST);
                    sb.append(right).append(' ').append(right).append(' ');
                    sb.append(Tokens.T_ROWS).append(' ').append(Tokens.T_ONLY);
                    sb.append(' ');
                }
                break;

            case OpTypes.ZONE_MODIFIER :
                sb.append(left).append(' ').append(Tokens.T_AT).append(' ');

                if (nodes[RIGHT] == null) {
                    sb.append(Tokens.T_LOCAL).append(' ');

                    break;
                }

                sb.append(Tokens.T_TIME).append(' ').append(Tokens.T_ZONE);
                sb.append(' ');
                sb.append(right);
                break;

            case OpTypes.CONCAT_WS :
                sb.append(Tokens.T_CONCAT_WS).append(Tokens.OPENBRACKET);
                sb.append(left);

                for (int i = 0; i < nodes.length; i++) {
                    sb.append(',').append(nodes[i].getSQL());
                }

                sb.append(Tokens.CLOSEBRACKET);
                break;

            default :
                throw Error.runtimeError(ErrorCode.U_S0500, "ExpressionOp");
        }

        return sb.toString();
    }

    protected String describe(Session session, int blanks) {

        StringBuffer sb = new StringBuffer(64);

        sb.append('\n');

        for (int i = 0; i < blanks; i++) {
            sb.append(' ');
        }

        switch (opType) {

            case OpTypes.VALUE :
                sb.append("VALUE = ").append(
                    dataType.convertToSQLString(valueData));
                sb.append(", TYPE = ").append(dataType.getNameString());

                return sb.toString();

            case OpTypes.LIKE_ARG :
                sb.append(Tokens.T_LIKE).append(' ').append("ARG ");
                sb.append(dataType.getTypeDefinition());
                sb.append(' ');
                break;

            case OpTypes.VALUELIST :
                sb.append(Tokens.T_VALUE).append(' ').append("LIST ");

                for (int i = 0; i < nodes.length; i++) {
                    sb.append(nodes[i].describe(session, blanks + 1));
                    sb.append(' ');
                }

                return sb.toString();

            case OpTypes.CAST :
                sb.append(Tokens.T_CAST).append(' ');
                sb.append(dataType.getTypeDefinition());
                sb.append(' ');
                break;

            case OpTypes.CASEWHEN :
                sb.append(Tokens.T_CASEWHEN).append(' ');
                break;

            case OpTypes.CONCAT_WS :
                sb.append(Tokens.T_CONCAT_WS).append(' ');
                break;
        }

        if (getLeftNode() != null) {
            sb.append(" arg_left=[");
            sb.append(nodes[LEFT].describe(session, blanks + 1));
            sb.append(']');
        }

        if (getRightNode() != null) {
            sb.append(" arg_right=[");
            sb.append(nodes[RIGHT].describe(session, blanks + 1));
            sb.append(']');
        }

        return sb.toString();
    }

    public HsqlList resolveColumnReferences(Session session,
            RangeGroup rangeGroup, int rangeCount, RangeGroup[] rangeGroups,
            HsqlList unresolvedSet, boolean acceptsSequences) {

        if (opType == OpTypes.VALUE) {
            return unresolvedSet;
        }

        switch (opType) {

            case OpTypes.CASEWHEN :
                acceptsSequences = false;
                break;
        }

        for (int i = 0; i < nodes.length; i++) {
            if (nodes[i] == null) {
                continue;
            }

            unresolvedSet = nodes[i].resolveColumnReferences(session,
                    rangeGroup, rangeCount, rangeGroups, unresolvedSet,
                    acceptsSequences);
        }

        return unresolvedSet;
    }

    public void resolveTypes(Session session, Expression parent) {

        switch (opType) {

            case OpTypes.CASEWHEN :
                break;

            default :
                for (int i = 0; i < nodes.length; i++) {
                    if (nodes[i] != null) {
                        nodes[i].resolveTypes(session, this);
                    }
                }
        }

        switch (opType) {

            case OpTypes.VALUE :
                break;

            case OpTypes.LIKE_ARG : {
                dataType = nodes[LEFT].dataType;

                if (nodes[LEFT].opType == OpTypes.VALUE
                        && (nodes[RIGHT] == null
                            || nodes[RIGHT].opType == OpTypes.VALUE)) {
                    setAsConstantValue(session, parent);

                    break;
                }

                break;
            }
            case OpTypes.CAST : {
                Expression node     = nodes[LEFT];
                Type       nodeType = node.dataType;

                if (nodeType != null && !dataType.canConvertFrom(nodeType)) {
                    throw Error.error(ErrorCode.X_42561);
                }

                if (node.opType == OpTypes.VALUE) {
                    setAsConstantValue(session, parent);

                    node.dataType  = dataType;
                    node.valueData = valueData;

                    if (parent != null) {
                        parent.replaceNode(this, node);
                    }
                } else if (nodes[LEFT].opType == OpTypes.DYNAMIC_PARAM) {
                    node.dataType = dataType;
                }

                break;
            }
            case OpTypes.ZONE_MODIFIER :
                if (nodes[LEFT].dataType == null) {
                    throw Error.error(ErrorCode.X_42567);
                }

                if (nodes[RIGHT] != null) {
                    if (nodes[RIGHT].dataType == null) {
                        nodes[RIGHT].dataType =
                            Type.SQL_INTERVAL_HOUR_TO_MINUTE;
                    }

                    if (nodes[RIGHT].dataType.typeCode
                            != Types.SQL_INTERVAL_HOUR_TO_MINUTE) {
                        if (nodes[RIGHT].opType == OpTypes.VALUE) {
                            nodes[RIGHT].valueData =
                                Type.SQL_INTERVAL_HOUR_TO_MINUTE.castToType(
                                    session, nodes[RIGHT].valueData,
                                    nodes[RIGHT].dataType);
                            nodes[RIGHT].dataType =
                                Type.SQL_INTERVAL_HOUR_TO_MINUTE;
                        } else {
                            throw Error.error(ErrorCode.X_42563);
                        }
                    }
                }

                switch (nodes[LEFT].dataType.typeCode) {

                    case Types.SQL_TIME :
                        dataType = DateTimeType.getDateTimeType(
                            Types.SQL_TIME_WITH_TIME_ZONE,
                            nodes[LEFT].dataType.scale);
                        break;

                    case Types.SQL_TIMESTAMP :
                        dataType = DateTimeType.getDateTimeType(
                            Types.SQL_TIMESTAMP_WITH_TIME_ZONE,
                            nodes[LEFT].dataType.scale);
                        break;

                    case Types.SQL_TIMESTAMP_WITH_TIME_ZONE :
                    case Types.SQL_TIME_WITH_TIME_ZONE :
                        dataType = nodes[LEFT].dataType;
                        break;

                    default :
                        throw Error.error(ErrorCode.X_42563);
                }

                // no constant optimisation as value dependent on session zone
                break;

            case OpTypes.CASEWHEN :

                // We use CASEWHEN as parent type.
                // In the parent, left node is the condition, and right node is
                // the leaf, tagged as type ALTERNATIVE; its left node is
                // case 1 (how to get the value when the condition in
                // the parent evaluates to true), while its right node is case 2
                // (how to get the value when the condition in
                // the parent evaluates to false).
                resolveTypesForCaseWhen(session);
                break;

            case OpTypes.CONCAT_WS :
                for (int i = 0; i < nodes.length; i++) {
                    nodes[i].dataType = Type.SQL_VARCHAR_DEFAULT;
                }

                dataType = Type.SQL_VARCHAR_DEFAULT;
                break;

            case OpTypes.ALTERNATIVE :
                break;

            case OpTypes.LIMIT :
                if (nodes[LEFT] != null) {
                    if (nodes[LEFT].dataType == null) {
                        throw Error.error(ErrorCode.X_42567);
                    }

                    if (!nodes[LEFT].dataType.isIntegralType()) {
                        throw Error.error(ErrorCode.X_42563);
                    }
                }

                if (nodes[RIGHT] != null) {
                    if (nodes[RIGHT].dataType == null) {
                        throw Error.error(ErrorCode.X_42567);
                    }

                    if (!nodes[RIGHT].dataType.isIntegralType()) {
                        throw Error.error(ErrorCode.X_42563);
                    }
                }
                break;

            case OpTypes.PREFIX :
                break;

            default :
                throw Error.runtimeError(ErrorCode.U_S0500, "ExpressionOp");
        }
    }

    /**
     * For CASE WHEN and its special cases section 9.3 of the SQL standard
     * on type aggregation is implemented.
     */
    void resolveTypesForCaseWhen(Session session) {

        if (dataType != null) {
            return;
        }

        Expression expr = this;

        while (expr.opType == OpTypes.CASEWHEN) {
            expr.nodes[LEFT].resolveTypes(session, expr);

            if (expr.nodes[LEFT].isUnresolvedParam()) {
                expr.nodes[LEFT].dataType = Type.SQL_BOOLEAN;
            }

            expr.nodes[RIGHT].nodes[LEFT].resolveTypes(session,
                    expr.nodes[RIGHT]);

            if (expr.nodes[RIGHT].nodes[RIGHT].opType != OpTypes.CASEWHEN) {
                expr.nodes[RIGHT].nodes[RIGHT].resolveTypes(session,
                        expr.nodes[RIGHT]);
            }

            expr = expr.nodes[RIGHT].nodes[RIGHT];
        }

        if (exprSubType == OpTypes.CAST) {
            if (nodes[RIGHT].nodes[RIGHT].dataType != null
                    && nodes[RIGHT].nodes[RIGHT].dataType
                       != nodes[RIGHT].nodes[LEFT].dataType) {
                Type castType = nodes[RIGHT].nodes[RIGHT].dataType;

                if (castType.isCharacterType()) {
                    castType = Type.SQL_VARCHAR_DEFAULT;
                }

                nodes[RIGHT].nodes[LEFT] =
                    new ExpressionOp(nodes[RIGHT].nodes[LEFT], castType);
            }
        }

        expr = this;

        while (expr.opType == OpTypes.CASEWHEN) {
            dataType =
                Type.getAggregateType(expr.nodes[RIGHT].nodes[LEFT].dataType,
                                      dataType);
            dataType =
                Type.getAggregateType(expr.nodes[RIGHT].nodes[RIGHT].dataType,
                                      dataType);
            expr = expr.nodes[RIGHT].nodes[RIGHT];
        }

        expr = this;

        while (expr.opType == OpTypes.CASEWHEN) {
            if (expr.nodes[RIGHT].nodes[LEFT].dataType == null) {
                expr.nodes[RIGHT].nodes[LEFT].dataType = dataType;
            }

            if (expr.nodes[RIGHT].nodes[RIGHT].dataType == null) {
                expr.nodes[RIGHT].nodes[RIGHT].dataType = dataType;
            }

            if (expr.nodes[RIGHT].dataType == null) {
                expr.nodes[RIGHT].dataType = dataType;
            }

            expr = expr.nodes[RIGHT].nodes[RIGHT];
        }

        if (dataType == null || dataType.typeCode == Types.SQL_ALL_TYPES) {
            throw Error.error(ErrorCode.X_42567);
        }
    }

    public Object getValue(Session session) {

        switch (opType) {

            case OpTypes.VALUE :
                return valueData;

            case OpTypes.LIKE_ARG : {
                boolean hasEscape  = nodes[RIGHT] != null;
                int     escapeChar = Integer.MAX_VALUE;

                if (dataType.isBinaryType()) {
                    BinaryData left =
                        (BinaryData) nodes[LEFT].getValue(session);

                    if (left == null) {
                        return null;
                    }

                    if (hasEscape) {
                        BinaryData right =
                            (BinaryData) nodes[RIGHT].getValue(session);

                        if (right == null) {
                            return null;
                        }

                        if (right.length(session) != 1) {
                            throw Error.error(ErrorCode.X_2200D);
                        }

                        escapeChar = right.getBytes()[0];
                    }

                    byte[]  array       = left.getBytes();
                    byte[]  newArray    = new byte[array.length];
                    boolean wasEscape   = false;
                    int     escapeCount = 0;
                    int     i           = 0;
                    int     j           = 0;

                    for (; i < array.length; i++) {
                        if (array[i] == escapeChar) {
                            if (wasEscape) {
                                escapeCount++;

                                newArray[j++] = array[i];
                                wasEscape     = false;

                                continue;
                            }

                            wasEscape = true;

                            if (i == array.length - 1) {
                                throw Error.error(ErrorCode.X_22025);
                            }

                            continue;
                        }

                        if (array[i] == '_' || array[i] == '%') {
                            if (wasEscape) {
                                escapeCount++;

                                newArray[j++] = array[i];
                                wasEscape     = false;

                                continue;
                            }

                            break;
                        }

                        if (wasEscape) {
                            throw Error.error(ErrorCode.X_22025);
                        }

                        newArray[j++] = array[i];
                    }

                    newArray =
                        (byte[]) ArrayUtil.resizeArrayIfDifferent(newArray, j);

                    return new BinaryData(newArray, false);
                } else {
                    String left =
                        (String) Type.SQL_VARCHAR.convertToType(session,
                            nodes[LEFT].getValue(session),
                            nodes[LEFT].getDataType());

                    if (left == null) {
                        return null;
                    }

                    if (hasEscape) {
                        String right =
                            (String) Type.SQL_VARCHAR.convertToType(session,
                                nodes[RIGHT].getValue(session),
                                nodes[RIGHT].getDataType());

                        if (right == null) {
                            return null;
                        }

                        if (right.length() != 1) {
                            throw Error.error(ErrorCode.X_22019);
                        }

                        escapeChar = right.getBytes()[0];
                    }

                    char[]  array       = left.toCharArray();
                    char[]  newArray    = new char[array.length];
                    boolean wasEscape   = false;
                    int     escapeCount = 0;
                    int     i           = 0;
                    int     j           = 0;

                    for (; i < array.length; i++) {
                        if (array[i] == escapeChar) {
                            if (wasEscape) {
                                escapeCount++;

                                newArray[j++] = array[i];
                                wasEscape     = false;

                                continue;
                            }

                            wasEscape = true;

                            if (i == array.length - 1) {
                                throw Error.error(ErrorCode.X_22025);
                            }

                            continue;
                        }

                        if (array[i] == '_' || array[i] == '%') {
                            if (wasEscape) {
                                escapeCount++;

                                newArray[j++] = array[i];
                                wasEscape     = false;

                                continue;
                            }

                            break;
                        }

                        if (wasEscape) {
                            throw Error.error(ErrorCode.X_22025);
                        }

                        newArray[j++] = array[i];
                    }

                    return new String(newArray, 0, j);
                }
            }
            case OpTypes.SIMPLE_COLUMN : {
                Object value =
                    session.sessionContext.rangeIterators[rangePosition]
                        .getCurrent(columnIndex);

                return value;
            }
            case OpTypes.ORDER_BY :
                return nodes[LEFT].getValue(session);

            case OpTypes.PREFIX : {
                if (nodes[LEFT].dataType.isCharacterType()) {
                    Object value = nodes[RIGHT].getValue(session);

                    if (value == null) {
                        return null;
                    }

                    CharacterType type = (CharacterType) nodes[RIGHT].dataType;
                    long length =
                        ((CharacterType) nodes[RIGHT].dataType).size(session,
                            value);

                    type  = (CharacterType) nodes[LEFT].dataType;
                    value = nodes[LEFT].getValue(session);

                    if (value == null) {
                        return null;
                    }

                    return type.substring(session, value, 0, length, true,
                                          false);
                } else {
                    BinaryData value =
                        (BinaryData) nodes[RIGHT].getValue(session);

                    if (value == null) {
                        return null;
                    }

                    long       length = value.length(session);
                    BinaryType type   = (BinaryType) nodes[LEFT].dataType;

                    value = (BinaryData) nodes[LEFT].getValue(session);

                    if (value == null) {
                        return null;
                    }

                    return type.substring(session, value, 0, length, true);
                }
            }
            case OpTypes.CAST : {
                Object value =
                    dataType.castToType(session,
                                        nodes[LEFT].getValue(session),
                                        nodes[LEFT].dataType);

                if (dataType.userTypeModifier != null) {
                    Constraint[] constraints =
                        dataType.userTypeModifier.getConstraints();

                    for (int i = 0; i < constraints.length; i++) {
                        constraints[i].checkCheckConstraint(session, null,
                                                            null, value);
                    }
                }

                return value;
            }
            case OpTypes.CASEWHEN : {
                Boolean result = (Boolean) nodes[LEFT].getValue(session);

                if (Boolean.TRUE.equals(result)) {
                    return nodes[RIGHT].nodes[LEFT].getValue(session,
                            dataType);
                } else {
                    return nodes[RIGHT].nodes[RIGHT].getValue(session,
                            dataType);
                }
            }
            case OpTypes.CONCAT_WS : {
                String sep = (String) nodes[LEFT].getValue(session);

                if (sep == null) {
                    return null;
                }

                StringBuffer sb       = new StringBuffer("");
                boolean      hasValue = false;

                for (int i = 1; i < nodes.length; i++) {
                    String value = (String) nodes[i].getValue(session);

                    if (value == null) {
                        continue;
                    }

                    if (hasValue) {
                        sb.append(sep);
                    }

                    sb.append(value);

                    hasValue = true;
                }

                return sb.toString();
            }
            case OpTypes.ZONE_MODIFIER : {
                Object leftValue  = nodes[LEFT].getValue(session);
                Object rightValue = nodes[RIGHT] == null ? null
                                                         : nodes[RIGHT]
                                                             .getValue(
                                                                 session);

                if (leftValue == null) {
                    return null;
                }

                if (nodes[RIGHT] != null && rightValue == null) {
                    return null;
                }

                long zoneSeconds = nodes[RIGHT] == null
                                   ? session.getZoneSeconds()
                                   : ((IntervalType) nodes[RIGHT].dataType)
                                       .getSeconds(rightValue);

                return ((DateTimeType) dataType).changeZone(leftValue,
                        nodes[LEFT].dataType, (int) zoneSeconds,
                        session.getZoneSeconds());
            }
            case OpTypes.LIMIT :

            // fall through
            default :
                throw Error.runtimeError(ErrorCode.U_S0500, "ExpressionOp");
        }
    }
}


text:
Common Mistakes #3

(*) not real books


text:
easyconfig
==========

1. Subscriber/Publisher model.
2. Realtime server pushing mechanism when configuration changed.
3. Support configuration for different environments - daily/test/prod.


text:
#include <aws/cognito-sync/model/UnsubscribeFromDatasetResult.h>
#include <aws/core/utils/json/JsonSerializer.h>
#include <aws/core/AmazonWebServiceResult.h>
#include <aws/core/utils/StringUtils.h>
#include <aws/core/utils/UnreferencedParam.h>

#include <utility>

using namespace Aws::CognitoSync::Model;
using namespace Aws::Utils::Json;
using namespace Aws::Utils;
using namespace Aws;

UnsubscribeFromDatasetResult::UnsubscribeFromDatasetResult()
{
}

UnsubscribeFromDatasetResult::UnsubscribeFromDatasetResult(const AmazonWebServiceResult<JsonValue>& result)
{
  *this = result;
}

UnsubscribeFromDatasetResult& UnsubscribeFromDatasetResult::operator =(const AmazonWebServiceResult<JsonValue>& result)
{
  AWS_UNREFERENCED_PARAM(result);

  return *this;
}


text:
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">

<title>co.vandenham.telegram.botapi.requests Class Hierarchy (TelegramBots4Java)</title>
<meta name="date" content="2015-09-01">
<link rel="stylesheet" type="text/css" href="../../../../../stylesheet.css" title="Style">

JavaScript is disabled on your browser.

Skip navigation links

Overview

Package

Class

Tree

Deprecated

Index

Help

Prev

Next

Frames

No Frames

All Classes

Hierarchy For Package co.vandenham.telegram.botapi.requests

Package Hierarchies:

All Packages

Class Hierarchy

java.lang.Object

co.vandenham.telegram.botapi.requests.ApiRequestExecutor

co.vandenham.telegram.botapi.requests.ApiResult<T>

co.vandenham.telegram.botapi.requests.ForwardMessageRequest

co.vandenham.telegram.botapi.requests.GetMeRequest

co.vandenham.telegram.botapi.requests.GetUpdatesRequest

co.vandenham.telegram.botapi.requests.GetUserProfilePhotosRequest

co.vandenham.telegram.botapi.requests.OptionalArgs

co.vandenham.telegram.botapi.requests.SendAudioRequest

co.vandenham.telegram.botapi.requests.SendChatActionRequest

co.vandenham.telegram.botapi.requests.SendDocumentRequest

co.vandenham.telegram.botapi.requests.SendLocationRequest

co.vandenham.telegram.botapi.requests.SendMessageRequest

co.vandenham.telegram.botapi.requests.SendPhotoRequest

co.vandenham.telegram.botapi.requests.SendStickerRequest

co.vandenham.telegram.botapi.requests.SendVideoRequest

co.vandenham.telegram.botapi.requests.SendVoiceRequest

co.vandenham.telegram.botapi.requests.TelegramApi

java.lang.Throwable (implements java.io.Serializable)

java.lang.Exception

java.lang.RuntimeException

co.vandenham.telegram.botapi.requests.ApiException

Interface Hierarchy

co.vandenham.telegram.botapi.requests.ApiResponse<T>

Enum Hierarchy

java.lang.Object

java.lang.Enum<E> (implements java.lang.Comparable<T>, java.io.Serializable)

co.vandenham.telegram.botapi.requests.ChatAction

Skip navigation links

Overview

Package

Class

Tree

Deprecated

Index

Help

Prev

Next

Frames

No Frames

All Classes


text:
/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.facebook.presto.plugin.jdbc;

import com.facebook.presto.spi.ColumnHandle;
import com.facebook.presto.spi.ConnectorPartition;
import com.facebook.presto.spi.ConnectorPartitionResult;
import com.facebook.presto.spi.ConnectorSession;
import com.facebook.presto.spi.ConnectorSplit;
import com.facebook.presto.spi.ConnectorSplitManager;
import com.facebook.presto.spi.ConnectorSplitSource;
import com.facebook.presto.spi.ConnectorTableHandle;
import com.facebook.presto.spi.FixedSplitSource;
import com.facebook.presto.spi.TupleDomain;
import com.google.common.collect.ImmutableList;

import javax.inject.Inject;

import java.util.List;

import static com.facebook.presto.plugin.jdbc.Types.checkType;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;

public class JdbcSplitManager
        implements ConnectorSplitManager
{
    private final String connectorId;
    private final JdbcClient jdbcClient;

    @Inject
    public JdbcSplitManager(JdbcConnectorId connectorId, JdbcClient jdbcClient)
    {
        this.connectorId = checkNotNull(connectorId, "connectorId is null").toString();
        this.jdbcClient = checkNotNull(jdbcClient, "client is null");
    }

    @Override
    public ConnectorPartitionResult getPartitions(ConnectorSession session, ConnectorTableHandle tableHandle, TupleDomain<ColumnHandle> tupleDomain)
    {
        JdbcTableHandle handle = checkType(tableHandle, JdbcTableHandle.class, "tableHandle");
        return jdbcClient.getPartitions(handle, tupleDomain);
    }

    @Override
    public ConnectorSplitSource getPartitionSplits(ConnectorSession session, ConnectorTableHandle tableHandle, List<ConnectorPartition> partitions)
    {
        if (partitions.isEmpty()) {
            return new FixedSplitSource(connectorId, ImmutableList.<ConnectorSplit>of());
        }

        checkArgument(partitions.size() == 1, "Expected one partition but got %s", partitions.size());
        JdbcPartition partition = checkType(partitions.get(0), JdbcPartition.class, "partition");

        return jdbcClient.getPartitionSplits(partition);
    }
}


text:
package io.straas.android.media.demo.widget;

import android.os.SystemClock;
import android.support.annotation.NonNull;
import android.support.v4.app.FragmentActivity;
import android.support.v4.media.MediaMetadataCompat;
import android.support.v4.media.session.MediaControllerCompat;
import android.support.v4.media.session.PlaybackStateCompat;
import android.widget.MediaController;

import io.straas.android.sdk.media.StraasMediaCore;

import static io.straas.android.sdk.media.StraasMediaCore.LIVE_EXTRA_CURRENT_DATE_TIME;

/**
 * An implementation of {@link MediaController.MediaPlayerControl} for controlling an {@link MediaControllerCompat} instance.
 *

 * This class is provided for convenience, however it is expected that most applications will
 * implement their own player controls and therefore not require this class.
 * You could use {@link FragmentActivity#setSupportMediaController(MediaControllerCompat)} for later retrieval via
 * {@link FragmentActivity#getSupportMediaController()}
 *

 *     MediaControllerCompat mediaController = new MediaControllerCompat(this, token);
 *     setSupportMediaController(mediaController);
 *     getSupportMediaController().getTransportControls().playFromMediaId("blablabla", null);
 *

 *
 * @see {@link FragmentActivity#setSupportMediaController(MediaControllerCompat)}
 * @see {@link FragmentActivity#getSupportMediaController()}
 */
public class PlayerControl implements MediaController.MediaPlayerControl {
    private final MediaControllerCompat mMediaControllerCompat;
    private PlaybackStateCompat mLastPlaybackState;
    private MediaMetadataCompat mLastMediaMetadataCompat;
    private final MediaControllerCompat.Callback mCallback;

    public PlayerControl(@NonNull MediaControllerCompat mediaControllerCompat, final MediaController mediaController) {
        mMediaControllerCompat = mediaControllerCompat;
        mCallback = new MediaControllerCompat.Callback() {
            @Override
            public void onPlaybackStateChanged(PlaybackStateCompat state) {
                processPlaybackState(state, mediaController);
            }

            @Override
            public void onMetadataChanged(MediaMetadataCompat metadata) {
                mLastMediaMetadataCompat = metadata;
            }
        };
        mMediaControllerCompat.registerCallback(mCallback);
        mLastMediaMetadataCompat = mMediaControllerCompat.getMetadata();
        processPlaybackState(mMediaControllerCompat.getPlaybackState(), mediaController);
    }

    private void processPlaybackState(PlaybackStateCompat state, MediaController mediaController) {
        if (state == null || state.getActiveQueueItemId() == StraasMediaCore.AD_PLAYBACK_ID) {
            return;
        }
        mLastPlaybackState = state;
        if (mediaController != null && mediaController.isShowing()) {
            mediaController.show();
        }
    }

    public PlayerControl(@NonNull FragmentActivity fragmentActivity, MediaController mediaController) {
        this(MediaControllerCompat.getMediaController(fragmentActivity), mediaController);
    }

    public void release() {
        mMediaControllerCompat.unregisterCallback(mCallback);
    }

    @Override
    public void start() {
        mMediaControllerCompat.getTransportControls().play();
    }

    @Override
    public void pause() {
        mMediaControllerCompat.getTransportControls().pause();
    }

    @Override
    public int getDuration() {
        if (mLastMediaMetadataCompat == null) {
            return 0;
        }
        return (int) mLastMediaMetadataCompat.getLong(MediaMetadataCompat.METADATA_KEY_DURATION);
    }

    @Override
    public int getCurrentPosition() {
        if (mLastPlaybackState == null) {
            return 0;
        }
        long currentPosition = mLastPlaybackState.getPosition();
        if (mLastPlaybackState.getState() == PlaybackStateCompat.STATE_PLAYING) {
            // Calculate the elapsed time between the last position update and now and unless
            // paused, we can assume (delta * speed) + current position is approximately the
            // latest position. This ensure that we do not repeatedly call the getPlaybackState()
            // on MediaControllerCompat.
            long timeDelta = SystemClock.elapsedRealtime() -
                    mLastPlaybackState.getLastPositionUpdateTime();
            currentPosition += (int) timeDelta * mLastPlaybackState.getPlaybackSpeed();
        }
        return (int) currentPosition;
    }

    @Override
    public void seekTo(int pos) {
        mMediaControllerCompat.getTransportControls().seekTo(pos);
    }

    @Override
    public boolean isPlaying() {
        return mLastPlaybackState != null && mLastPlaybackState.getState() == PlaybackStateCompat.STATE_PLAYING;
    }

    @Override
    public int getBufferPercentage() {
        if (mLastPlaybackState == null) {
            return 0;
        }
        long bufferedPosition = mLastPlaybackState.getBufferedPosition();
        long duration = getDuration();
        return (int) (duration == 0 ? 100 : (bufferedPosition * 100) / duration);
    }

    @Override
    public boolean canPause() {
        return true;
    }

    @Override
    public boolean canSeekBackward() {
        return mLastPlaybackState == null || (mLastPlaybackState.getActions() & PlaybackStateCompat.ACTION_SEEK_TO) != 0;
    }

    @Override
    public boolean canSeekForward() {
        return mLastPlaybackState == null || (mLastPlaybackState.getActions() & PlaybackStateCompat.ACTION_SEEK_TO) != 0;
    }

    @Override
    public int getAudioSessionId() {
        throw new UnsupportedOperationException();
    }

    public long getCurrentLiveDateTime() {
        if (mLastPlaybackState == null || mLastPlaybackState.getExtras() == null ||
                !mLastPlaybackState.getExtras().containsKey(LIVE_EXTRA_CURRENT_DATE_TIME)) {
            return 0L;
        }

        long currentLiveDateTime = mLastPlaybackState.getExtras().getLong(LIVE_EXTRA_CURRENT_DATE_TIME);
        if (mLastPlaybackState.getState() == PlaybackStateCompat.STATE_PLAYING) {
            long timeDelta = SystemClock.elapsedRealtime() -
                    mLastPlaybackState.getLastPositionUpdateTime();
            currentLiveDateTime += (long) (timeDelta * mLastPlaybackState.getPlaybackSpeed());
        }
        return currentLiveDateTime;
    }

}