text:
from __future__ import print_function

from collections import defaultdict
from collections import namedtuple
from distutils.version import LooseVersion
import itertools
import os
import re
import shutil
import sys
import time

from .common import get_debian_package_name
from .common import get_release_view_name
from .common import get_short_arch
from .common import Target
from .config import get_index as get_config_index
from .config import get_release_build_files
from .debian_repo import get_debian_repo_data
from .status_page_input import get_rosdistro_info
from .status_page_input import RosPackage
from .templates import expand_template
from .templates import get_template_path

def build_release_status_page(
        config_url, rosdistro_name, release_build_name,
        cache_dir, output_dir, copy_resources=False):
    from rosdistro import get_cached_distribution
    from rosdistro import get_index

    start_time = time.time()

    config = get_config_index(config_url)
    release_build_files = get_release_build_files(config, rosdistro_name)
    build_file = release_build_files[release_build_name]

    index = get_index(config.rosdistro_index_url)

    # get targets
    targets = []
    for os_name in sorted(build_file.targets.keys()):
        if os_name not in ['debian', 'ubuntu']:
            continue
        for os_code_name in sorted(build_file.targets[os_name].keys()):
            targets.append(Target(os_name, os_code_name, 'source'))
            for arch in sorted(build_file.targets[os_name][os_code_name]):
                targets.append(Target(os_name, os_code_name, arch))
    print('The build file contains the following targets:')
    for _, os_code_name, arch in targets:
        print('  - %s %s' % (os_code_name, arch))

    # get all input data
    dist = get_cached_distribution(index, rosdistro_name)

    rosdistro_info = get_rosdistro_info(dist, build_file)

    # derive testing and main urls from building url
    building_repo_url = build_file.target_repository
    base_url = os.path.dirname(building_repo_url)
    testing_repo_url = os.path.join(base_url, 'testing')
    main_repo_url = os.path.join(base_url, 'main')

    building_repo_data = get_debian_repo_data(
        building_repo_url, targets, cache_dir)
    testing_repo_data = get_debian_repo_data(
        testing_repo_url, targets, cache_dir)
    main_repo_data = get_debian_repo_data(main_repo_url, targets, cache_dir)

    repos_data = [building_repo_data, testing_repo_data, main_repo_data]

    # compute derived attributes
    package_descriptors = get_rosdistro_package_descriptors(
        rosdistro_info, rosdistro_name)

    affected_by_sync = get_affected_by_sync(
        package_descriptors, targets, testing_repo_data, main_repo_data)

    regressions = get_regressions(
        package_descriptors, targets,
        building_repo_data, testing_repo_data, main_repo_data)

    version_status = get_version_status(
        package_descriptors, targets, repos_data, strip_version=True)

    homogeneous = get_homogeneous(package_descriptors, targets, repos_data)

    package_counts = get_package_counts(
        package_descriptors, targets, repos_data)

    jenkins_job_urls = get_jenkins_job_urls(
        rosdistro_name, config.jenkins_url, release_build_name, targets)

    # generate output
    repo_urls = [building_repo_url, testing_repo_url, main_repo_url]
    repo_names = get_url_names(repo_urls)

    ordered_pkgs = []
    for pkg_name in sorted(rosdistro_info.keys()):
        ordered_pkgs.append(rosdistro_info[pkg_name])

    template_name = 'status/release_status_page.html.em'
    data = {
        'title': 'ROS packages for %s' % rosdistro_name.capitalize(),
        'start_time': start_time,
        'start_time_local_str': time.strftime('%Y-%m-%d %H:%M:%S %z', time.localtime(start_time)),

        'resource_hashes': get_resource_hashes(),

        'repo_names': repo_names,
        'repo_urls': repo_urls,

        'has_repository_column': True,
        'has_status_column': True,
        'has_maintainer_column': True,

        'ordered_pkgs': ordered_pkgs,
        'targets': targets,
        'short_arches': dict(
            [(t.arch, get_short_arch(t.arch)) for t in targets]),
        'repos_data': repos_data,

        'affected_by_sync': affected_by_sync,
        'homogeneous': homogeneous,
        'jenkins_job_urls': jenkins_job_urls,
        'package_counts': package_counts,
        'regressions': regressions,
        'version_status': version_status,
    }
    html = expand_template(template_name, data)
    output_filename = os.path.join(
        output_dir, 'ros_%s_%s.html' % (rosdistro_name, release_build_name))
    print("Generating status page '%s':" % output_filename)
    with open(output_filename, 'w') as h:
        h.write(html)

    additional_resources(output_dir, copy_resources=copy_resources)

def build_debian_repos_status_page(
        rosdistro_name, repo_urls, os_code_name_and_arch_tuples,
        cache_dir, output_name, output_dir):
    start_time = time.time()

    # get targets
    targets = []
    for os_code_name_and_arch in os_code_name_and_arch_tuples:
        assert os_code_name_and_arch.count(':') == 1, \
            'The string (%s) does not contain single colon separating an ' + \
            'OS code name and an architecture'
        os_code_name, arch = os_code_name_and_arch.split(':')
        targets.append(Target('ubuntu', os_code_name, arch))

    # get all input data
    repos_data = []
    for repo_url in repo_urls:
        repo_data = get_debian_repo_data(repo_url, targets, cache_dir)
        repos_data.append(repo_data)

    # compute derived attributes
    package_descriptors = get_repos_package_descriptors(repos_data, targets)

    version_status = get_version_status(
        package_descriptors, targets, repos_data, strip_os_code_name=True)

    homogeneous = get_homogeneous(package_descriptors, targets, repos_data)

    package_counts = get_package_counts(
        package_descriptors, targets, repos_data)

    # generate output
    repo_names = get_url_names(repo_urls)

    ordered_pkgs = []
    for debian_pkg_name in sorted(package_descriptors.keys()):
        pkg = RosPackage(debian_pkg_name)
        pkg.debian_name = debian_pkg_name
        pkg.version = package_descriptors[debian_pkg_name].version

        # set unavailable attributes
        pkg.repository_name = None
        pkg.repository_url = None
        pkg.status = None
        pkg.status_description = None
        pkg.maintainers = []
        pkg.url = None

        ordered_pkgs.append(pkg)

    template_name = 'status/release_status_page.html.em'
    data = {
        'title': 'All packages for %s targets' % rosdistro_name.capitalize(),
        'start_time': start_time,
        'start_time_local_str': time.strftime('%Y-%m-%d %H:%M:%S %z', time.localtime(start_time)),

        'resource_hashes': get_resource_hashes(),

        'repo_names': repo_names,
        'repo_urls': repo_urls,

        'has_repository_column': False,
        'has_status_column': False,
        'has_maintainer_column': False,

        'ordered_pkgs': ordered_pkgs,
        'targets': targets,
        'short_arches': dict(
            [(t.arch, get_short_arch(t.arch)) for t in targets]),
        'repos_data': repos_data,

        'affected_by_sync': None,
        'homogeneous': homogeneous,
        'jenkins_job_urls': None,
        'package_counts': package_counts,
        'regressions': None,
        'version_status': version_status,
    }
    html = expand_template(template_name, data)
    output_filename = os.path.join(
        output_dir, '%s.html' % output_name)
    print("Generating status page '%s':" % output_filename)
    with open(output_filename, 'w') as h:
        h.write(html)

    additional_resources(output_dir)

PackageDescriptor = namedtuple(
    'PackageDescriptor', 'pkg_name debian_pkg_name version')

def get_rosdistro_package_descriptors(rosdistro_info, rosdistro_name):
    descriptors = {}
    for pkg_name, pkg in rosdistro_info.items():
        debian_pkg_name = get_debian_package_name(rosdistro_name, pkg_name)
        descriptors[pkg_name] = PackageDescriptor(
            pkg_name, debian_pkg_name, pkg.version)
    return descriptors

def get_repos_package_descriptors(repos_data, targets):
    descriptors = {}
    # the highest version is the reference
    for target in targets:
        for repo_data in repos_data:
            repo_index = repo_data[target]
            for debian_pkg_name, version in repo_index.items():
                version = _strip_os_code_name_suffix(
                    version, target.os_code_name)
                if debian_pkg_name not in descriptors:
                    descriptors[debian_pkg_name] = PackageDescriptor(
                        debian_pkg_name, debian_pkg_name, version)
                    continue
                if not version:
                    continue
                other_version = descriptors[debian_pkg_name].version
                if not other_version:
                    continue
                # update version if higher
                if _version_is_gt_other(version, other_version):
                    descriptors[debian_pkg_name] = PackageDescriptor(
                        debian_pkg_name, debian_pkg_name, version)
    return descriptors

def get_url_names(urls):
    names = []
    for url in urls:
        basename = os.path.basename(url)
        if basename == 'ubuntu':
            basename = os.path.basename(os.path.dirname(url))
        names.append(basename)
    return names

def get_affected_by_sync(
        package_descriptors, targets,
        testing_repo_data, main_repo_data):
    """
    For each package and target check if it is affected by a sync.

    This is the case when the package version in the testing repo is different
    from the version in the main repo.

    :return: a dict indexed by package names containing
      dicts indexed by targets containing a boolean flag
    """
    affected_by_sync = {}
    for package_descriptor in package_descriptors.values():
        pkg_name = package_descriptor.pkg_name
        debian_pkg_name = package_descriptor.debian_pkg_name

        affected_by_sync[pkg_name] = {}
        for target in targets:
            testing_version = _strip_version_suffix(
                testing_repo_data.get(target, {}).get(debian_pkg_name, None))
            main_version = _strip_version_suffix(
                main_repo_data.get(target, {}).get(debian_pkg_name, None))

            affected_by_sync[pkg_name][target] = \
                testing_version != main_version
    return affected_by_sync

def get_regressions(
        package_descriptors, targets,
        building_repo_data, testing_repo_data, main_repo_data):
    """
    For each package and target check if it is a regression.

    This is the case if the main repo contains a package version which is
    higher then in any of the other repos or if any of the other repos does not
    contain that package at all.

    :return: a dict indexed by package names containing
      dicts indexed by targets containing a boolean flag
    """
    regressions = {}
    for package_descriptor in package_descriptors.values():
        pkg_name = package_descriptor.pkg_name
        debian_pkg_name = package_descriptor.debian_pkg_name

        regressions[pkg_name] = {}
        for target in targets:
            regressions[pkg_name][target] = False
            main_version = \
                main_repo_data.get(target, {}).get(debian_pkg_name, None)
            if main_version is not None:
                main_ver_loose = LooseVersion(main_version)
                for repo_data in [building_repo_data, testing_repo_data]:
                    version = \
                        repo_data.get(target, {}).get(debian_pkg_name, None)
                    if not version or main_ver_loose > LooseVersion(version):
                        regressions[pkg_name][target] = True
    return regressions

def get_version_status(
        package_descriptors, targets, repos_data,
        strip_version=False, strip_os_code_name=False):
    """
    For each package and target check if it is affected by a sync.

    This is the case when the package version in the testing repo is different
    from the version in the main repo.

    :return: a dict indexed by package names containing
      dicts indexed by targets containing
      a list of status strings (one for each repo)
    """
    status = {}
    for package_descriptor in package_descriptors.values():
        pkg_name = package_descriptor.pkg_name
        debian_pkg_name = package_descriptor.debian_pkg_name
        ref_version = package_descriptor.version
        if strip_version:
            ref_version = _strip_version_suffix(ref_version)

        status[pkg_name] = {}
        for target in targets:
            statuses = []
            for repo_data in repos_data:
                version = repo_data.get(target, {}).get(debian_pkg_name, None)
                if strip_version:
                    version = _strip_version_suffix(version)
                if strip_os_code_name:
                    version = _strip_os_code_name_suffix(
                        version, target.os_code_name)

                if ref_version:
                    if not version:
                        statuses.append('missing')
                    elif version.startswith(ref_version):  # including equal
                        statuses.append('equal')
                    else:
                        if _version_is_gt_other(version, ref_version):
                            statuses.append('higher')
                        else:
                            statuses.append('lower')
                else:
                    if not version:
                        statuses.append('ignore')
                    else:
                        statuses.append('obsolete')
            status[pkg_name][target] = statuses
    return status

version_regex = re.compile(r'[0-9.-]+[0-9]')

def _strip_version_suffix(version):
    """
    Remove trailing junk from the version number.

    >>> strip_version_suffix('')
    ''
    >>> strip_version_suffix('None')
    'None'
    >>> strip_version_suffix('1.2.3-4trusty-20140131-1359-+0000')
    '1.2.3-4'
    >>> strip_version_suffix('1.2.3-foo')
    '1.2.3'
    """
    global version_regex
    if not version:
        return version
    match = version_regex.search(version)
    return match.group(0) if match else version

def _strip_os_code_name_suffix(version, os_code_name):
    if version:
        index = version.find(os_code_name)
        if index != -1:
            version = version[:index]
    return version

def get_homogeneous(package_descriptors, targets, repos_data):
    """
    For each package check if the version in one repo is equal for all targets.

    The version could be different in different repos though.

    :return: a dict indexed by package names containing a boolean flag
    """
    homogeneous = {}
    for package_descriptor in package_descriptors.values():
        pkg_name = package_descriptor.pkg_name
        debian_pkg_name = package_descriptor.debian_pkg_name

        versions = []
        for repo_data in repos_data:
            versions.append(set([]))
            for target in targets:
                version = _strip_version_suffix(
                    repo_data.get(target, {}).get(debian_pkg_name, None))
                versions[-1].add(version)
        homogeneous[pkg_name] = max([len(v) for v in versions]) == 1
    return homogeneous

def get_package_counts(package_descriptors, targets, repos_data):
    """
    Get the number of packages per target and repository.

    :return: a dict indexed by targets containing
      a list of integer values (one for each repo)
    """
    counts = {}
    for target in targets:
        counts[target] = [0] * len(repos_data)
    for package_descriptor in package_descriptors.values():
        debian_pkg_name = package_descriptor.debian_pkg_name

        for target in targets:
            for i, repo_data in enumerate(repos_data):
                version = repo_data.get(target, {}).get(debian_pkg_name, None)
                if version:
                    counts[target][i] += 1
    return counts

def get_jenkins_job_urls(
        rosdistro_name, jenkins_url, release_build_name, targets):
    """
    Get the Jenkins job urls for each target.

    The placeholder {pkg} needs to be replaced with the ROS package name.

    :return: a dict indexed by targets containing a string
    """
    urls = {}
    for target in targets:
        view_name = get_release_view_name(
            rosdistro_name, release_build_name,
            target.os_name, target.os_code_name, target.arch)
        base_url = jenkins_url + '/view/%s/job/%s__{pkg}__' % \
            (view_name, view_name)
        if target.arch == 'source':
            urls[target] = base_url + '%s_%s__source' % \
                (target.os_name, target.os_code_name)
        else:
            urls[target] = base_url + '%s_%s_%s__binary' % \
                (target.os_name, target.os_code_name, target.arch)
    return urls

def additional_resources(output_dir, copy_resources=False):
    for subfolder in ['css', 'js']:
        dst = os.path.join(output_dir, subfolder)
        if not os.path.exists(dst):
            src = get_template_path(os.path.join('status', subfolder))
            if copy_resources:
                shutil.copytree(src, dst)
            else:
                os.symlink(os.path.abspath(src), dst)

def get_resource_hashes():
    hashes = {}
    for subfolder in ['css', 'js']:
        path = get_template_path(os.path.join('status', subfolder))
        for filename in os.listdir(path):
            if filename.endswith('.%s' % subfolder):
                with open(os.path.join(path, filename)) as f:
                    hashes[filename] = hash(tuple(f.read()))
    return hashes

def _version_is_gt_other(version, other_version):
    try:
        # might raise TypeError: http://bugs.python.org/issue14894
        return LooseVersion(version) > LooseVersion(other_version)
    except TypeError:
        loose_version, other_loose_version = \
            _get_comparable_loose_versions(version, other_version)
        return loose_version < other_loose_version

def _get_comparable_loose_versions(version_str1, version_str2):
    loose_version1 = LooseVersion(version_str1)
    loose_version2 = LooseVersion(version_str2)
    if sys.version_info[0] > 2:
        # might raise TypeError in Python 3: http://bugs.python.org/issue14894
        version_parts1 = loose_version1.version
        version_parts2 = loose_version2.version
        for i in range(min(len(version_parts1), len(version_parts2))):
            try:
                version_parts1[i] < version_parts2[i]
            except TypeError:
                version_parts1[i] = str(version_parts1[i])
                version_parts2[i] = str(version_parts2[i])
    return loose_version1, loose_version2

def build_blocked_releases_page(
    config_url, rosdistro_name,
    output_dir, repo_names=None, copy_resources=False
):
    start_time = time.localtime()

    repos_info = _get_blocked_releases_info(config_url, rosdistro_name, repo_names=repo_names)
    repos_data = [_format_repo_table_row(name, data) for name, data in sorted(repos_info.items())]

    template_name = 'status/blocked_releases_page.html.em'
    data = {
        'title': 'ROS %s - blocked releases' % rosdistro_name,
        'start_time': time.strftime('%Y-%m-%d %H:%M:%S %z', start_time),

        'resource_hashes': get_resource_hashes(),

        'rosdistro_name': rosdistro_name.capitalize(),

        'repos_data': repos_data,
    }
    html = expand_template(template_name, data)
    output_filename = os.path.join(
        output_dir, 'blocked_releases_%s.html' % rosdistro_name)
    print("Generating blocked releases page '%s':" % output_filename)
    with open(output_filename, 'w') as h:
        h.write(html)

    additional_resources(output_dir, copy_resources=copy_resources)

def _div_wrap(data, name=None):
    if_attr = ' id="%s"' % name if name is not None else ''
    return '<div{0}>{1}</div>'.format(if_attr, data)

def _filter_tag_wrap(label):
    return '<span class="ht">label="{0}"</span>'.format(label)

def _name_query_wrap(name):
    try:
        from urllib.parse import quote
    except ImportError:
        from urllib import quote
    query = 'id="{0}"'.format(name)
    return '<a href="?q={0}">{1}</a>'.format(quote(query), name)

def _format_repo_table_row(name, data):
    # convert data in dictionary into data that can be used as an html table row
    row = {}

    repos_blocking = data.get('repos_blocking', [])
    repos_blocked_by = data.get('repos_blocked_by', {})

    # tags for filtering
    tags = ''
    if data.get('released', False):
        tags += _filter_tag_wrap('RELEASED')
    else:
        tags += _filter_tag_wrap('UNRELEASED')

        if len(repos_blocked_by) > 0:
            tags += _filter_tag_wrap('BLOCKED')
        elif len(repos_blocking) > 0:
            tags += _filter_tag_wrap('UNBLOCKED')
            tags += _filter_tag_wrap('UNBLOCKED_BLOCKING')
        else:
            tags += _filter_tag_wrap('UNBLOCKED')
            tags += _filter_tag_wrap('UNBLOCKED_UNBLOCKING')

    url = data.get('url', None)
    if url:
        name_html = '<a href="{0}">{1}</a>'.format(url, name)
    else:
        name_html = name
    name_cell = name_html + tags

    # repo name cell
    row['name'] = _div_wrap(name_cell, name)

    # whether or not the repo has been released
    row['version'] = _div_wrap(data.get('version', 'Not released'))

    # num repos blocked by
    row['num_repos_blocked_by'] = _div_wrap(len(repos_blocked_by))

    # repos blocked by
    row['repos_blocked_by'] = _div_wrap('<br />'.join(
        _name_query_wrap(repo) for repo in sorted(repos_blocked_by.keys())))

    # maintainers info
    maintainers = data.get('maintainers', {})
    maintainers_cell = ''
    for repo_name in sorted(maintainers.keys()):
        maintainers_cell += repo_name + ':<br />'
        maintainers_cell += '<br />'.join(
            '<a href="mailto:{0}" style="padding-left: 1em">{1}</a>'
            .format(maintainers[repo_name][name], name)
            for name in sorted(maintainers[repo_name].keys()))
        maintainers_cell += '<br />'
    row['maintainers_of_repos_blocked_by'] = _div_wrap(maintainers_cell)

    # num repos blocking recursively
    row['num_repos_recursively_blocked'] = _div_wrap(
        len(data.get('recursive_repos_blocking', [])))

    # num repos blocking
    row['num_repos_blocked'] = _div_wrap(len(repos_blocking))

    # repos blocking by not being released
    row['repos_blocked'] = _div_wrap('<br />'.join(
        _name_query_wrap(repo) for repo in sorted(repos_blocking)))

    return row

def _is_released(repo, dist_file):
    return repo in dist_file.repositories and \
        dist_file.repositories[repo].release_repository is not None and \
        dist_file.repositories[repo].release_repository.version is not None

def _get_blocked_releases_info(config_url, rosdistro_name, repo_names=None):
    import rosdistro
    from rosdistro.dependency_walker import DependencyWalker
    from catkin_pkg.package import InvalidPackage, parse_package_string

    prev_rosdistro_name = None

    config = get_config_index(config_url)

    index = rosdistro.get_index(config.rosdistro_index_url)
    valid_rosdistro_names = list(index.distributions.keys())
    valid_rosdistro_names.sort()
    if rosdistro_name is None:
        rosdistro_name = valid_rosdistro_names[-1]
    print('Checking packages for "%s" distribution' % rosdistro_name)

    # Find the previous distribution to the current one
    try:
        i = valid_rosdistro_names.index(rosdistro_name)
    except ValueError:
        print('Distribution key not found in list of valid distributions.', file=sys.stderr)
        exit(-1)
    if i == 0:
        print('No previous distribution found.', file=sys.stderr)
        exit(-1)
    prev_rosdistro_name = valid_rosdistro_names[i - 1]

    cache = rosdistro.get_distribution_cache(index, rosdistro_name)
    distro_file = cache.distribution_file

    prev_cache = rosdistro.get_distribution_cache(index, prev_rosdistro_name)
    prev_distribution = rosdistro.get_cached_distribution(
        index, prev_rosdistro_name, cache=prev_cache)

    prev_distro_file = prev_cache.distribution_file

    dependency_walker = DependencyWalker(prev_distribution)

    if repo_names is None:
        # Check missing dependencies for packages that were in the previous
        # distribution that have not yet been released in the current distribution
        # Filter repos without a version or a release repository
        keys = prev_distro_file.repositories.keys()
        prev_repo_names = set(
            repo for repo in keys if _is_released(repo, prev_distro_file))
        repo_names = prev_repo_names
        ignored_inputs = []
    else:
        prev_repo_names = set(
            repo for repo in repo_names if _is_released(repo, prev_distro_file))
        ignored_inputs = list(set(repo_names).difference(prev_repo_names))
        if len(ignored_inputs) > 0:
            print(
                'Ignoring inputs for which repository info not found in previous distribution '
                '(did you list a package instead of a repository?):')
            print('\n'.join(
                sorted('\t{0}'.format(repo) for repo in ignored_inputs)))

    keys = distro_file.repositories.keys()
    current_repo_names = set(
        repo for repo in keys if _is_released(repo, distro_file))

    # Get a list of currently released packages
    current_package_names = set(
        pkg for repo in current_repo_names
        for pkg in distro_file.repositories[repo].release_repository.package_names)

    released_repos = prev_repo_names.intersection(
        current_repo_names)

    unreleased_repos = list(prev_repo_names.difference(
        current_repo_names))

    if len(unreleased_repos) == 0:
        print('All inputs already released in {0}.'.format(
            rosdistro_name))

    repos_info = defaultdict(dict)
    unprocessed_repos = prev_repo_names
    while unprocessed_repos:
        print('Processing repos:\n%s' %
              '\n'.join(['- %s' % r for r in sorted(unprocessed_repos)]))
        new_repos_to_process = set()  # set containing repos that come up while processing others

        for repo_name in unprocessed_repos:
            repos_info[repo_name]['released'] = repo_name in released_repos

            if repo_name in released_repos:
                repo = distro_file.repositories[repo_name]
                version = repo.release_repository.version
                repos_info[repo_name]['version'] = version

            else:
                # Gather info on which required repos have not been released yet
                # Assume dependencies will be the same as in the previous distribution and find
                # which ones have been released
                repo = prev_distro_file.repositories[repo_name]
                release_repo = repo.release_repository
                package_dependencies = set()
                packages = release_repo.package_names
                # Accumulate all dependencies for those packages
                for package in packages:
                    try:
                        package_dependencies |= dependency_walker.get_recursive_depends(
                            package, ['build', 'buildtool', 'run', 'test'], ros_packages_only=True,
                            limit_depth=1)
                    except AssertionError as e:
                        print(e, file=sys.stderr)

                # For all package dependencies, check if they are released yet
                unreleased_pkgs = package_dependencies.difference(
                    current_package_names)
                # Remove the packages which this repo provides
                unreleased_pkgs = unreleased_pkgs.difference(packages)

                # Get maintainer info and repo of unreleased packages
                maintainers = {}
                repos_blocked_by = set()
                for pkg_name in unreleased_pkgs:
                    unreleased_repo_name = \
                        prev_distro_file.release_packages[pkg_name].repository_name
                    repos_blocked_by.add(unreleased_repo_name)
                    pkg_xml = prev_distribution.get_release_package_xml(pkg_name)
                    if pkg_xml is not None:
                        try:
                            pkg = parse_package_string(pkg_xml)
                        except InvalidPackage:
                            pass
                        else:
                            pkg_maintainers = {m.name: m.email for m in pkg.maintainers}
                            if unreleased_repo_name not in maintainers:
                                maintainers[unreleased_repo_name] = {}
                            maintainers[unreleased_repo_name].update(pkg_maintainers)
                if maintainers:
                    repos_info[repo_name]['maintainers'] = maintainers

                repos_info[repo_name]['repos_blocked_by'] = {}
                for blocking_repo_name in repos_blocked_by:
                    # Get url of blocking repos
                    repo_url = None
                    blocking_repo = prev_distro_file.repositories[blocking_repo_name]
                    if blocking_repo.source_repository:
                        repo_url = blocking_repo.source_repository.url
                    elif blocking_repo.doc_repository:
                        repo_url = blocking_repo.doc_repository.url
                    repos_info[repo_name]['repos_blocked_by'].update(
                        {blocking_repo_name: repo_url})

                    # Mark blocking relationship in other direction
                    if blocking_repo_name not in repos_info:
                        new_repos_to_process.add(blocking_repo_name)
                        repos_info[blocking_repo_name] = {}
                    if 'repos_blocking' not in repos_info[blocking_repo_name]:
                        repos_info[blocking_repo_name]['repos_blocking'] = set([])
                    repos_info[blocking_repo_name]['repos_blocking'].add(repo_name)

            # Get url of repo
            repo_url = None
            if repo.source_repository:
                repo_url = repo.source_repository.url
            elif repo.doc_repository:
                repo_url = repo.doc_repository.url
            if repo_url:
                repos_info[repo_name]['url'] = repo_url

            new_repos_to_process.discard(repo_name)  # this repo has been fully processed now

        for repo_name in repos_info.keys():
            # Recursively get all repos being blocked by this repo
            recursive_blocks = set([])
            repos_to_check = set([repo_name])
            while repos_to_check:
                next_repo_to_check = repos_to_check.pop()
                blocks = repos_info[next_repo_to_check].get('repos_blocking', set([]))
                new_blocks = blocks - recursive_blocks
                repos_to_check |= new_blocks
                recursive_blocks |= new_blocks
            if recursive_blocks:
                repos_info[repo_name]['recursive_repos_blocking'] = recursive_blocks
        unprocessed_repos = new_repos_to_process

    return repos_info

def build_release_compare_page(
        config_url, rosdistro_names,
        output_dir, copy_resources=False):
    from rosdistro import get_cached_distribution
    from rosdistro import get_index

    start_time = time.time()

    config = get_config_index(config_url)

    index = get_index(config.rosdistro_index_url)

    # get all input data
    distros = [get_cached_distribution(index, d) for d in rosdistro_names]

    pkg_names = [d.release_packages.keys() for d in distros]
    pkg_names = [x for y in pkg_names for x in y]

    pkgs_data = {}
    for pkg_name in pkg_names:
        pkg_data = _compare_package_version(distros, pkg_name)
        if pkg_data:
            pkgs_data[pkg_name] = pkg_data

    template_name = 'status/release_compare_page.html.em'
    data = {
        'title': 'ROS packages in %s' % ' '.join([x.capitalize() for x in rosdistro_names]),

        'start_time': start_time,
        'start_time_local_str': time.strftime('%Y-%m-%d %H:%M:%S %z', time.localtime(start_time)),

        'resource_hashes': get_resource_hashes(),

        'rosdistro_names': rosdistro_names,

        'pkgs_data': pkgs_data,
    }
    html = expand_template(template_name, data)
    output_filename = os.path.join(
        output_dir, 'compare_%s.html' % '_'.join(rosdistro_names))
    print("Generating compare page: '%s'" % output_filename)
    with open(output_filename, 'w') as h:
        h.write(html)

    additional_resources(output_dir, copy_resources=copy_resources)

class CompareRow(object):

    def __init__(self, pkg_name):
        self.pkg_name = pkg_name
        self.repo_name = ''
        self.repo_urls = []
        self.maintainers = {}
        self.versions = []
        self.branches = []

    def get_repo_name_with_link(self):
        valid_urls = [u for u in self.repo_urls if u]
        if len(set(valid_urls)) == 1:
            return '<a href="%s">%s</a>' % (valid_urls[0], self.repo_name)

        unique_urls = []
        [unique_urls.append(u) for u in valid_urls if u not in unique_urls]
        parts = [self.repo_name]
        for i, repo_url in enumerate(unique_urls):
            parts.append(' [<a href="%s">%d</a>]' % (repo_url, i + 1))
        return ' '.join(parts)

    def get_maintainers(self):
        return ' '.join([self.maintainers[k] for k in sorted(self.maintainers.keys())])

    def get_labels(self, distros):
        all_versions = [LooseVersion(v) if v else v for v in self.versions]
        valid_versions = [v for v in all_versions if v]
        labels = []
        if any([
            _is_only_patch_is_different(p[0], p[1])
            for p in itertools.combinations(valid_versions, 2)]
        ):
            labels.append('DIFF_PATCH')
        if any([_is_greater(p[0], p[1]) for p in itertools.combinations(valid_versions, 2)]):
            labels.append('DOWNGRADE_VERSION')

        versions_and_branches = zip(
            itertools.combinations(all_versions, 2), itertools.combinations(self.branches, 2))
        if any([
            _is_same_version_but_different_branch(vb[0][0], vb[0][1], vb[1][0], vb[1][1])
            for vb in versions_and_branches
        ]):
            labels.append('DIFF_BRANCH_SAME_VERSION')
        return labels

def _is_only_patch_is_different(a, b):
    return a.version[0] == b.version[0] and \
        a.version[1] == b.version[1] and a.version[2] != b.version[2]

def _is_greater(a, b):
    return a.version[0] > b.version[0] or \
        (a.version[0] == b.version[0] and a.version[1] > b.version[1])

def _is_same_version_but_different_branch(version_a, version_b, branch_a, branch_b):
    # skip when any version is unknown
    if not version_a or not version_b:
        return False
    # skip when any branch is unknown or they are equal
    if not branch_a or not branch_b or branch_a == branch_b:
        return False
    return version_a.version[0] == version_b.version[0] and \
        version_a.version[1] == version_b.version[1]

def _compare_package_version(distros, pkg_name):
    from catkin_pkg.package import InvalidPackage, parse_package_string
    row = CompareRow(pkg_name)
    for distro in distros:
        repo_url = None
        version = None
        branch = None
        if pkg_name in distro.release_packages:
            pkg = distro.release_packages[pkg_name]
            row.repo_name = pkg.repository_name
            repo = distro.repositories[pkg.repository_name]

            rel_repo = repo.release_repository
            if rel_repo:
                version = rel_repo.version
                pkg_xml = distro.get_release_package_xml(pkg_name)
                if pkg_xml is not None:
                    try:
                        pkg = parse_package_string(pkg_xml)
                        for m in pkg.maintainers:
                            row.maintainers[m.name] = '<a href="mailto:%s">%s</a>' % \
                                (m.email, m.name)
                    except InvalidPackage:
                        row.maintainers['zzz'] = '<b>invalid package.xml in %s</b>' % \
                            distro.name

                if repo.source_repository:
                    repo_url = repo.source_repository.url
                elif repo.doc_repository:
                    repo_url = repo.doc_repository.url

            source_repo = repo.source_repository
            if source_repo:
                branch = source_repo.version
            else:
                doc_repo = repo.source_repository
                if doc_repo:
                    branch = doc_repo.version

        row.repo_urls.append(repo_url)
        row.versions.append(version)
        row.branches.append(branch)

    # skip if no versions available
    if not [v for v in row.versions if v]:
        return None

    data = [row.pkg_name, row.get_repo_name_with_link(), row.get_maintainers()] + \
        [v if v else '' for v in row.versions]

    labels = row.get_labels(distros)
    if len(labels) > 0:
        data[1] += ' <span class="ht">%s</span>' % ' '.join(labels)

    # div-wrap all cells for layout reasons
    for i, value in enumerate(data):
        data[i] = '<div>%s</div>' % value

    return data


text:
import {CompileDirectiveMetadata, CompileDirectiveSummary, CompileIdentifierMetadata, CompileNgModuleMetadata, CompileNgModuleSummary, CompilePipeMetadata, CompileProviderMetadata, CompileStylesheetMetadata, CompileSummaryKind, CompileTypeMetadata, CompileTypeSummary, componentFactoryName, createHostComponentMeta, flatten, identifierName, sourceUrl, templateSourceUrl} from '../compile_metadata';
import {CompilerConfig} from '../config';
import {Identifiers, createTokenForExternalReference} from '../identifiers';
import {CompileMetadataResolver} from '../metadata_resolver';
import {NgModuleCompiler} from '../ng_module_compiler';
import {OutputEmitter} from '../output/abstract_emitter';
import * as o from '../output/output_ast';
import {CompiledStylesheet, StyleCompiler} from '../style_compiler';
import {SummaryResolver} from '../summary_resolver';
import {TemplateParser} from '../template_parser/template_parser';
import {OutputContext, syntaxError} from '../util';
import {ViewCompileResult, ViewCompiler} from '../view_compiler/view_compiler';

import {AotCompilerHost} from './compiler_host';
import {GeneratedFile} from './generated_file';
import {StaticReflector} from './static_reflector';
import {StaticSymbol} from './static_symbol';
import {ResolvedStaticSymbol, StaticSymbolResolver} from './static_symbol_resolver';
import {createForJitStub, serializeSummaries} from './summary_serializer';
import {ngfactoryFilePath, splitTypescriptSuffix, summaryFileName, summaryForJitFileName, summaryForJitName} from './util';

export class AotCompiler {
  constructor(
      private _config: CompilerConfig, private _host: AotCompilerHost,
      private _reflector: StaticReflector, private _metadataResolver: CompileMetadataResolver,
      private _templateParser: TemplateParser, private _styleCompiler: StyleCompiler,
      private _viewCompiler: ViewCompiler, private _ngModuleCompiler: NgModuleCompiler,
      private _outputEmitter: OutputEmitter,
      private _summaryResolver: SummaryResolver<StaticSymbol>, private _localeId: string|null,
      private _translationFormat: string|null, private _enableSummariesForJit: boolean|null,
      private _symbolResolver: StaticSymbolResolver) {}

  clearCache() { this._metadataResolver.clearCache(); }

  analyzeModulesSync(rootFiles: string[]): NgAnalyzedModules {
    const programSymbols = extractProgramSymbols(this._symbolResolver, rootFiles, this._host);
    const analyzeResult =
        analyzeAndValidateNgModules(programSymbols, this._host, this._metadataResolver);
    analyzeResult.ngModules.forEach(
        ngModule => this._metadataResolver.loadNgModuleDirectiveAndPipeMetadata(
            ngModule.type.reference, true));
    return analyzeResult;
  }

  analyzeModulesAsync(rootFiles: string[]): Promise<NgAnalyzedModules> {
    const programSymbols = extractProgramSymbols(this._symbolResolver, rootFiles, this._host);
    const analyzeResult =
        analyzeAndValidateNgModules(programSymbols, this._host, this._metadataResolver);
    return Promise
        .all(analyzeResult.ngModules.map(
            ngModule => this._metadataResolver.loadNgModuleDirectiveAndPipeMetadata(
                ngModule.type.reference, false)))
        .then(() => analyzeResult);
  }

  emitAllStubs(analyzeResult: NgAnalyzedModules): GeneratedFile[] {
    const {files} = analyzeResult;
    const sourceModules =
        files.map(file => this._compileStubFile(file.srcUrl, file.directives, file.ngModules));
    return flatten(sourceModules);
  }

  emitAllImpls(analyzeResult: NgAnalyzedModules): GeneratedFile[] {
    const {ngModuleByPipeOrDirective, files} = analyzeResult;
    const sourceModules = files.map(
        file => this._compileImplFile(
            file.srcUrl, ngModuleByPipeOrDirective, file.directives, file.pipes, file.ngModules,
            file.injectables));
    return flatten(sourceModules);
  }

  private _compileStubFile(
      srcFileUrl: string, directives: StaticSymbol[], ngModules: StaticSymbol[]): GeneratedFile[] {
    const fileSuffix = splitTypescriptSuffix(srcFileUrl, true)[1];
    const generatedFiles: GeneratedFile[] = [];

    const jitSummaryStmts: o.Statement[] = [];
    const ngFactoryStms: o.Statement[] = [];

    const ngFactoryOutputCtx = this._createOutputContext(ngfactoryFilePath(srcFileUrl, true));
    const jitSummaryOutputCtx = this._createOutputContext(summaryForJitFileName(srcFileUrl, true));

    // create exports that user code can reference
    ngModules.forEach((ngModuleReference) => {
      this._ngModuleCompiler.createStub(ngFactoryOutputCtx, ngModuleReference);
      createForJitStub(jitSummaryOutputCtx, ngModuleReference);
    });
    // Note: we are creating stub ngfactory/ngsummary for all source files,
    // as the real calculation requires almost the same logic as producing the real content for
    // them.
    // Our pipeline will filter out empty ones at the end.
    generatedFiles.push(this._codegenSourceModule(srcFileUrl, ngFactoryOutputCtx));
    generatedFiles.push(this._codegenSourceModule(srcFileUrl, jitSummaryOutputCtx));

    // create stubs for external stylesheets (always empty, as users should not import anything from
    // the generated code)
    directives.forEach((dirType) => {
      const compMeta = this._metadataResolver.getDirectiveMetadata(<any>dirType);
      if (!compMeta.isComponent) {
        return;
      }
      // Note: compMeta is a component and therefore template is non null.
      compMeta.template !.externalStylesheets.forEach((stylesheetMeta) => {
        generatedFiles.push(this._codegenSourceModule(
            stylesheetMeta.moduleUrl !,
            this._createOutputContext(_stylesModuleUrl(
                stylesheetMeta.moduleUrl !, this._styleCompiler.needsStyleShim(compMeta),
                fileSuffix))));
      });
    });
    return generatedFiles;
  }

  private _compileImplFile(
      srcFileUrl: string, ngModuleByPipeOrDirective: Map<StaticSymbol, CompileNgModuleMetadata>,
      directives: StaticSymbol[], pipes: StaticSymbol[], ngModules: StaticSymbol[],
      injectables: StaticSymbol[]): GeneratedFile[] {
    const fileSuffix = splitTypescriptSuffix(srcFileUrl, true)[1];
    const generatedFiles: GeneratedFile[] = [];

    const outputCtx = this._createOutputContext(ngfactoryFilePath(srcFileUrl, true));

    generatedFiles.push(
        ...this._createSummary(srcFileUrl, directives, pipes, ngModules, injectables, outputCtx));

    // compile all ng modules
    ngModules.forEach((ngModuleType) => this._compileModule(outputCtx, ngModuleType));

    // compile components
    directives.forEach((dirType) => {
      const compMeta = this._metadataResolver.getDirectiveMetadata(<any>dirType);
      if (!compMeta.isComponent) {
        return;
      }
      const ngModule = ngModuleByPipeOrDirective.get(dirType);
      if (!ngModule) {
        throw new Error(
            `Internal Error: cannot determine the module for component ${identifierName(compMeta.type)}!`);
      }

      // compile styles
      const componentStylesheet = this._styleCompiler.compileComponent(outputCtx, compMeta);
      // Note: compMeta is a component and therefore template is non null.
      compMeta.template !.externalStylesheets.forEach((stylesheetMeta) => {
        generatedFiles.push(
            this._codegenStyles(stylesheetMeta.moduleUrl !, compMeta, stylesheetMeta, fileSuffix));
      });

      // compile components
      const compViewVars = this._compileComponent(
          outputCtx, compMeta, ngModule, ngModule.transitiveModule.directives, componentStylesheet,
          fileSuffix);
      this._compileComponentFactory(outputCtx, compMeta, ngModule, fileSuffix);
    });
    if (outputCtx.statements.length > 0) {
      const srcModule = this._codegenSourceModule(srcFileUrl, outputCtx);
      generatedFiles.unshift(srcModule);
    }
    return generatedFiles;
  }

  private _createSummary(
      srcFileUrl: string, directives: StaticSymbol[], pipes: StaticSymbol[],
      ngModules: StaticSymbol[], injectables: StaticSymbol[],
      ngFactoryCtx: OutputContext): GeneratedFile[] {
    const symbolSummaries = this._symbolResolver.getSymbolsOf(srcFileUrl)
                                .map(symbol => this._symbolResolver.resolveSymbol(symbol));
    const typeData: {
      summary: CompileTypeSummary,
      metadata: CompileNgModuleMetadata | CompileDirectiveMetadata | CompilePipeMetadata |
          CompileTypeMetadata
    }[] =
        [
          ...ngModules.map(ref => ({
                             summary: this._metadataResolver.getNgModuleSummary(ref) !,
                             metadata: this._metadataResolver.getNgModuleMetadata(ref) !
                           })),
          ...directives.map(ref => ({
                              summary: this._metadataResolver.getDirectiveSummary(ref) !,
                              metadata: this._metadataResolver.getDirectiveMetadata(ref) !
                            })),
          ...pipes.map(ref => ({
                         summary: this._metadataResolver.getPipeSummary(ref) !,
                         metadata: this._metadataResolver.getPipeMetadata(ref) !
                       })),
          ...injectables.map(ref => ({
                               summary: this._metadataResolver.getInjectableSummary(ref) !,
                               metadata: this._metadataResolver.getInjectableSummary(ref) !.type
                             }))
        ];
    const forJitOutputCtx = this._createOutputContext(summaryForJitFileName(srcFileUrl, true));
    const {json, exportAs} = serializeSummaries(
        forJitOutputCtx, this._summaryResolver, this._symbolResolver, symbolSummaries, typeData);
    exportAs.forEach((entry) => {
      ngFactoryCtx.statements.push(
          o.variable(entry.exportAs).set(ngFactoryCtx.importExpr(entry.symbol)).toDeclStmt(null, [
            o.StmtModifier.Exported
          ]));
    });
    const summaryJson = new GeneratedFile(srcFileUrl, summaryFileName(srcFileUrl), json);
    if (this._enableSummariesForJit) {
      return [summaryJson, this._codegenSourceModule(srcFileUrl, forJitOutputCtx)];
    };

    return [summaryJson];
  }

  private _compileModule(outputCtx: OutputContext, ngModuleType: StaticSymbol): void {
    const ngModule = this._metadataResolver.getNgModuleMetadata(ngModuleType) !;
    const providers: CompileProviderMetadata[] = [];

    if (this._localeId) {
      providers.push({
        token: createTokenForExternalReference(this._reflector, Identifiers.LOCALE_ID),
        useValue: this._localeId,
      });
    }

    if (this._translationFormat) {
      providers.push({
        token: createTokenForExternalReference(this._reflector, Identifiers.TRANSLATIONS_FORMAT),
        useValue: this._translationFormat
      });
    }

    this._ngModuleCompiler.compile(outputCtx, ngModule, providers);
  }

  private _compileComponentFactory(
      outputCtx: OutputContext, compMeta: CompileDirectiveMetadata,
      ngModule: CompileNgModuleMetadata, fileSuffix: string): void {
    const hostType = this._metadataResolver.getHostComponentType(compMeta.type.reference);
    const hostMeta = createHostComponentMeta(
        hostType, compMeta, this._metadataResolver.getHostComponentViewClass(hostType));
    const hostViewFactoryVar =
        this._compileComponent(outputCtx, hostMeta, ngModule, [compMeta.type], null, fileSuffix)
            .viewClassVar;
    const compFactoryVar = componentFactoryName(compMeta.type.reference);
    const inputsExprs: o.LiteralMapEntry[] = [];
    for (let propName in compMeta.inputs) {
      const templateName = compMeta.inputs[propName];
      // Don't quote so that the key gets minified...
      inputsExprs.push(new o.LiteralMapEntry(propName, o.literal(templateName), false));
    }
    const outputsExprs: o.LiteralMapEntry[] = [];
    for (let propName in compMeta.outputs) {
      const templateName = compMeta.outputs[propName];
      // Don't quote so that the key gets minified...
      outputsExprs.push(new o.LiteralMapEntry(propName, o.literal(templateName), false));
    }

    outputCtx.statements.push(
        o.variable(compFactoryVar)
            .set(o.importExpr(Identifiers.createComponentFactory).callFn([
              o.literal(compMeta.selector), outputCtx.importExpr(compMeta.type.reference),
              o.variable(hostViewFactoryVar), new o.LiteralMapExpr(inputsExprs),
              new o.LiteralMapExpr(outputsExprs),
              o.literalArr(
                  compMeta.template !.ngContentSelectors.map(selector => o.literal(selector)))
            ]))
            .toDeclStmt(
                o.importType(
                    Identifiers.ComponentFactory,
                    [o.expressionType(outputCtx.importExpr(compMeta.type.reference)) !],
                    [o.TypeModifier.Const]),
                [o.StmtModifier.Final, o.StmtModifier.Exported]));
  }

  private _compileComponent(
      outputCtx: OutputContext, compMeta: CompileDirectiveMetadata,
      ngModule: CompileNgModuleMetadata, directiveIdentifiers: CompileIdentifierMetadata[],
      componentStyles: CompiledStylesheet|null, fileSuffix: string): ViewCompileResult {
    const directives =
        directiveIdentifiers.map(dir => this._metadataResolver.getDirectiveSummary(dir.reference));
    const pipes = ngModule.transitiveModule.pipes.map(
        pipe => this._metadataResolver.getPipeSummary(pipe.reference));

    const {template: parsedTemplate, pipes: usedPipes} = this._templateParser.parse(
        compMeta, compMeta.template !.template !, directives, pipes, ngModule.schemas,
        templateSourceUrl(ngModule.type, compMeta, compMeta.template !));
    const stylesExpr = componentStyles ? o.variable(componentStyles.stylesVar) : o.literalArr([]);
    const viewResult = this._viewCompiler.compileComponent(
        outputCtx, compMeta, parsedTemplate, stylesExpr, usedPipes);
    if (componentStyles) {
      _resolveStyleStatements(
          this._symbolResolver, componentStyles, this._styleCompiler.needsStyleShim(compMeta),
          fileSuffix);
    }
    return viewResult;
  }

  private _createOutputContext(genFilePath: string): OutputContext {
    const importExpr = (symbol: StaticSymbol, typeParams: o.Type[] | null = null) => {
      if (!(symbol instanceof StaticSymbol)) {
        throw new Error(`Internal error: unknown identifier ${JSON.stringify(symbol)}`);
      }
      const arity = this._symbolResolver.getTypeArity(symbol) || 0;
      const {filePath, name, members} = this._symbolResolver.getImportAs(symbol) || symbol;
      const moduleName = this._symbolResolver.fileNameToModuleName(filePath, genFilePath);
      // If we are in a type expression that refers to a generic type then supply
      // the required type parameters. If there were not enough type parameters
      // supplied, supply any as the type. Outside a type expression the reference
      // should not supply type parameters and be treated as a simple value reference
      // to the constructor function itself.
      const suppliedTypeParams = typeParams || [];
      const missingTypeParamsCount = arity - suppliedTypeParams.length;
      const allTypeParams =
          suppliedTypeParams.concat(new Array(missingTypeParamsCount).fill(o.DYNAMIC_TYPE));
      return members.reduce(
          (expr, memberName) => expr.prop(memberName),
          <o.Expression>o.importExpr(
              new o.ExternalReference(moduleName, name, null), allTypeParams));
    };

    return {statements: [], genFilePath, importExpr};
  }

  private _codegenStyles(
      srcFileUrl: string, compMeta: CompileDirectiveMetadata,
      stylesheetMetadata: CompileStylesheetMetadata, fileSuffix: string): GeneratedFile {
    const outputCtx = this._createOutputContext(_stylesModuleUrl(
        stylesheetMetadata.moduleUrl !, this._styleCompiler.needsStyleShim(compMeta), fileSuffix));
    const compiledStylesheet =
        this._styleCompiler.compileStyles(outputCtx, compMeta, stylesheetMetadata);
    _resolveStyleStatements(
        this._symbolResolver, compiledStylesheet, this._styleCompiler.needsStyleShim(compMeta),
        fileSuffix);
    return this._codegenSourceModule(srcFileUrl, outputCtx);
  }

  private _codegenSourceModule(srcFileUrl: string, ctx: OutputContext): GeneratedFile {
    return new GeneratedFile(srcFileUrl, ctx.genFilePath, ctx.statements);
  }
}

function _resolveStyleStatements(
    symbolResolver: StaticSymbolResolver, compileResult: CompiledStylesheet, needsShim: boolean,
    fileSuffix: string): void {
  compileResult.dependencies.forEach((dep) => {
    dep.setValue(symbolResolver.getStaticSymbol(
        _stylesModuleUrl(dep.moduleUrl, needsShim, fileSuffix), dep.name));
  });
}

function _stylesModuleUrl(stylesheetUrl: string, shim: boolean, suffix: string): string {
  return `${stylesheetUrl}${shim ? '.shim' : ''}.ngstyle${suffix}`;
}

export interface NgAnalyzedModules {
  ngModules: CompileNgModuleMetadata[];
  ngModuleByPipeOrDirective: Map<StaticSymbol, CompileNgModuleMetadata>;
  files: Array<{
    srcUrl: string,
    directives: StaticSymbol[],
    pipes: StaticSymbol[],
    ngModules: StaticSymbol[],
    injectables: StaticSymbol[]
  }>;
  symbolsMissingModule?: StaticSymbol[];
}

export interface NgAnalyzeModulesHost { isSourceFile(filePath: string): boolean; }

// Returns all the source files and a mapping from modules to directives
export function analyzeNgModules(
    programStaticSymbols: StaticSymbol[], host: NgAnalyzeModulesHost,
    metadataResolver: CompileMetadataResolver): NgAnalyzedModules {
  const {ngModules, symbolsMissingModule} =
      _createNgModules(programStaticSymbols, host, metadataResolver);
  return _analyzeNgModules(programStaticSymbols, ngModules, symbolsMissingModule, metadataResolver);
}

export function analyzeAndValidateNgModules(
    programStaticSymbols: StaticSymbol[], host: NgAnalyzeModulesHost,
    metadataResolver: CompileMetadataResolver): NgAnalyzedModules {
  const result = analyzeNgModules(programStaticSymbols, host, metadataResolver);
  if (result.symbolsMissingModule && result.symbolsMissingModule.length) {
    const messages = result.symbolsMissingModule.map(
        s =>
            `Cannot determine the module for class ${s.name} in ${s.filePath}! Add ${s.name} to the NgModule to fix it.`);
    throw syntaxError(messages.join('\n'));
  }
  return result;
}

function _analyzeNgModules(
    programSymbols: StaticSymbol[], ngModuleMetas: CompileNgModuleMetadata[],
    symbolsMissingModule: StaticSymbol[],
    metadataResolver: CompileMetadataResolver): NgAnalyzedModules {
  const moduleMetasByRef = new Map<any, CompileNgModuleMetadata>();
  ngModuleMetas.forEach((ngModule) => moduleMetasByRef.set(ngModule.type.reference, ngModule));
  const ngModuleByPipeOrDirective = new Map<StaticSymbol, CompileNgModuleMetadata>();
  const ngModulesByFile = new Map<string, StaticSymbol[]>();
  const ngDirectivesByFile = new Map<string, StaticSymbol[]>();
  const ngPipesByFile = new Map<string, StaticSymbol[]>();
  const ngInjectablesByFile = new Map<string, StaticSymbol[]>();
  const filePaths = new Set<string>();

  // Make sure we produce an analyzed file for each input file
  programSymbols.forEach((symbol) => {
    const filePath = symbol.filePath;
    filePaths.add(filePath);
    if (metadataResolver.isInjectable(symbol)) {
      ngInjectablesByFile.set(filePath, (ngInjectablesByFile.get(filePath) || []).concat(symbol));
    }
  });

  // Looping over all modules to construct:
  // - a map from file to modules `ngModulesByFile`,
  // - a map from file to directives `ngDirectivesByFile`,
  // - a map from file to pipes `ngPipesByFile`,
  // - a map from directive/pipe to module `ngModuleByPipeOrDirective`.
  ngModuleMetas.forEach((ngModuleMeta) => {
    const srcFileUrl = ngModuleMeta.type.reference.filePath;
    filePaths.add(srcFileUrl);
    ngModulesByFile.set(
        srcFileUrl, (ngModulesByFile.get(srcFileUrl) || []).concat(ngModuleMeta.type.reference));

    ngModuleMeta.declaredDirectives.forEach((dirIdentifier) => {
      const fileUrl = dirIdentifier.reference.filePath;
      filePaths.add(fileUrl);
      ngDirectivesByFile.set(
          fileUrl, (ngDirectivesByFile.get(fileUrl) || []).concat(dirIdentifier.reference));
      ngModuleByPipeOrDirective.set(dirIdentifier.reference, ngModuleMeta);
    });
    ngModuleMeta.declaredPipes.forEach((pipeIdentifier) => {
      const fileUrl = pipeIdentifier.reference.filePath;
      filePaths.add(fileUrl);
      ngPipesByFile.set(
          fileUrl, (ngPipesByFile.get(fileUrl) || []).concat(pipeIdentifier.reference));
      ngModuleByPipeOrDirective.set(pipeIdentifier.reference, ngModuleMeta);
    });
  });

  const files: {
    srcUrl: string,
    directives: StaticSymbol[],
    pipes: StaticSymbol[],
    ngModules: StaticSymbol[],
    injectables: StaticSymbol[]
  }[] = [];

  filePaths.forEach((srcUrl) => {
    const directives = ngDirectivesByFile.get(srcUrl) || [];
    const pipes = ngPipesByFile.get(srcUrl) || [];
    const ngModules = ngModulesByFile.get(srcUrl) || [];
    const injectables = ngInjectablesByFile.get(srcUrl) || [];
    files.push({srcUrl, directives, pipes, ngModules, injectables});
  });

  return {
    // map directive/pipe to module
    ngModuleByPipeOrDirective,
    // list modules and directives for every source file
    files,
    ngModules: ngModuleMetas, symbolsMissingModule
  };
}

export function extractProgramSymbols(
    staticSymbolResolver: StaticSymbolResolver, files: string[],
    host: NgAnalyzeModulesHost): StaticSymbol[] {
  const staticSymbols: StaticSymbol[] = [];
  files.filter(fileName => host.isSourceFile(fileName)).forEach(sourceFile => {
    staticSymbolResolver.getSymbolsOf(sourceFile).forEach((symbol) => {
      const resolvedSymbol = staticSymbolResolver.resolveSymbol(symbol);
      const symbolMeta = resolvedSymbol.metadata;
      if (symbolMeta) {
        if (symbolMeta.__symbolic != 'error') {
          // Ignore symbols that are only included to record error information.
          staticSymbols.push(resolvedSymbol.symbol);
        }
      }
    });
  });

  return staticSymbols;
}

// Load the NgModules and check
// that all directives / pipes that are present in the program
// are also declared by a module.
function _createNgModules(
    programStaticSymbols: StaticSymbol[], host: NgAnalyzeModulesHost,
    metadataResolver: CompileMetadataResolver):
    {ngModules: CompileNgModuleMetadata[], symbolsMissingModule: StaticSymbol[]} {
  const ngModules = new Map<any, CompileNgModuleMetadata>();
  const programPipesAndDirectives: StaticSymbol[] = [];
  const ngModulePipesAndDirective = new Set<StaticSymbol>();

  const addNgModule = (staticSymbol: any) => {
    if (ngModules.has(staticSymbol) || !host.isSourceFile(staticSymbol.filePath)) {
      return false;
    }
    const ngModule = metadataResolver.getNgModuleMetadata(staticSymbol, false);
    if (ngModule) {
      ngModules.set(ngModule.type.reference, ngModule);
      ngModule.declaredDirectives.forEach((dir) => ngModulePipesAndDirective.add(dir.reference));
      ngModule.declaredPipes.forEach((pipe) => ngModulePipesAndDirective.add(pipe.reference));
      // For every input module add the list of transitively included modules
      ngModule.transitiveModule.modules.forEach(modMeta => addNgModule(modMeta.reference));
    }
    return !!ngModule;
  };
  programStaticSymbols.forEach((staticSymbol) => {
    if (!addNgModule(staticSymbol) &&
        (metadataResolver.isDirective(staticSymbol) || metadataResolver.isPipe(staticSymbol))) {
      programPipesAndDirectives.push(staticSymbol);
    }
  });

  // Throw an error if any of the program pipe or directives is not declared by a module
  const symbolsMissingModule =
      programPipesAndDirectives.filter(s => !ngModulePipesAndDirective.has(s));

  return {ngModules: Array.from(ngModules.values()), symbolsMissingModule};
}

redpajama_set_name: RedPajamaGithub


text:
describe Email::Provider::GMail do
  describe "#host" do
    it "returns host" do
      expect(subject.host).to eq("imap.gmail.com")
    end
  end
end
# rubocop:enable RSpec/FilePath


text:
package com.gsma.android.oneapi.discovery;

import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.util.Calendar;
import java.util.HashMap;

import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.params.ClientPNames;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.conn.ssl.X509HostnameVerifier;
import org.apache.http.params.HttpParams;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;

import com.gsma.android.utils.CookieManagement;
import com.gsma.android.utils.HttpUtils;
import com.gsma.android.utils.JsonUtils;

import android.content.Context;
import android.content.Intent;
import android.net.Uri;
import android.os.AsyncTask;
import android.util.Log;

/**
 * This is a background task which makes an initial connection to the discovery
 * service - it will handle a variety of initial response types. Extends AsyncTask<Void, Void, JSONObject>
 */
public class DiscoveryTask extends AsyncTask<Void, Void, JSONObject> {
	private static final String TAG = "DiscoveryTask";

	String serviceUri;
	String consumerKey;
	String consumerSecret;
	String sourceIP;
	boolean usingMobileData;
	String msisdn;
	String mcc;
	String mnc;
	DiscoveryCallbackReceiver getJSONListener;
	String credentials;
	String redirectUri;
	Context context;
	boolean followRedirect;

	/**
	 * Standard constructor
	 *
	 * @param serviceUri
	 * @param consumerKey
	 * @param consumerSecret
	 * @param usingMobileData
	 * @param msisdn
	 * @param mcc
	 * @param mnc
	 * @param getJSONListener
	 * @param credentials (none, plain, sha256)
	 * @param redirectUri
	 * @param context
	 * @param followRedirect
	 */
	public DiscoveryTask(String serviceUri, String consumerKey,
			String consumerSecret, String sourceIP, boolean usingMobileData,
			String msisdn, String mcc, String mnc,
			DiscoveryCallbackReceiver getJSONListener, String credentials, String redirectUri, Context context, boolean followRedirect) {
		Log.d(TAG, "Instantiated DiscoveryTask");
		this.serviceUri = serviceUri;
		this.consumerKey = consumerKey;
		this.consumerSecret = consumerSecret;
		this.sourceIP = sourceIP;
		this.usingMobileData = usingMobileData;
		this.msisdn = msisdn;
		this.mcc = mcc;
		this.mnc = mnc;
		this.getJSONListener = getJSONListener;
		this.credentials = credentials;
		this.redirectUri = redirectUri;
		this.context = context;
		this.followRedirect = followRedirect;
	}

	/**
	 * The doInBackground function does the actual background processing.
	 * @see android.os.AsyncTask#doInBackground(Params[])
	 *
	 * @param params
	 * @return JSONObject
	 * @throws UnsupportedEncodingException
	 * @throws ClientProtocolException
	 * @throws IOException
	 * @throws JSONException
	 */
	@Override
	protected JSONObject doInBackground(Void... params) {
//		android.os.Debug.waitForDebugger();

		JSONObject response = null;
		String cookie = null;
		String phase1Uri=serviceUri;

		/*
		 * sets up the HTTP request with a redirect_uri parameter - in practice
		 * we're looking for mcc/mnc added to the redirect_uri if this step is
		 * necessary
		 */
		if(redirectUri!=null && redirectUri.trim().length() > 0){
			phase1Uri = phase1Uri + "?redirect_uri="+redirectUri;//http://gsma.com/oneapi
			if (mcc != null && mcc.trim().length() > 0 && mnc != null && mnc.trim().length() > 0) {
				phase1Uri = phase1Uri + "&mcc_mnc=" + mcc + "_" + mnc;
			} else if ((cookie = CookieManagement.getCookie("mcc_mnc")) != null) {
				phase1Uri = phase1Uri + "&mcc_mnc=" + cookie;
			}
		}else if (mcc != null && mcc.trim().length() > 0 && mnc != null && mnc.trim().length() > 0) {
			phase1Uri = phase1Uri + "?mcc_mnc=" + mcc + "_" + mnc;
		} else if ((cookie = CookieManagement.getCookie("mcc_mnc")) != null) {
			phase1Uri = phase1Uri + "?mcc_mnc=" + cookie;
		}

		if(msisdn!=null && msisdn.trim().length() > 0) {
			if (phase1Uri.indexOf("?")== -1) {
				phase1Uri = phase1Uri + "?msisdn=" + msisdn;
			} else {
				phase1Uri = phase1Uri + "&msisdn=" + msisdn;
			}
		}

		Log.d(TAG, "Started discovery process via " + phase1Uri);

		HttpGet httpRequest = new HttpGet(phase1Uri);

		if (usingMobileData || (mcc != null && mcc.trim().length() > 0 && mnc != null && mnc.trim().length() > 0 && (msisdn == null || msisdn.trim().length() <= 0)) || cookie != null) {
			httpRequest.addHeader("Accept", "application/json");
			Log.d(TAG, "Add application/json");
		} else {
			httpRequest.addHeader("Accept", "text/html");// msisdn and redirect cases
			Log.d(TAG, "Add text/html");
		}

		if (sourceIP != null) {
			httpRequest.addHeader("x-source-ip", sourceIP);
			Log.d(TAG, "Add x-source-ip " + sourceIP);
		}

		try {
			HostnameVerifier hostnameVerifier = org.apache.http.conn.ssl.SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER;
			SchemeRegistry registry = new SchemeRegistry();
			SSLSocketFactory socketFactory = SSLSocketFactory.getSocketFactory();
			socketFactory.setHostnameVerifier((X509HostnameVerifier) hostnameVerifier);
			registry.register(new Scheme("https", socketFactory, 443));
			HttpsURLConnection.setDefaultHostnameVerifier(hostnameVerifier);

			HttpClient httpClient = HttpUtils.getHttpAuthorizationClient(phase1Uri, consumerKey, consumerSecret, credentials, httpRequest);
			HttpParams httpParams = httpRequest.getParams();
			httpParams.setParameter(ClientPNames.HANDLE_REDIRECTS,Boolean.FALSE);
			httpRequest.setParams(httpParams);

			Log.d(TAG, "Add consumerKey and consumerSecret; " + consumerKey
					+ " - " + consumerSecret);

			Log.d(TAG, "Making " + httpRequest.getMethod() + " request to " + httpRequest.getURI());

			Calendar calendar = Calendar.getInstance();
			calendar.add(Calendar.DATE, 1);
			if (mcc != null && mcc.trim().length() > 0 && mnc != null && mnc.trim().length() > 0)
				CookieManagement.addCookieExpireOneDay("mcc_mnc", mcc + "_" + mnc);
			CookieManagement.updateCookieStore(httpClient);
			HttpResponse httpResponse = httpClient.execute(httpRequest);

			Log.d(TAG, "Request executed and completed with status=" + httpResponse.getStatusLine().getStatusCode());

			HashMap<String, String> headerMap = HttpUtils.getHeaders(httpResponse);

			Log.d(TAG, "headerMap =" + headerMap);

			String contentType = headerMap.get("content-type");
			String location = headerMap.get("location");

			int statusCode = httpResponse.getStatusLine().getStatusCode();

			Log.d(TAG, "status=" + statusCode + " CT=" + contentType + " Loc="
					+ location + " JSON?" + HttpUtils.isJSON(contentType)
					+ " HTML?" + HttpUtils.isHTML(contentType));

			if (statusCode == HttpStatus.SC_OK) {

				if (HttpUtils.isJSON(contentType)) {

					HttpEntity httpEntity = httpResponse.getEntity();
					InputStream is = httpEntity.getContent();
					Log.d(TAG, "Converting discovery data OK (JSON)");
					response = JsonUtils.readJSON(is);

				} else if (HttpUtils.isHTML(contentType)) {
					Log.d(TAG,"Have OK HTML content - needs to be handled through the browser");
					// msisdn
				}

			} else if (statusCode == HttpStatus.SC_MOVED_TEMPORARILY && location != null) {
				Log.d(TAG, "302 response " + location);

				try {
					Thread.sleep(1000);
				} catch (InterruptedException ie) {
				}

				if (location.indexOf("mcc_mnc") > -1) {
					String[] parts = location.split("mcc_mnc", 2);
					if (parts.length == 2) {
						String mcc_mnc = parts[1].replaceFirst("=", "").trim();
						Log.d(TAG, "To try again with mcc_mnc = " + mcc_mnc);
						response = ProcessDiscoveryToken.start(mcc_mnc,consumerKey, serviceUri);
					}
				} else {
					Log.d(TAG, "Redirect requested to " + location);
					if(followRedirect){
						Intent intent = new Intent(Intent.ACTION_VIEW);
						intent.setData(Uri.parse(location));
						String title = "Choose a browser";
						Intent chooser = Intent.createChooser(intent, title);
						context.startActivity(chooser.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK));
					}
					response = new JSONObject();
					response.put("Location", location);
				}
			} else if (statusCode == HttpStatus.SC_ACCEPTED) {

				Log.d(TAG, "202 response ");

				HttpEntity httpEntity = httpResponse.getEntity();
				InputStream is = httpEntity.getContent();
				JSONObject jo = JsonUtils.readJSON(is);
				Link[] link = null;
				String href = "";
				if (jo != null) {
					JSONArray linkArray = jo.getJSONArray("links");
					if (linkArray != null) {
						link = new Link[linkArray.length()];
						for (int i = 0; i < linkArray.length(); i++) {
							link[i] = new Link(linkArray.getJSONObject(i));
						}
					}
				}
				if (link[0] != null)
					href = link[0].getHref();

				try {
					Thread.sleep(1000);
				} catch (InterruptedException ie) {
				}

				if (href.indexOf("mcc_mnc") > -1) {
					String[] parts = href.split("mcc_mnc", 2);
					if (parts.length == 2) {
						String mcc_mnc = parts[1].replaceFirst("=", "").trim();
						Log.d(TAG, "To try again with mcc_mnc = " + mcc_mnc);
						response = ProcessDiscoveryToken.start(mcc_mnc, consumerKey, serviceUri);
					}
				} else {
					Log.d(TAG, "Redirect requested to " + href);
					if(followRedirect){
						Intent intent = new Intent(Intent.ACTION_VIEW);
						intent.setData(Uri.parse(href));
						String title = "Choose a browser";
						Intent chooser = Intent.createChooser(intent, title);
						context.startActivity(chooser.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK));
					}
					response = new JSONObject();
					response.put("operatorSelection", href);
				}

			} else if (statusCode >= HttpStatus.SC_BAD_REQUEST) {
				Log.d(TAG, "Bad request response " + statusCode);

				HttpEntity httpEntity = httpResponse.getEntity();
				InputStream is = httpEntity.getContent();
				String contents = HttpUtils.getContentsFromInputStream(is);

				if (HttpUtils.isJSON(contentType)) {
					Log.d(TAG, "Bad request response " + contentType);
					Object rawJSON = JsonUtils.convertContent(contents, contentType);
					if (rawJSON != null && rawJSON instanceof JSONObject) {
						response = (JSONObject) rawJSON;
					}
				} else {
					Log.d(TAG, "Bad request response " + contentType);
					response = JsonUtils.simpleError("HTTP " + statusCode, "HTTP " + statusCode);
				}

			}

		} catch (UnsupportedEncodingException e) {
			Log.d(TAG, "UnsupportedEncodingException=" + e.getMessage());
			response = JsonUtils.simpleError("UnsupportedEncodingException",
					"UnsupportedEncodingException - " + e.getMessage());
		} catch (ClientProtocolException e) {
			Log.d(TAG, "ClientProtocolException=" + e.getMessage());
			response = JsonUtils.simpleError("ClientProtocolException",
					"ClientProtocolException - " + e.getMessage());
		} catch (IOException e) {
			Log.d(TAG, "IOException=" + e.getMessage());
			response = JsonUtils.simpleError("IOException", "IOException - "
					+ e.getMessage());
		} catch (JSONException e) {
			Log.d(TAG, "JSONException=" + e.getMessage());
			response = JsonUtils.simpleError("JSONException",
					"JSONException - " + e.getMessage());
		}

		return response;
	}

	/**
	 * On completion of this background task either this task has started the
	 * next part of the process or an error has occurred.
	 * @see android.os.AsyncTask#onPostExecute(java.lang.Object)
	 *
	 * @param response
	 */
	@Override
	protected void onPostExecute(JSONObject response) {
//		android.os.Debug.waitForDebugger();
		Log.d(TAG, "onPostExecute for " + response);
		getJSONListener.receiveDiscoveryData(response);
	}

}


text:
'''
Automatically create jenkins jobs for the branches in a svn repository.
Documentation: https://github.com/gvalkov/jenkins-autojobs/
'''

from __future__ import absolute_import

import os
import sys
import subprocess

from . import job, main, utils

def svn_ls(url, username=None, password=None, dirsonly=True):
    cmd = ['svn', 'ls', '--trust-server-cert', '--non-interactive']

    # :todo: plaintext (will probably have to use the bindings).
    if username:
        cmd += ['--username', username]
    if password:
        cmd += ['--password', password]

    cmd.append(url)
    out = utils.check_output(cmd).decode('utf8').split(os.linesep)

    if dirsonly:
        out = [i.rstrip('/') for i in out if i.endswith('/')]

    return out

def svn_wildcard_ls(url, repo, username=None, password=None, dirsonly=True):
    left, wcard, right = url.partition('*')
    right = right.lstrip('/')
    branches = []

    if not wcard:
        try:
            return svn_ls(left, username, password, True)
        except subprocess.CalledProcessError:
            return []
    else:
        for dirname in svn_ls(left, username, password, True):
            url = os.path.join(left, dirname, right)
            res = svn_wildcard_ls(url, repo, username, password, True)
            rel = url.replace(repo, '')
            branches.extend(os.path.join(rel, i) for i in res)
        return branches

def list_branches(config):
    c = config
    branches = []

    for url in c['branches']:
        if '*' in url:
            res = svn_wildcard_ls(url, c['repo'], c['scm-username'], c['scm-password'])
            res = [i.lstrip('/') for i in res]
        else:
            res = svn_ls(url, c['scm-username'], c['scm-password'])
            rel = url.replace(c['repo'], '').lstrip('/')
            res = [os.path.join(rel, i) for i in res]
        branches.extend(res)
    return branches

def create_job(branch, template, config, branch_config):
    '''Create a jenkins job.
       :param branch: svn branch name (ex: branches/feature-one)
       :param template: the config of the template job to use
       :param config: global config (parsed yaml)
       :param branch_config: the effective config for this branch
    '''

    print('\nprocessing branch: %s' % branch)

    # Job names with '/' in them are problematic.
    sanitized_branch = branch.replace('/', branch_config['namesep'])

    match = branch_config['re'].match(branch)
    groups, groupdict = match.groups(), match.groupdict()

    # Placeholders available to the 'substitute' and 'namefmt' options.
    fmtdict = {
        'branch': branch.split('/')[-1],
        'path': branch.replace('/', branch_config['namesep']),
        'repo': utils.sanitize(config['repo'], branch_config['sanitize']),
        'path-orig': branch,
        'repo-orig': config['repo'],
    }

    job_name = branch_config['namefmt'].format(*groups, **utils.merge(groupdict, fmtdict))
    job_obj = job.Job(job_name, branch, template, main.jenkins)

    fmtdict['job_name'] = job_name

    print('. job name: %s' % job_obj.name)
    print('. job exists: %s' % job_obj.exists)

    try:
        scm_el = job_obj.xml.xpath('scm[@class="hudson.scm.SubversionSCM"]')[0]
    except IndexError:
        msg = 'Template job %s is not configured to use SVN as an SCM'
        raise RuntimeError(msg % template)  # :bug:

    # set branch
    el = scm_el.xpath('//remote')[0]
    el.text = os.path.join(config['repo'], branch)

    # Set the branch that git plugin will locally checkout to.
    el = scm_el.xpath('//local')[0]
    el.text = '.'

    # Set the state of the newly created job.
    job_obj.set_state(branch_config['enable'])

    # Since some plugins (such as sidebar links) can't interpolate the
    # job name, we do it for them.
    job_obj.substitute(list(branch_config['substitute'].items()), fmtdict, groups, groupdict)

    job_obj.create(
        overwrite=branch_config['overwrite'],
        build_on_create=branch_config['build-on-create'],
        dryrun=config['dryrun'],
        tag=branch_config['tag'],
        tag_method=config['tag-method']
    )

    if config['debug']:
        main.debug_refconfig(branch_config)
    return job_name

def _main(argv=sys.argv, config=None):
    main.main(argv[1:], config=config, create_job=create_job, list_branches=list_branches)

if __name__ == '__main__':
    _main()


text:
<?xml version="1.0" encoding="ISO-8859-1"?>
<geo:GeodesyML xsi:schemaLocation="urn:xml-gov-au:icsm:egeodesy:0.3" xmlns:geo="urn:xml-gov-au:icsm:egeodesy:0.3" xmlns:gml="http://www.opengis.net/gml/3.2" xmlns:ns9="http://www.w3.org/1999/xlink" xmlns:gmd="http://www.isotc211.org/2005/gmd" xmlns:gmx="http://www.isotc211.org/2005/gmx" xmlns:om="http://www.opengis.net/om/2.0" xmlns:gco="http://www.isotc211.org/2005/gco" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" gml:id="GeodesyMLType_242">
   <geo:siteLog gml:id="SiteLogType_242">
      <geo:formInformation>
         <geo:preparedBy>Lands Department, Hong Kong</geo:preparedBy>
         <geo:datePrepared>2012-03-07T13:00:00Z</geo:datePrepared>
         <geo:reportType>UPDATE</geo:reportType>
      </geo:formInformation>
      <geo:siteIdentification>
         <geo:siteName>Ngong Ping</geo:siteName>
         <geo:fourCharacterID>HKNP</geo:fourCharacterID>
         <geo:iersDOMESNumber>HKM000067</geo:iersDOMESNumber>
         <geo:cdpNumber></geo:cdpNumber>
         <geo:monumentDescription>PILLAR</geo:monumentDescription>
         <geo:monumentFoundation>CONCRETE BLOCK</geo:monumentFoundation>
         <geo:markerDescription>CONCRETE PILLAR</geo:markerDescription>
         <geo:dateInstalled>2004-07-31T14:00:00Z</geo:dateInstalled>
         <geo:notes>Pillar</geo:notes>
      </geo:siteIdentification>
      <geo:siteLocation>
         <geo:city>Ngong Ping</geo:city>
         <geo:state>Hong Kong</geo:state>
         <geo:countryCodeISO>CHN</geo:countryCodeISO>
         <geo:tectonicPlate codeSpace="">Eurasian</geo:tectonicPlate>
         <geo:approximatePositionITRF>
            <geo:xCoordinateInMeters>-2392360.654</geo:xCoordinateInMeters>
            <geo:yCoordinateInMeters>5400226.132</geo:yCoordinateInMeters>
            <geo:zCoordinateInMeters>2400094.305</geo:zCoordinateInMeters>
            <geo:latitude-North>22.24906449166666533301395247690379619598388671875</geo:latitude-North>
            <geo:longitude-East>113.8938789888889147050576866604387760162353515625</geo:longitude-East>
            <geo:elevation-m_ellips.>354.007 (m)</geo:elevation-m_ellips.>
         </geo:approximatePositionITRF>
      </geo:siteLocation>
      <geo:gnssReceivers>
         <geo:gnssReceiver gml:id="GnssReceiverType_1056">
            <geo:manufacturerSerialNumber>455678</geo:manufacturerSerialNumber>
            <geo:receiverType codeList="codelist" codeListValue="codeListValue" codeSpace="DummyCodeSpace">LEICA GRX1200+GNSS</geo:receiverType>
            <geo:satelliteSystem>GPS+GLONASS</geo:satelliteSystem>
            <geo:serialNumber>455678</geo:serialNumber>
            <geo:firmwareVersion>8.20/4.007</geo:firmwareVersion>
            <geo:elevationCutoffSetting>0.0</geo:elevationCutoffSetting>
            <geo:dateInstalled>2011-11-18T00:00:00Z</geo:dateInstalled>
            <geo:dateRemoved></geo:dateRemoved>
            <geo:temperatureStabilization>0.0</geo:temperatureStabilization>
         </geo:gnssReceiver>
         <geo:dateInserted>2011-11-18T00:00:00Z</geo:dateInserted>
      </geo:gnssReceivers>
      <geo:gnssReceivers>
         <geo:gnssReceiver gml:id="GnssReceiverType_1057">
            <geo:manufacturerSerialNumber>455685</geo:manufacturerSerialNumber>
            <geo:receiverType codeList="codelist" codeListValue="codeListValue" codeSpace="DummyCodeSpace">LEICA GRX1200PRO</geo:receiverType>
            <geo:satelliteSystem>GPS+GLONASS</geo:satelliteSystem>
            <geo:serialNumber>455685</geo:serialNumber>
            <geo:firmwareVersion>8.10/2.125</geo:firmwareVersion>
            <geo:elevationCutoffSetting>0.0</geo:elevationCutoffSetting>
            <geo:dateInstalled>2011-06-16T00:00:00Z</geo:dateInstalled>
            <geo:dateRemoved>2011-09-05T23:59:00Z</geo:dateRemoved>
            <geo:temperatureStabilization>0.0</geo:temperatureStabilization>
         </geo:gnssReceiver>
         <geo:dateInserted>2011-06-16T00:00:00Z</geo:dateInserted>
      </geo:gnssReceivers>
      <geo:gnssReceivers>
         <geo:gnssReceiver gml:id="GnssReceiverType_1058">
            <geo:manufacturerSerialNumber>450234</geo:manufacturerSerialNumber>
            <geo:receiverType codeList="codelist" codeListValue="codeListValue" codeSpace="DummyCodeSpace">LEICA GRX1200PRO</geo:receiverType>
            <geo:satelliteSystem>GPS+GLONASS</geo:satelliteSystem>
            <geo:serialNumber>450234</geo:serialNumber>
            <geo:firmwareVersion>7.80/2.125</geo:firmwareVersion>
            <geo:elevationCutoffSetting>0.0</geo:elevationCutoffSetting>
            <geo:dateInstalled>2004-08-01T00:00:00Z</geo:dateInstalled>
            <geo:dateRemoved>2011-06-15T23:59:00Z</geo:dateRemoved>
            <geo:temperatureStabilization>0.0</geo:temperatureStabilization>
         </geo:gnssReceiver>
         <geo:dateInserted>2004-08-01T00:00:00Z</geo:dateInserted>
      </geo:gnssReceivers>
      <geo:gnssReceivers>
         <geo:gnssReceiver gml:id="GnssReceiverType_1059">
            <geo:manufacturerSerialNumber>39519</geo:manufacturerSerialNumber>
            <geo:receiverType codeList="codelist" codeListValue="codeListValue" codeSpace="DummyCodeSpace">LEICA SR530</geo:receiverType>
            <geo:satelliteSystem>GPS+GLONASS</geo:satelliteSystem>
            <geo:serialNumber>39519</geo:serialNumber>
            <geo:firmwareVersion>5.10/1.39</geo:firmwareVersion>
            <geo:elevationCutoffSetting>0.0</geo:elevationCutoffSetting>
            <geo:dateInstalled>2011-09-06T00:00:00Z</geo:dateInstalled>
            <geo:dateRemoved>2011-11-18T00:00:00Z</geo:dateRemoved>
            <geo:temperatureStabilization>0.0</geo:temperatureStabilization>
         </geo:gnssReceiver>
         <geo:dateInserted>2011-09-06T00:00:00Z</geo:dateInserted>
      </geo:gnssReceivers>
      <geo:gnssAntennas>
         <geo:gnssAntenna gml:id="GnssAntennaType_482">
            <geo:manufacturerSerialNumber>SMO_Unknown4</geo:manufacturerSerialNumber>
            <geo:antennaType codeList="codelist" codeListValue="codeListValue" codeSpace="codespace">LEIAT504        LEIS</geo:antennaType>
            <geo:serialNumber>SMO_Unknown4</geo:serialNumber>
            <geo:antennaReferencePoint codeSpace="">BPA</geo:antennaReferencePoint>
            <geo:marker-arpUpEcc.>0.0083</geo:marker-arpUpEcc.>
            <geo:marker-arpNorthEcc.>0.0</geo:marker-arpNorthEcc.>
            <geo:marker-arpEastEcc.>0.0</geo:marker-arpEastEcc.>
            <geo:alignmentFromTrueNorth>0.0</geo:alignmentFromTrueNorth>
            <geo:antennaRadomeType codeSpace="CodeSpaceAttr">LEIS</geo:antennaRadomeType>
            <geo:radomeSerialNumber></geo:radomeSerialNumber>
            <geo:antennaCableType></geo:antennaCableType>
            <geo:antennaCableLength>0.0</geo:antennaCableLength>
            <geo:dateInstalled>2004-08-01T00:00:00Z</geo:dateInstalled>
            <geo:dateRemoved>2011-11-18T00:00:00Z</geo:dateRemoved>
         </geo:gnssAntenna>
         <geo:dateInserted>2004-08-01T00:00:00Z</geo:dateInserted>
      </geo:gnssAntennas>
      <geo:gnssAntennas>
         <geo:gnssAntenna gml:id="GnssAntennaType_483">
            <geo:manufacturerSerialNumber>725201</geo:manufacturerSerialNumber>
            <geo:antennaType codeList="codelist" codeListValue="codeListValue" codeSpace="codespace">LEIAR25.R4      LEIT</geo:antennaType>
            <geo:serialNumber>725201</geo:serialNumber>
            <geo:antennaReferencePoint codeSpace="">BPA</geo:antennaReferencePoint>
            <geo:marker-arpUpEcc.>0.0083</geo:marker-arpUpEcc.>
            <geo:marker-arpNorthEcc.>0.0</geo:marker-arpNorthEcc.>
            <geo:marker-arpEastEcc.>0.0</geo:marker-arpEastEcc.>
            <geo:alignmentFromTrueNorth>0.0</geo:alignmentFromTrueNorth>
            <geo:antennaRadomeType codeSpace="CodeSpaceAttr">LEIT</geo:antennaRadomeType>
            <geo:radomeSerialNumber></geo:radomeSerialNumber>
            <geo:antennaCableType></geo:antennaCableType>
            <geo:antennaCableLength>0.0</geo:antennaCableLength>
            <geo:dateInstalled>2011-11-18T00:00:00Z</geo:dateInstalled>
            <geo:dateRemoved></geo:dateRemoved>
         </geo:gnssAntenna>
         <geo:dateInserted>2011-11-18T00:00:00Z</geo:dateInserted>
      </geo:gnssAntennas>
      <geo:frequencyStandards>
         <geo:frequencyStandard gml:id="FrequencyStandardType_384">
            <geo:standardType codeSpace="">QUARTZ/INTERNAL</geo:standardType>
            <geo:inputFrequency>0.0</geo:inputFrequency>
            <gml:validTime nilReason="">
               <gml:TimePeriod gml:id="TimePeriodType_562">
                  <gml:beginPosition>2004-08-01T00:00:00Z</gml:beginPosition>
                  <gml:endPosition></gml:endPosition>
               </gml:TimePeriod>
            </gml:validTime>
         </geo:frequencyStandard>
         <geo:dateInserted>2016-04-14T00:03:09Z</geo:dateInserted>
      </geo:frequencyStandards>
      <geo:siteContact gml:id="AgencyPropertyType_484">
         <gmd:MD_SecurityConstraints id="MDSecurityConstraintsType_484">
            <gmd:classification>
               <gmd:MD_ClassificationCode codeList="codelist" codeListValue="codeListValue"/>
            </gmd:classification>
         </gmd:MD_SecurityConstraints>
         <gmd:CI_ResponsibleParty id="CIResponsiblePartyType_484">
            <gmd:contactInfo>
               <gmd:CI_Contact id="CIContactType_484">
                  <gmd:address>
                     <gmd:CI_Address id="CIAddressType_484"/>
                  </gmd:address>
               </gmd:CI_Contact>
            </gmd:contactInfo>
            <gmd:role>
               <gmd:CI_RoleCode codeList="codelist" codeListValue="codeListValue"/>
            </gmd:role>
         </gmd:CI_ResponsibleParty>
      </geo:siteContact>
      <geo:siteMetadataCustodian gml:id="AgencyPropertyType_485">
         <gmd:MD_SecurityConstraints id="MDSecurityConstraintsType_485">
            <gmd:classification>
               <gmd:MD_ClassificationCode codeList="codelist" codeListValue="codeListValue"/>
            </gmd:classification>
         </gmd:MD_SecurityConstraints>
         <gmd:CI_ResponsibleParty id="CIResponsiblePartyType_485">
            <gmd:organisationName>
               <gco:CharacterString>Lands Department, HKSAR</gco:CharacterString>
            </gmd:organisationName>
            <gmd:contactInfo>
               <gmd:CI_Contact id="CIContactType_485">
                  <gmd:address>
                     <gmd:CI_Address id="CIAddressType_485">
                        <gmd:deliveryPoint>
                           <gco:CharacterString>333 Java Road, North Point, Hong Kong</gco:CharacterString>
                        </gmd:deliveryPoint>
                     </gmd:CI_Address>
                  </gmd:address>
               </gmd:CI_Contact>
            </gmd:contactInfo>
            <gmd:role>
               <gmd:CI_RoleCode codeList="codelist" codeListValue="codeListValue"/>
            </gmd:role>
         </gmd:CI_ResponsibleParty>
      </geo:siteMetadataCustodian>
      <geo:moreInformation>
         <geo:dataCenter></geo:dataCenter>
         <geo:urlForMoreInformation></geo:urlForMoreInformation>
         <geo:siteMap>Y</geo:siteMap>
         <geo:siteDiagram>Y</geo:siteDiagram>
         <geo:horizonMask>Y</geo:horizonMask>
         <geo:monumentDescription></geo:monumentDescription>
         <geo:sitePictures>Y</geo:sitePictures>
         <geo:antennaGraphicsWithDimensions></geo:antennaGraphicsWithDimensions>
         <geo:insertTextGraphicFromAntenna></geo:insertTextGraphicFromAntenna>
         <geo:DOI codeSpace="CodeSpaceTODO">CodeTypeTODO</geo:DOI>
      </geo:moreInformation>
   </geo:siteLog>
</geo:GeodesyML>


text:
.btn {
    position: relative;
    display: inline-block;
    padding: 6px 12px;
    font-size: 14px;
    font-weight: 600;
    line-height: 20px;
    color: #333;
    white-space: nowrap;
    vertical-align: middle;
    cursor: pointer;
    -webkit-user-select: none;
    -moz-user-select: none;
    -ms-user-select: none;
    user-select: none;
    background-color: #eee;
    background-image: -webkit-linear-gradient(#fcfcfc, #eee);
    background-image: linear-gradient(#fcfcfc, #eee);
    border: 1px solid #d5d5d5;
    border-radius: 3px;
    -webkit-appearance: none;
    -moz-appearance: none;
    appearance: none
}
.btn i {
    font-style: normal;
    font-weight: 500;
    opacity: 0.6
}
.btn .octicon {
    vertical-align: text-top
}
.btn .counter {
    text-shadow: none;
    background-color: #e5e5e5
}
.btn:focus {
    text-decoration: none;
    border-color: #51a7e8;
    outline: none;
    box-shadow: 0 0 5px rgba(81, 167, 232, 0.5)
}
.btn:focus:hover,
.btn.selected:focus {
    border-color: #51a7e8
}
.btn:hover,
.btn:active,
.btn.zeroclipboard-is-hover,
.btn.zeroclipboard-is-active {
    text-decoration: none;
    background-color: #ddd;
    background-image: -webkit-linear-gradient(#eee, #ddd);
    background-image: linear-gradient(#eee, #ddd);
    border-color: #ccc
}
.btn:active,
.btn.selected,
.btn.zeroclipboard-is-active {
    background-color: #dcdcdc;
    background-image: none;
    border-color: #b5b5b5;
    box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15)
}
.btn.selected:hover {
    background-color: #cfcfcf
}
.btn:disabled,
.btn:disabled:hover,
.btn.disabled,
.btn.disabled:hover {
    color: rgba(102, 102, 102, 0.5);
    cursor: default;
    background-color: rgba(229, 229, 229, 0.5);
    background-image: none;
    border-color: rgba(197, 197, 197, 0.5);
    box-shadow: none
}
.btn-primary {
    color: #fff;
    text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.15);
    background-color: #60b044;
    background-image: -webkit-linear-gradient(#8add6d, #60b044);
    background-image: linear-gradient(#8add6d, #60b044);
    border-color: #5ca941
}
.btn-primary .counter {
    color: #60b044;
    background-color: #fff
}
.btn-primary:hover {
    color: #fff;
    background-color: #569e3d;
    background-image: -webkit-linear-gradient(#79d858, #569e3d);
    background-image: linear-gradient(#79d858, #569e3d);
    border-color: #4a993e
}
.btn-primary:active,
.btn-primary.selected {
    text-shadow: 0 1px 0 rgba(0, 0, 0, 0.15);
    background-color: #569e3d;
    background-image: none;
    border-color: #418737
}
.btn-primary.selected:hover {
    background-color: #4c8b36
}
.btn-primary:disabled,
.btn-primary:disabled:hover,
.btn-primary.disabled,
.btn-primary.disabled:hover {
    color: #fefefe;
    background-color: #add39f;
    background-image: -webkit-linear-gradient(#c3ecb4, #add39f);
    background-image: linear-gradient(#c3ecb4, #add39f);
    border-color: #b9dcac #b9dcac #a7c89b
}
.btn-danger {
    color: #900
}
.btn-danger:hover {
    color: #fff;
    background-color: #b33630;
    background-image: -webkit-linear-gradient(#dc5f59, #b33630);
    background-image: linear-gradient(#dc5f59, #b33630);
    border-color: #cd504a
}
.btn-danger:active,
.btn-danger.selected {
    color: #fff;
    background-color: #b33630;
    background-image: none;
    border-color: #9f312c
}
.btn-danger.selected:hover {
    background-color: #9f302b
}
.btn-danger:disabled,
.btn-danger:disabled:hover,
.btn-danger.disabled,
.btn-danger.disabled:hover {
    color: #cb7f7f;
    background-color: #efefef;
    background-image: -webkit-linear-gradient(#fefefe, #efefef);
    background-image: linear-gradient(#fefefe, #efefef);
    border-color: #e1e1e1
}
.btn-danger:hover .counter,
.btn-danger:active .counter,
.btn-danger.selected .counter {
    color: #b33630;
    background-color: #fff
}
.btn-outline {
    color: #4078c0;
    background-color: #fff;
    background-image: none;
    border: 1px solid #e5e5e5
}
.btn-outline .counter {
    background-color: #eee
}
.btn-outline:hover,
.btn-outline:active,
.btn-outline.selected,
.btn-outline.zeroclipboard-is-hover,
.btn-outline.zeroclipboard-is-active {
    color: #fff;
    background-color: #4078c0;
    background-image: none;
    border-color: #4078c0
}
.btn-outline:hover .counter,
.btn-outline:active .counter,
.btn-outline.selected .counter,
.btn-outline.zeroclipboard-is-hover .counter,
.btn-outline.zeroclipboard-is-active .counter {
    color: #4078c0;
    background-color: #fff
}
.btn-outline.selected:hover {
    background-color: #396cad
}
.btn-outline:disabled,
.btn-outline:disabled:hover,
.btn-outline.disabled,
.btn-outline.disabled:hover {
    color: #767676;
    background-color: #fff;
    background-image: none;
    border-color: #e5e5e5
}
.btn-with-count {
    float: left;
    border-top-right-radius: 0;
    border-bottom-right-radius: 0
}
.btn-sm {
    padding: 3px 10px;
    font-size: 12px;
    line-height: 20px
}


text:
var ezZoomContextMenu = {
	title : "EZ ZOOM",
	create : function() {
		this.destory();
		chrome.contextMenus.create({
			"title" : this.title,
			"contexts" : ["page", "selection", "link", "editable", "image", "video", "audio"],
			"onclick" : function(info, tab) {
				chrome.tabs.sendMessage(tab.id, {
					method : "zoomTarget"
				}, function(response) {
					// do nothing
				});
			}
		});
	},
	destory : function() {
		chrome.contextMenus.removeAll();
	}
};


text:
import mock

from sahara import context
from sahara.tests.unit import base
from sahara.utils.notification import sender
from sahara.utils import rpc as messaging

class NotificationTest(base.SaharaTestCase):

    def setUp(self):
        super(NotificationTest, self).setUp()

    def _make_sample(self):
        ctx = context.ctx()

        self.ctx = ctx
        self.cluster_id = 'someId'
        self.cluster_name = 'someName'
        self.cluster_status = 'someStatus'

        sender.notify(ctx, self.cluster_id, self.cluster_name,
                      self.cluster_status, "update")

        self.create_mock('update')

    def create_mock(self, action):

        self.expected = mock.call(self.ctx,
                                  'sahara.cluster.%s' % action,
                                  {'cluster_id': self.cluster_id,
                                   'cluster_name': self.cluster_name,
                                   'cluster_status': self.cluster_status,
                                   'project_id': self.ctx.tenant_id,
                                   'user_id': self.ctx.user_id})

    @mock.patch('oslo_messaging.notify.notifier.Notifier.info')
    def test_update_cluster(self, mock_notify):
        self.override_config("enable_notifications", True)
        messaging.setup()

        self._make_sample()
        self.assertEqual([self.expected],
                         mock_notify.call_args_list)

        if messaging.TRANSPORT:
            messaging.TRANSPORT.cleanup()
            messaging.TRANSPORT = messaging.NOTIFIER = None


text:
<resources>
    <string name="app_name">HeaderViewPager</string>
</resources>


text:
.class Lcom/android/systemui/statusbar/phone/FingerprintUnlockController$1;
.super Ljava/lang/Object;
.source "FingerprintUnlockController.java"

# interfaces
.implements Ljava/lang/Runnable;

# annotations
.annotation system Ldalvik/annotation/EnclosingClass;
    value = Lcom/android/systemui/statusbar/phone/FingerprintUnlockController;
.end annotation

.annotation system Ldalvik/annotation/InnerClass;
    accessFlags = 0x0
    name = null
.end annotation

# instance fields
.field final synthetic this$0:Lcom/android/systemui/statusbar/phone/FingerprintUnlockController;

# direct methods
.method constructor <init>(Lcom/android/systemui/statusbar/phone/FingerprintUnlockController;)V
    .locals 0

    iput-object p1, p0, Lcom/android/systemui/statusbar/phone/FingerprintUnlockController$1;->this$0:Lcom/android/systemui/statusbar/phone/FingerprintUnlockController;

    invoke-direct {p0}, Ljava/lang/Object;-><init>()V

    return-void
.end method

# virtual methods
.method public run()V
    .locals 2

    const-string/jumbo v0, "FingerprintController"

    const-string/jumbo v1, "fp wakelock: TIMEOUT!!"

    invoke-static {v0, v1}, Landroid/util/Log;->i(Ljava/lang/String;Ljava/lang/String;)I

    iget-object v0, p0, Lcom/android/systemui/statusbar/phone/FingerprintUnlockController$1;->this$0:Lcom/android/systemui/statusbar/phone/FingerprintUnlockController;

    invoke-static {v0}, Lcom/android/systemui/statusbar/phone/FingerprintUnlockController;->-wrap0(Lcom/android/systemui/statusbar/phone/FingerprintUnlockController;)V

    return-void
.end method