text:
# Debian Packaging Script
# Currently Intended to build on ubuntu14.04

set -e

SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
  SOURCE="$(readlink "$SOURCE")"
  [[ "$SOURCE" != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"

source "$DIR/../common/_common.sh"
REPOROOT="$DIR/../.."

help(){
    echo "Usage: $0"
    echo ""
    echo "Options:"
    echo "  --version <version>                Specify a version for the package."
    echo "  --input <input directory>          Package the entire contents of the directory tree."
    echo "  --manpages <man pages directory>   Directory containing man pages for the package (Optional)."
    echo "  --output <output debfile>          The full path to which the package will be written."
    echo "  --package-name <package name>      Package to identify during installation. Example - 'dotnet-nightly', 'dotnet'"
    echo "  --brand-name <brand name>          Brand name of the package, used for 'short_description' of the deb file. Example - '.NET CLI SDK'"
    echo "  --framework-nuget-name <name>      The name of the nuget package that produced this shared framework."
    echo "  --framework-nuget-version <ver>    The versionf of the nuget package that produced this shared framework."
    echo "  --previous-version-url <url>       Url to the previous version of the debian packge against which to run the upgrade tests."
    echo "  --obj-root <object root>           Root folder for intermediate objects."
    exit 1
}

parseargs(){

    while [[ $# > 0 ]]; do
        lowerI="$(echo $1 | awk '{print tolower($0)}')"
        case $lowerI in
        -m|--manpages)
            MANPAGE_DIR=$2
            shift
            ;;
        -o|--output)
            OUTPUT_DEBIAN_FILE=$2
            shift
            ;;
        -i|--input)
            REPO_BINARIES_DIR=$2
            shift
            ;;
        -p|--package-name)
            DOTNET_DEB_PACKAGE_NAME=$2
            shift
            ;;
        -b|--brand-name)
            DOTNET_CLI_BRAND_NAME=$2
            shift
            ;;
        -v|--version)
            DOTNET_CLI_VERSION=$2
            shift
            ;;
        --previous-version-url)
            PREVIOUS_VERSION_URL=$2
            shift
            ;;
        --framework-debian-package-name)
            SHARED_FRAMEWORK_DEBIAN_PACKAGE_NAME=$2
            shift
            ;;
        --framework-nuget-name)
            SHARED_FRAMEWORK_NUGET_NAME=$2
            shift
            ;;
        --framework-nuget-version)
            SHARED_FRAMEWORK_NUGET_VERSION=$2
            shift
            ;;
        --obj-root)
            OBJECT_DIR=$2
            shift
            ;;
        --help)
            help
            ;;
        *)
            break
            ;;
        esac
        shift
    done

    if [ -z "$DOTNET_CLI_VERSION" ]; then
        echo "Provide a version number. Missing option '--version'" && help
    fi

    if [ -z "$OUTPUT_DEBIAN_FILE" ]; then
        echo "Provide an output deb. Missing option '--output'" && help
    fi

    if [ -z "$REPO_BINARIES_DIR" ]; then
        echo "Provide an input directory. Missing option '--input'" && help
    fi

    if [ -z "$DOTNET_DEB_PACKAGE_NAME" ]; then
        echo "Provide an the name for the debian package. Missing option '--package-name'" && help
    fi

    if [ -z "$SHARED_FRAMEWORK_NUGET_NAME" ]; then
        echo "Provide the NuGet name of the targetted Shared Framework. Missing option '--framework-nuget-name'" && help
    fi

    if [ -z "$PREVIOUS_VERSION_URL" ]; then
        echo "Provide a URL to the previous debian pacakge (Required for running upgrade tests). Missing option '--previous-version-url'" && help
    fi

    if [ -z "$SHARED_FRAMEWORK_NUGET_VERSION" ]; then
        echo "Provide the NuGet version of the targetted Shared Framework. Missing option '--framework-nuget-version'" && help
    fi

    if [ ! -d "$REPO_BINARIES_DIR" ]; then
        echo "'$REPO_BINARIES_DIR' - is either missing or not a directory" 1>&2
        exit 1
    fi

}

parseargs "$@"

PACKAGING_ROOT="$REPOROOT/packaging/debian"
PACKAGING_TOOL_DIR="$REPOROOT/tools/DebianPackageTool"

PACKAGE_OUTPUT_DIR="$OBJECT_DIR/deb_output"
PACKAGE_LAYOUT_DIR="$OBJECT_DIR/deb_intermediate"
TEST_STAGE_DIR="$OBJECT_DIR/debian_tests"

# remove any residual deb files from earlier builds
rm -f "$PACKAGE_OUTPUT_DIR/*.deb"

execute_build(){
    create_empty_debian_layout
    copy_files_to_debian_layout
	update_debian_json
    create_debian_package
}

execute_test(){
    test_debian_package
}

create_empty_debian_layout(){
    header "Creating empty debian package layout"

    rm -rf "$PACKAGE_LAYOUT_DIR"
    mkdir -p "$PACKAGE_LAYOUT_DIR"

    mkdir "$PACKAGE_LAYOUT_DIR/\$"
    mkdir "$PACKAGE_LAYOUT_DIR/package_root"
    mkdir "$PACKAGE_LAYOUT_DIR/samples"
    mkdir "$PACKAGE_LAYOUT_DIR/docs"
    mkdir "$PACKAGE_LAYOUT_DIR/debian"
}

copy_files_to_debian_layout(){
    header "Copying files to debian layout"

    # Copy Built Binaries
    cp -a "$REPO_BINARIES_DIR/." "$PACKAGE_LAYOUT_DIR/package_root"

    # Copy config file
    cp "$PACKAGING_ROOT/dotnet-debian_config.json" "$PACKAGE_LAYOUT_DIR/debian_config.json"

    # Copy Manpages
    cp -a "$MANPAGE_DIR/sdk/." "$PACKAGE_LAYOUT_DIR/docs"

    # Append Version to all manpage files
    for manpage in "$PACKAGE_LAYOUT_DIR/docs/"*.1; do mv "$manpage" "${manpage%.1}"; done
    for manpage in "$PACKAGE_LAYOUT_DIR/docs/"*; do mv "$manpage" "${manpage}-${DOTNET_CLI_VERSION}.1"; done

    # Copy postinstall
    cp "$PACKAGING_ROOT/postinst" "$PACKAGE_LAYOUT_DIR/debian/postinst"
}

create_debian_package(){
    header "Packing .deb"

    mkdir -p "$PACKAGE_OUTPUT_DIR"

    "$PACKAGING_TOOL_DIR/package_tool" -i "$PACKAGE_LAYOUT_DIR" -o "$PACKAGE_OUTPUT_DIR" -v $DOTNET_CLI_VERSION -n "$DOTNET_DEB_PACKAGE_NAME"
}

update_debian_json()
{
    header "Updating debian.json file"
    sed -i "s/%SHARED_FRAMEWORK_DEBIAN_PACKAGE_NAME%/$SHARED_FRAMEWORK_DEBIAN_PACKAGE_NAME/g" "$PACKAGE_LAYOUT_DIR"/debian_config.json
    sed -i "s/%SHARED_FRAMEWORK_NUGET_NAME%/$SHARED_FRAMEWORK_NUGET_NAME/g" "$PACKAGE_LAYOUT_DIR"/debian_config.json
    sed -i "s/%SHARED_FRAMEWORK_NUGET_VERSION%/$SHARED_FRAMEWORK_NUGET_VERSION/g" "$PACKAGE_LAYOUT_DIR"/debian_config.json
    sed -i "s/%SDK_NUGET_VERSION%/$DOTNET_CLI_VERSION/g" "$PACKAGE_LAYOUT_DIR"/debian_config.json
    sed -i "s/%CLI_SDK_BRAND_NAME%/$DOTNET_CLI_BRAND_NAME/g" "$PACKAGE_LAYOUT_DIR"/debian_config.json
}

test_debian_package(){
    header "Testing debian package"

    install_bats
    run_package_integrity_tests
}

install_bats() {
    rm -rf $TEST_STAGE_DIR
    git clone https://github.com/sstephenson/bats.git $TEST_STAGE_DIR
}

run_package_integrity_tests() {
    # Set LAST_VERSION_URL to enable upgrade tests
    # Temporarily disable last version until we have one with shared fx
    # export LAST_VERSION_URL="$PREVIOUS_VERSION_URL"

    $TEST_STAGE_DIR/bin/bats $PACKAGE_OUTPUT_DIR/test_package.bats
}

execute_build

DEBIAN_FILE=$(find $PACKAGE_OUTPUT_DIR -iname "*.deb")

execute_test

mv -f "$DEBIAN_FILE" "$OUTPUT_DEBIAN_FILE"


text:
using System;
using System.Linq;
using System.Reflection;
using Microsoft.WindowsAPICodePack.Shell;
using Xunit;

namespace Tests
{
    public class StockIconsTests
    {
        [Fact]
        public void StockIconsNotNull()
        {
            PropertyInfo[] infos = typeof(StockIcons).GetType().GetProperties(BindingFlags.Public);

            // BUG: This (StockIcons) doesn't follow the same pattern as most of the other factory style patterned classes?
            StockIcons icons = new StockIcons();

            foreach (PropertyInfo info in infos)
            {
                if (info.PropertyType == typeof(StockIcon))
                {
                    Assert.DoesNotThrow(new Assert.ThrowsDelegate(() =>
                    {
                        StockIcon test = (StockIcon)info.GetValue(icons, null);
                    }));
                }
            }
        }

        [Fact]
        public void StockIconsContainedInAllCollection()
        {
            // TODO: Redesign test, this is a pretty redundant test with the way the class is designed.

            StockIcons icons = new StockIcons();

            PropertyInfo[] infos = typeof(StockIcons).GetType().GetProperties(BindingFlags.Public);
            foreach (var info in infos)
            {
                if (info.PropertyType != typeof(StockIcon)) continue;

                StockIcon icon = null;
                try
                {
                    //this will throw an exception if the icon does not exist on the system
                    icon = (StockIcon)info.GetValue(icons, null);
                }
                catch { continue; }

                Assert.Contains<StockIcon>(icon, icons.AllStockIcons);
            }
        }

        [Fact]
        public void VerifyAllStockIconsArePropertized()
        {
            PropertyInfo[] infos = typeof(StockIcons).GetProperties(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance);

            int count = infos.Count(x => x.PropertyType == typeof(StockIcon));

            Assert.Equal(count, Enum.GetValues(typeof(StockIconIdentifier)).Length);
        }
    }
}

redpajama_set_name: RedPajamaGithub


text:
title: TogetherJS Setup and Usage for Beginners
author: Team Koding
categories: [frameworks platforms, togetherjs, javascript]

template: page.toffee
---

# TogetherJS Setup and Usage for Beginners

In this article we will go over beginner steps for TogetherJS.

## What is TogetherJS?

TogetherJS is a drop-in collaboration framework. Think of it like easily adding Google Docs to any site. Normally when you program a real time site, you use a framework like SocketIO, SockJS, or Firebase, and it takes a considerable amount of work just to make content sync up between clients. Adding text chat/mouse tracking/voice chat is a lot of additional work that is often outside the scope of your project. TogetherJS takes away that problem. You focus on your application, and TogetherJS can focus on the difficulty of mouse tracking. With just two lines of code, you get all of these features. Easy!

## Setting up TogetherJS

In the following example, we'll setup TogetherJS by using a plain html page and the TogetherJS Scripts.

And just like that, we have a static page with text chat, voice chat, and mouse tracking built into our html page! And because we used [Koding](https://koding.com) we can share this project with our friends easily. On localhost you would have to run a web server, expose and forward your ports, figure out your ip, and finally share it with a friend. Koding saves you that localhost-hassle, and makes this a one-step process. Just share your domain and you're done.


text:
package com.google.firebase.quickstart;

import com.google.auth.oauth2.GoogleCredentials;
import com.google.firebase.FirebaseApp;
import com.google.firebase.FirebaseOptions;
import com.google.firebase.database.*;
import com.google.firebase.quickstart.email.MyEmailer;
import com.google.firebase.quickstart.model.Post;
import com.google.firebase.quickstart.model.User;
import java.io.IOException;
import org.knowm.sundial.SundialJobScheduler;

import java.io.FileInputStream;

/**
 * Firebase Database quickstart sample for the Java Admin SDK.
 * See: https://firebase.google.com/docs/admin/setup#add_firebase_to_your_app
 */
public class Database {

    private static final String DATABASE_URL = "https://<YOUR-DATABASE>.firebaseio.com/";

    private static DatabaseReference database;

    /**
     * Notify a user of a new start and then update the last notification time.
     */
    private static void sendNotificationToUser(final String uid, final String postId) {
        // [START single_value_read]
        final DatabaseReference userRef = database.child("users").child(uid);
        userRef.addListenerForSingleValueEvent(new ValueEventListener() {
            public void onDataChange(DataSnapshot dataSnapshot) {
                User user = dataSnapshot.getValue(User.class);
                if (user.email != null) {
                    // Send email notification
                    MyEmailer.sendNotificationEmail(user.email, uid, postId);
                }
            }

            public void onCancelled(DatabaseError databaseError) {
                System.out.println("Unable to get user data from " + userRef.getKey());
                System.out.println("Error: " + databaseError.getMessage());
            }
        });
        // [END single_value_read]
    }

    /**
     * Update the startCount value to equal the number of stars in the map.
     */
    private static void updateStarCount(DatabaseReference postRef) {
        // [START post_stars_transaction]
        postRef.runTransaction(new Transaction.Handler() {
            public Transaction.Result doTransaction(MutableData mutableData) {
                Post post = mutableData.getValue(Post.class);
                if (post != null) {
                    // Update the starCount to be the same as the number of members in the stars map.
                    if (post.stars != null) {
                        post.starCount = post.stars.size();
                    } else {
                        post.starCount = 0;
                    }

                    mutableData.setValue(post);
                    return Transaction.success(mutableData);
                } else {
                    return Transaction.success(mutableData);
                }
            }

            public void onComplete(DatabaseError databaseError, boolean complete, DataSnapshot dataSnapshot) {
                System.out.println("updateStarCount:onComplete:" + complete);
            }
        });
        // [END post_stars_transaction]
    }

    /**
     * Start global listener for all Posts.
     */
    public static void startListeners() {
        database.child("posts").addChildEventListener(new ChildEventListener() {

            public void onChildAdded(DataSnapshot dataSnapshot, String prevChildName) {
                final String postId = dataSnapshot.getKey();
                final Post post = dataSnapshot.getValue(Post.class);

                // Listen for changes in the number of stars and update starCount
                addStarsChangedListener(post, postId);

                // Listen for new stars on the post, notify users on changes
                addNewStarsListener(dataSnapshot.getRef(), post);
            }

            public void onChildChanged(DataSnapshot dataSnapshot, String prevChildName) {}

            public void onChildRemoved(DataSnapshot dataSnapshot) {}

            public void onChildMoved(DataSnapshot dataSnapshot, String prevChildName) {}

            public void onCancelled(DatabaseError databaseError) {
                System.out.println("startListeners: unable to attach listener to posts");
                System.out.println("startListeners: " + databaseError.getMessage());
            }
        });
    }

    /**
     * Listen for stars added or removed and update the starCount.
     */
    private static void addStarsChangedListener(Post post, String postId) {
        // Get references to the post in both locations
        final DatabaseReference postRef = database.child("posts").child(postId);
        final DatabaseReference userPostRef = database.child("user-posts").child(post.uid).child(postId);

        // When the post changes, update the star counts
        // [START post_value_event_listener]
        postRef.child("stars").addValueEventListener(new ValueEventListener() {
            public void onDataChange(DataSnapshot dataSnapshot) {
                updateStarCount(postRef);
                // [START_EXCLUDE]
                updateStarCount(userPostRef);
                // [END_EXCLUDE]
            }

            public void onCancelled(DatabaseError databaseError) {
                System.out.println("Unable to attach listener to stars for post: " + postRef.getKey());
                System.out.println("Error: " + databaseError.getMessage());
            }
        });
        // [END post_value_event_listener]
    }

    /**
     * Send email to author when new star is received.
     */
    private static void addNewStarsListener(final DatabaseReference postRef, final Post post) {
        // [START child_event_listener_recycler]
        postRef.child("stars").addChildEventListener(new ChildEventListener() {
            public void onChildAdded(DataSnapshot dataSnapshot, String prevChildName) {
                // New star added, notify the author of the post
                sendNotificationToUser(post.uid, postRef.getKey());
            }

            public void onChildChanged(DataSnapshot dataSnapshot, String prevChildName) {}

            public void onChildRemoved(DataSnapshot dataSnapshot) {}

            public void onChildMoved(DataSnapshot dataSnapshot, String prevChildName) {}

            public void onCancelled(DatabaseError databaseError) {
                System.out.println("Unable to attach new star listener to: " + postRef.getKey());
                System.out.println("Error: " + databaseError.getMessage());
            }
        });
        // [END child_event_listener_recycler]
    }

    /**
     * Send an email listing the top posts every Sunday.
     */
    private static void startWeeklyTopPostEmailer() {
        SundialJobScheduler.startScheduler("com.google.firebase.quickstart.email");
    }

    public static void main(String[] args) {
        // Initialize Firebase
        try {
            // [START initialize]
            FileInputStream serviceAccount = new FileInputStream("service-account.json");
            FirebaseOptions options = new FirebaseOptions.Builder()
                    .setCredentials(GoogleCredentials.fromStream(serviceAccount))
                    .setDatabaseUrl(DATABASE_URL)
                    .build();
            FirebaseApp.initializeApp(options);
            // [END initialize]
        } catch (IOException e) {
            System.out.println("ERROR: invalid service account credentials. See README.");
            System.out.println(e.getMessage());

            System.exit(1);
        }

        // Shared Database reference
        database = FirebaseDatabase.getInstance().getReference();

        // Start listening to the Database
        startListeners();

        // Kick off weekly email task
        startWeeklyTopPostEmailer();
    }

}


text:
package mobi.tarantino.stub.auto.di;

import android.content.Context;

import com.google.gson.Gson;
import com.squareup.picasso.Picasso;

import javax.inject.Singleton;

import dagger.Component;
import mobi.tarantino.stub.auto.IPreferencesManager;
import mobi.tarantino.stub.auto.IntentStarter;
import mobi.tarantino.stub.auto.Logger;
import mobi.tarantino.stub.auto.MobiApplication;
import mobi.tarantino.stub.auto.SchedulerProvider;
import mobi.tarantino.stub.auto.feature.auth.AuthComponent;
import mobi.tarantino.stub.auto.feature.auth.AuthModule;
import mobi.tarantino.stub.auto.feature.demoTour.DemoTourActivity;
import mobi.tarantino.stub.auto.feature.notifications.NotificationsActivity;
import mobi.tarantino.stub.auto.feature.notifications.NotificationsComponent;
import mobi.tarantino.stub.auto.feature.notifications.NotificationsModule;
import mobi.tarantino.stub.auto.gcm.MobiGcmListenerService;
import mobi.tarantino.stub.auto.gcm.RegistrationIntentService;
import mobi.tarantino.stub.auto.model.additionalData.MobiAdditionalDataApi;
import mobi.tarantino.stub.auto.model.auth.MobiAuthApi;
import mobi.tarantino.stub.auto.model.database.DatabaseHelperFactory;
import mobi.tarantino.stub.auto.model.database.DatabaseModule;
import mobi.tarantino.stub.auto.model.location.LocationModel;
import mobi.tarantino.stub.auto.retrofitUtils.HttpLoggingInterceptor;
import mobi.tarantino.stub.auto.retrofitUtils.errorHandler.ErrorHandler;

@Singleton
@Component(modules = {
        BaseModule.class,
        SchedulersModule.class,
        AuthApiModule.class,
        NavigationModule.class,
        AdditionalDataModule.class,
        DatabaseModule.class,
})
public interface BaseComponent extends AppComponent {
    SessionComponent.Builder buildSessionComponent();

    AuthComponent plus(AuthModule trafficFinesModule);

    NotificationsComponent plus(NotificationsModule notificationsModule);

    Context context();

    Logger logger();

    Picasso providesPicasso();

    HttpLoggingInterceptor.Logger httpLoggingInterceptor();

    MobiAuthApi mobiAuthApi();

    MobiAdditionalDataApi mobiAdditionalDataApi();

    Gson gson();

    IPreferencesManager providesPreferencesManager();

    SchedulerProvider providesSchedulers();

    DatabaseHelperFactory providesDatabaseHelperFactory();

    LocationModel provideLocationModel();

    ErrorHandler errorHandler();

    IntentStarter providesIntentStarter();

    void inject(MobiApplication mobiApplication);

    void inject(DependencyComponentManager dependencyComponentManager);

    void inject(DemoTourActivity demoTourActivity);

    void inject(IntentStarter intentStarter);

    void inject(RegistrationIntentService registrationIntentService);

    void inject(MobiGcmListenerService mobiGcmListenerService);

    void inject(NotificationsActivity notificationsActivity);
}


text:
#!/usr/bin/python
#- Copyright 2014 GOTO 10.
#- Licensed under the Apache License, Version 2.0 (see LICENSE).

## # Hemingway
##
## A tool for turning markdown-based literate programs into HTML. Inspired by
## [Knuth](http://www.literateprogramming.com/knuthweb.pdf) and
## [pycco](http://fitzgen.github.io/pycco/) (and its ilk). This isn't the full
## literate programming model as envisioned by Knuth but an attempt to get some
## of the benefits without requiring extensions to the source language.
##
## This page is the output from running the tool on its own source code,
## [hemingway.py](https://github.com/goto-10/hemingway/blob/master/hemingway/main.py).
##
## ## Installation
##
## To install hemingway in a linux-like environment do
##
##     git clone https://github.com/goto-10/hemingway.git
##     cd hemingway
##     sudo python setup.py install
##
## This will fetch the dependencies and make the `hemingway` command available.
##
## ## Basic usage
##
## Hemingway is intended to be run on a collection of files located under the
## same root directory. You specify the argument like so,
##
##     hemingway --root path/to/src "**/*.c" "**/*.h"
##
## What this means is: starting from `path/to/src`, find all files that match
## the globs `**/*.c` and `**/*.h` relative to that root and convert them. The
## output will be stored in a subdirectory, `doc/` by default, also using paths
## relative to the root. So `path/to/src/foo/bar/baz.c` will be converted to
## `doc/foo/bar/baz.c.html`. See
##
##     hemingway --help
##
## for an overview of all the options supported by the tool.
##
## ## Watchdog
##
## Besides doing batch conversion Hemingway also supports live editing. If you
## pass the `--watchdog` flag the script won't terminate, instead it will watch
## the file system and re-convert any files that change. To end the program just
## Ctrl-C it.
##
## ## Implementation
##
## The script has a few different components that each have a separate
## responsibility in the process. To keep things simple and not have to worry
## about `$PYTHONPATH` everything is in this one file.
##
##   - The {{#SourceIndex}}(index) keeps track of the files to convert. During
##     initialization it locates all the files that match the arguments. After
##     that it answers queries from the other components but the index stays the
##     same.
##   - The {{#Scheduler}}(scheduler) schedules file conversions. It's has an
##     async queue which other parts of the program can add conversions to and
##     then the scheduler takes care of getting the conversions done.
##   - The {{#Conversion}}(conversion) code does the actual work of splitting
##     the source of a file into code and comment parts and then converting
##     each part appropriately.
##   - The file handling (open, closing, watching) is done by the {{#Main}}(main)
##     class.
##   - The {{diagram.py}}(diagram processor) analyzes ascii diagrams and produces
##     corresponding images.
##
## The {{#UnitTests}}(please_run_the_tests) function runs the unit
## tests.

import argparse
import glob2
import logging
import linesboxes.diagram
import linesboxes.dom
import os.path
import Queue
import re
import sys
import threading

## Import the nonstandard libraries defensively since they are more likely than
## the others to fail and theres a reasonable action the user can take to fix
## the issue.

# Try importing markdown to see whether it's available.
try:
  import markdown
except ImportError, e:
  print "Hemingway requires the markdown package. Try 'pip install markdown'."

# Try importing pygments to see whether it's available.
try:
  import pygments
except ImportError, e:
  print "Hemingway requires the pygments package. Try 'pip install pygments'."

import pkg_resources

# Import the subpackages we're actually going to use.
import markdown.extensions
import markdown.inlinepatterns
import markdown.preprocessors
import markdown.treeprocessors
import markdown.util
import pygments.formatters
import pygments.lexers

_HTML_TEMPLATE = """\

    %(header)s
    <link rel="stylesheet" href="%(assets)s/markdown.css" type="text/css" />
    <link rel="stylesheet" href="%(assets)s/code.css" type="text/css" />
    <link rel="stylesheet" href="%(assets)s/diagram.css" type="text/css" />

        %(contents)s

"""

## ## Markdown extensions
##
## To support the integration of markdown with source code it's convenient to
## add a few extensions to the markdown processor. Luckily the python markdown
## framework has an api intended to allow exactly the kinds of extensions that
## we want. Good work guys. The extensions are,
##
##   - {{#CrossReferences}}(Cross references). The ability to create references
##     to other source files and program elements and have the processor resolve
##     them to HTML links. The syntax is `{{ref}}(title)` where the title is
##     optional.
##   - {{#AutomaticAnchorNames}}(Automatic anchor names). This is a post-processor
##     that adds anchor names to generated markdown headers such that the cross
##     references have something to link to besides just toplevel files.

## ### Cross References
##
## The cross reference syntax fits pretty nicely into the markdown library's
## inline pattern API hook. The framework matches based on the regexp passed to
## the constructor and then we perform a secondary match on the contents of the
## cross reference.

class CrossRefPattern(markdown.inlinepatterns.Pattern):

  MARKUP_PATTERN = r"\{\{([^\{\}]*)\}\}(?:\(([^)]+)\))?"
  TARGET_PATTERN = re.compile(r"([^#]*)(#.*)?")

  def __init__(self, context):
    super(CrossRefPattern, self).__init__(CrossRefPattern.MARKUP_PATTERN)
    self.context = context

  def handleMatch(self, match):
    name = match.group(2)
    title = match.group(3)
    if title is None:
      title = name
    ref_match = CrossRefPattern.TARGET_PATTERN.match(name)
    filename = ref_match.group(1)
    anchor = ref_match.group(2)
    if filename.strip() == "":
      destination = ""
    else:
      destination = self.context.resolve_cross_reference(filename)
    if destination is None:
      element = markdown.util.etree.Element('em')
      element.text = title
      return element
    else:
      if not anchor is None:
        destination += anchor
      element = markdown.util.etree.Element('a')
      element.text = title
      element.set("href", destination)
      return element

## ### Automatic Anchor Names
##
## This tree processor takes an element tree that is about to be serialized and
## adds named anchors to all the headers such that they can be linked to from
## cross references.
##
## Again the markdown library has a convenient API that lets us traverse the
## DOM before it gets serialized and wrap all headers in appropriately named
## anchors.

class AutoAnchorProcessor(markdown.treeprocessors.Treeprocessor):

  TAGS_TO_ANCHOR = ['h1', 'h2', 'h3']

  # Recursively find all headers and interject anchor nodes around them.
  def extend_headers(self, node):
    if markdown.util.etree.iselement(node):
      child_index = 0
      while child_index < len(node):
        child = node[child_index]
        self.extend_headers(child)
        new_child = self.get_extended_header(child)
        if not new_child is child:
          node[child_index] = new_child
        child_index += 1

  # Given a node, if it is a header returns a replacement with an anchor
  # otherwise returns the node itself.
  def get_extended_header(self, node):
    if not node.tag in AutoAnchorProcessor.TAGS_TO_ANCHOR:
      return node
    text = AutoAnchorProcessor.get_node_text(node)
    name = AutoAnchorProcessor.header_to_anchor_name(text)
    anchor = markdown.util.etree.Element('a')
    anchor.set("name", name)
    anchor.append(node)
    return anchor

  # Given the raw text contents of a header returns the anchor name to use.
  @staticmethod
  def header_to_anchor_name(text):
    # Strip out any non alpha numeric characters. Convert them to spaces so they
    # contribute to the titlecase behavior below.
    alnum = re.sub(r"\W", " ", text)
    # Split the string by spaces.
    raw_parts = re.split(r"\s+", alnum)
    # Capitalize the first character in each part.
    cap_parts = [part.title() for part in raw_parts]
    # Join the parts back together.
    return "".join(cap_parts)

  # Returns just the raw text contained in a given node.
  @staticmethod
  def get_node_text(node):
    result = []
    if not node.text is None:
      result.append(node.text)
    if markdown.util.etree.iselement(node):
      for child in node:
        result.append(AutoAnchorProcessor.get_node_text(child))
    if not node.tail is None:
      result.append(node.tail)
    return "".join(result)

  # The method called by the framework.
  def run(self, root):
    self.extend_headers(root)
    return root

## ### Extension hook
##
## The hook used to install the hemingway extension into the markdown processor.
## This is somewhat cargo-cult, I'm not sure what the scope is of an extension
## (I assume it's just the current markdown object) and I'm not clear on exactly
## what `not_strong` is.

class HemingwayExtension(markdown.extensions.Extension):

  def __init__(self, context):
    self.context = context

  def extendMarkdown(self, md, globals):
    md.inlinePatterns.add('crossref', CrossRefPattern(self.context), '>not_strong')
    md.treeprocessors['autoanchor'] = AutoAnchorProcessor()

## ## Source index
##
## This class is responsible for looking through the file system and sorting out
## which files to convert. This all happens at startup in {{#IndexLoad}}(load).
## After loading its main responsibility is resolving cross-references between
## the source files. That happens in {{#ResolvingCrossReferences}}(resolve_cross_reference).
## For each source file there's a {{#SourceFile}}(source file) object that
## keeps track of the meta-information we need about the file.

class SourceIndex(object):

  def __init__(self, root, patterns):
    self.root = root
    self.patterns = patterns
    self.sources = None

  # Generates all the source files in sorted order.
  def list_sources(self):
    for abspath in sorted(self.sources.keys()):
      source = self.sources[abspath]
      yield source

  # Returns the source file that corresponds to the given absolute path. If no
  # file exists None is returned.
  def get_absolute_source(self, path):
    return self.sources.get(path, None)

  ## ### Index load
  ##
  ## Loading the index is pretty straightforward, `glob` does all the work. We
  ## need to know both the absolute and relative paths for each file which would
  ## be easy if `glob` worked relative to a given path (in this case the root).
  ## Alas it doesn't so we have to join the root and paths together and then
  ## remove the root again from the files. If the root has no trailing path
  ## separator that is particularly sticky because one will be added by glob, so
  ## we ensure at the top that it does.
  def load(self):
    sources = {}
    if not self.root.endswith(os.sep):
      self.root += os.sep
    for pattern in self.patterns:
      path = os.path.join(self.root, pattern)
      for rootpath in glob2.glob(path):
        if not rootpath.startswith(self.root):
          raise AssertionError("Found %s under %s?" % (rootpath, self.root))
        relpath = rootpath[len(self.root):]
        abspath = os.path.abspath(rootpath)
        sources[abspath] = SourceFile(relpath, abspath)
    self.sources = sources

  ## Given two paths, a target and an input, returns the input with any paths
  ## prefixes shared between the two removed.
  def make_relative(self, target, input):
    target_parts = target.split(os.path.sep)
    input_parts = input.split(os.path.sep)
    while len(target_parts) != 0 and len(input_parts) != 0 and target_parts[0] == input_parts[0]:
      target_parts = target_parts[1:]
      input_parts = input_parts[1:]
    return os.path.sep.join(input_parts)

  ## ### Resolving cross references
  ##
  ## Cross references can be specified using short paths, so if there is a file
  ## called `foo/bar/baz.c` you can refer to it just as `baz.c`. The current
  ## implementation is kind of a hack but that's because I'd want to see what
  ## use cases are for more complex references before attempting to add support.
  ##
  ## Once a target has been found we need to strip the common prefix between the
  ## place where the reference will appear and the target, since otherwise it
  ## messes up the relative path.
  def resolve_cross_reference(self, name, whos_asking):
    for source in self.list_sources():
      if source.relpath.endswith(name):
        target_out = source.get_relative_output_path()
        asking_out = whos_asking.get_relative_output_path()
        return self.make_relative(asking_out, target_out)
    return None

## ### Source File
##
## A source file encapsulates the nitty-gritty of where the file is and what the
## output path is.

class SourceFile(object):

  def __init__(self, relpath, abspath):
    self.relpath = relpath
    self.abspath = abspath
    self.language = LanguageInfo.for_file(abspath)

  # Reads and returns the contents of this source file.
  def read(self):
    port = open(self.abspath, "rt")
    try:
      contents = port.read()
    finally:
      port.close()
    return contents

  # Returns the language info describing this file.
  def get_language(self):
    return self.language

  # Returns the output path to use for this file given that the output
  # directory root is the given path.
  def get_absolute_output_path(self, outdir):
    relpath = self.get_relative_output_path()
    return os.path.join(outdir, relpath)

  # Returns a relative path to the output corresponding to this file.
  def get_relative_output_path(self):
    return "%s.html" % self.relpath

  def __str__(self):
    return "source %s" % self.relpath

## ## Scheduler
##
## The scheduler is responsible for getting files converted. It runs
## asynchronously which may seem like an odd choice but it's really convenient
## to be able to reserve the main thread for the watchdog functionality and then
## have the conversion take place in the background on a different thread.
##
## The only part of the scheduler that's slightly complicated is the fact that
## when you ask it to stop (using `finish_up`) it doesn't stop immediately, it
## completes any pending conversion requests first. That way the non-watchdog
## version can get it started and ask it to finish immediately and then rely on
## it finishing up nicely.

class Scheduler(object):

  def __init__(self, context):
    self.context = context
    self.queue = Queue.Queue()
    self.keep_going = True

  # Spawns the scheduler thread then returns immediately.
  def start(self):
    threading.Thread(target=self.process).start()

  # Loop around processing conversions until asked to stop and the queue is
  # empty.
  def process(self):
    while self.keep_going or (not self.queue.empty()):
      next = self.queue.get(True)
      if next is None:
        return
      self.context.convert_file(next)

  def finish_up(self):
    self.keep_going = False
    self.queue.put(None)

  def schedule(self, source):
    self.queue.put(source)

## ## Conversion
##
## The source conversion process happens in two steps:
##
##   1. First the source is {{#Chopping}}(chopped into) contiguous comment and
##      code blocks according to the syntax of the {{#LanguageProcessing}}(language).
##   2. Then each block is asked to {{#BlockConversion}}(convert) its lines
##      which it does either using the custom
##      {{#MarkdownExtensions}}(markdown processor) or custom
##      {{#PygmentsExtensions}}(pygments formatter).

class Converter(object):

  def __init__(self, context):
    self.context = context
    extension = HemingwayExtension(context)
    self.markdown = markdown.Markdown(extensions=['footnotes', extension])

  # Converts the text of a source file, returning the result as a string.
  def convert_source(self, source):
    contents = source.read()
    language = source.get_language()
    lines = contents.splitlines()
    blocks = self.chop_into_blocks(language, lines)
    results = [block.to_html(self) for block in blocks]
    body = "".join(results)
    return _HTML_TEMPLATE % {
      "header": self.context.get_headers(),
      "contents": body,
      "assets": self.get_assets_path(source)
    }

  def get_assets_path(self, origin):
    depth = -1
    last_parent = origin.relpath
    parent = os.path.dirname(last_parent)
    while parent != last_parent:
      last_parent = parent
      parent = os.path.dirname(parent)
      depth += 1
    if depth == 0:
      relative = "."
    else:
      relative = "/".join([".."] * depth)
    return "%s/assets" % relative

  ## ### Chopping
  ##
  ## The lines of the source is chopped into blocks really straightforwardly:
  ## for each line, ask the language if it's a comment. If the type of the line
  ## is the same as the block we're building, keep building. Otherwise flush the
  ## current block and start a new one. Lines can be ignored in which case we
  ## just skip past them (which lines to ignore is user configurable) and whole
  ## blocks can be ignored, specifically empty code blocks which don't really
  ## makes sense. Empty comment blocks are keps because the render as vertical
  ## space and it turns out to look wrong if they're stripped.
  ##
  ## Of special consideration are hashbangs which are always ignored at the top
  ## of a file.

  # Given a list of lines returns a list of blocks where each block contains a
  # contiguous list of lines of the same type, code or comment.
  def chop_into_blocks(self, language, lines):
    blocks = []
    CurrentType = CodeBlock
    current_lines = []
    def add_block(block):
      if not block.is_empty():
        blocks.append(block)
    is_beginning = True
    for line in lines:
      if is_beginning:
        try:
          if line.startswith("#!"):
            continue
        finally:
          is_beginning = False
      if language.is_ignored(line):
        continue
      if language.is_comment(line):
        NextType = CommentBlock
      elif language.is_diagram(line):
        NextType = DiagramBlock
      else:
        NextType = CodeBlock
      if NextType != CurrentType:
        if len(current_lines) > 0:
          add_block(CurrentType(current_lines, language))
        CurrentType = NextType
        current_lines = []
      current_lines.append(line)
    if len(current_lines) > 0:
      add_block(CurrentType(current_lines, language))
    return blocks

  # Convertes the given text using the shared markdown converter.
  def convert_markdown(self, text):
    return self.markdown.convert(text)

## ### Block conversion
##
## The two types of blocks, {{#CommentBlocks}}(comments) and
## {{#CodeBlocks}}(code), each have their own way to do conversion. An
## alternative approach would be to convert everything as markdown with a
## preprocessor step that converts the code blocks and wraps them in `<div>`s.
## I actually tried that and it was painfully slow, presumably because the
## pygments output is large which slows down markdown processing.
##
## ### Comment blocks
##
## Comment blocks produce their output by stripping the literate comment marker
## (plus one space if present, which is usually is) and then just passing the
## result as a block to the {{#Converter}}(converter) to be markdowned.
## Initially I stripped all the whitespace after the marker but since whitespace
## is significant in markdown that turned out the be a bad choice.

# A contiguous block of end-of-line comments to be markdowned.
class CommentBlock(object):

  def __init__(self, lines, language):
    self.lines = lines
    self.language = language

  def is_empty(self):
    return False

  def to_html(self, context):
    marker = self.language.get_marker()
    stripped = CommentBlock.strip_prefixes(marker, self.lines)
    stripped.append("")
    return context.convert_markdown("\n".join(stripped))

  @staticmethod
  def strip_prefixes(prefix, lines):
    return [CommentBlock.strip_prefix(prefix, line) for line in lines]

  @staticmethod
  def strip_prefix(prefix, line):
    if prefix is None:
      return line
    index = line.find(prefix)
    if index == -1:
      return line
    else:
      removed = line[index+len(prefix):]
      if removed.startswith(" "):
        removed = removed[1:]
      return removed

## ### Pygments extensions
##
## It's convenient to have anchors added to type and function names in the
## pygments output. For this we're using a custom formatter that scans each
## line of output for those and wraps them in anchors. It's not the ideal
## approach but it works; how well also depends on how the language lexer
## chooses to annotate the source code. For instance, it does well with
## python but not nearly as well with C. In python the word `Foo` in
## `class Foo` and `f = Foo()` are annotated differently, the first is a class
## name and the second is just a name, whereas in C `typedef Foo` and `Foo f`
## are annotated the same which means that we can't distinguish which is the
## declaration.

class CustomHtmlFormatter(pygments.formatters.HtmlFormatter):

  PATTERN = re.compile(r"(([^>]*))")
  TEMPLATE = "%s"

  def __init__(self):
    super(CustomHtmlFormatter, self).__init__(cssclass='hll')

  # Overide the default wrapper implementation.
  def wrap(self, source, outfile):
    transformed = self.transform_lines(source)
    return super(CustomHtmlFormatter, self).wrap(transformed, outfile)

  # The transformed source must be a generator so this method returns one that
  # generates the transformed output.
  def transform_lines(self, source):
    for (depth, line) in source:
      new_line = CustomHtmlFormatter.PATTERN.sub(
        CustomHtmlFormatter.inject_anchor, line)
      yield (depth, new_line)

  # Given a match for the span pattern above, returns a string that wraps the
  # match appropriately in an anchor.
  @staticmethod
  def inject_anchor(match):
    everything = match.group(1)
    text = match.group(2)
    return CustomHtmlFormatter.TEMPLATE % (text, everything)

## ### Code blocks
##
## Code blocks are converted using the lexer that was used to construct the
## {{#DeterminingTheLanguage}}(language). Pretty straightforward.

# A block of source code to be formatted using pygments.
class CodeBlock(object):

  FORMATTER = CustomHtmlFormatter()

  def __init__(self, lines, language):
    self.language = language
    self.lines = lines

  def is_empty(self):
    return len("".join(self.lines).strip()) == 0

  def to_html(self, context):
    highlighted = pygments.highlight("\n".join(self.lines),
      self.language.get_lexer(), CodeBlock.FORMATTER)
    return "
%s
" % highlighted

_DIAGRAM_TEMPLATE = """

    <svg width="%(width)sem" height="%(height)sem" viewBox="0 0 %(width)s %(height)s" class="diagram">
      %(elements)s
    </svg>

"""

_BOX_TEMPLATE = """
<rect x="%(x)s" y="%(y)s" width="%(width)s" height="%(height)s" class="box"></rect>
<foreignObject x="%(x)s" y="%(y)s" width="%(width)s" height="%(height)s">

%(contents)s

</foreignObject>
"""

_ROW_TEMPLATE = """
<line class="tablerow" x1="%(x1)s" y1="%(y1)s" x2="%(x2)s" y2="%(y2)s"/>
"""

_COL_TEMPLATE = """
<line class="tablecol" x1="%(x1)s" y1="%(y1)s" x2="%(x2)s" y2="%(y2)s"/>
"""

_PATH_TEMPLATE = """
<polyline class="path" points="%(points)s" />
"""

def to_x(v):
  return v * DiagramBlock.VERT_PER_HORZ

def to_y(v):
  return v

class DiagramBlock(object):

  # A factor to multiply onto all coordinates to make shapes that look square
  # in text form correspond to square(-ish) shapes in SVG. This is a square
  # shape:
  #
  #   +---+
  #   |   |
  #   +---+
  VERT_PER_HORZ = 3.0 / 5.0

  def __init__(self, lines, language):
    self.lines = lines
    self.language = language

  def is_empty(self):
    return len("".join(self.lines).strip()) == 0

  # Pad the lines with spaces all the way round.
  def get_stripped_lines(self):
    marker = self.language.get_diagram()
    return CommentBlock.strip_prefixes(marker, self.lines)

  def to_html(self, context):
    padded_lines = self.get_stripped_lines()
    processor = linesboxes.diagram.DiagramProcessor(padded_lines)
    diagram = processor.get_diagram()
    elements = []
    right_x = 0
    bottom_y = 0
    for element in diagram.get_elements():
      ((lx, ty), (rx, by)) = element.get_absolute_bounds()
      right_x = max(right_x, rx)
      bottom_y = max(bottom_y, by)
      type = element.get_type()
      if type == linesboxes.dom.BoxShape.TYPE:
        elements.append(self.box_to_svg(element))
      elif type == linesboxes.dom.TableShape.TYPE:
        elements.append(self.table_to_svg(element))
      elif type == linesboxes.dom.Path.TYPE:
        elements.append(self.path_to_svg(element))
    return _DIAGRAM_TEMPLATE % {
      "elements": "\n".join(elements),
      "width": to_x(right_x + 1),
      "height": to_y(bottom_y + 1)
    }

  def box_to_svg(self, box):
    (x, y) = box.get_position()
    (w, h) = box.get_extent()
    return _BOX_TEMPLATE % {
      "x": to_x(x + 0.5),
      "y": to_y(y + 0.5),
      "width": to_x(w - 1),
      "height": to_y(h - 1),
      "contents": ""
    }

  def table_to_svg(self, table):
    (x, y) = table.get_position()
    (w, h) = table.get_extent()
    parts = []
    parts.append(self.box_to_svg(table.get_boundary()))
    for row in table.get_rows():
      part = _ROW_TEMPLATE % {
        "x1": to_x(x + 0.5),
        "y1": to_y(y + row + 0.5),
        "x2": to_x(x + w - 0.5),
        "y2": to_y(y + row + 0.5)
      }
      parts.append(part)
    for col in table.get_columns():
      part = _COL_TEMPLATE % {
        "x1": to_x(x + col + 0.5),
        "y1": to_y(y + 0.5),
        "x2": to_x(x + col + 0.5),
        "y2": to_y(y + h - 0.5)
      }
      parts.append(part)
    return "".join(parts)

  def path_to_svg(self, path):
    (lx, ty) = path.get_position()
    def to_absolute((x, y)):
      return (to_x(x + lx + 0.5), to_y(y + ty + 0.5))
    return _PATH_TEMPLATE % {
      "points": " ".join("%s,%s" % p for p in map(to_absolute, path.get_points()))
    }

## ## Language Processing
##
## The language specific processing is all done by the `LanguageInfo` class.
## The information is siphoned out of pygments -- it knows the syntax of the
## languages and it's feasible to extract it in a lot of cases. Besides the
## defaults there are some hardcoded overrides, and longer term it should be
## possible to specify custom overrides probably via YAML.

# The fallback comment syntax to use if we can't determine the syntax from the
# language.
_FALLBACK_COMMENT = r"#.*$"

_C_STYLE_COMMENT = r"//.*$"

_LANGUAGE_OVERRIDES = {
  "C": {
    "comments": [_C_STYLE_COMMENT]
  },
  "C++": {
    "comments": [_C_STYLE_COMMENT]
  },
  "MarkDown": {
    "comments": [r"########.*$"],
    "marker": ""
  }
}

_FILE_OVERRIDES = {
  ".md": "MarkDown"
}

# Collection of information needed to process a file in a particular language.
class LanguageInfo(object):

  def __init__(self, lexer, marker, ignore, diagram):
    self.lexer = lexer
    self.marker = marker
    self.diagram = diagram
    if ignore is None:
      self.ignore = None
    else:
      self.ignore = re.compile(ignore)

  # Returns the pygments lexer to use for this language.
  def get_lexer(self):
    return self.lexer

  # Returns the marker used to identify the lines that contain documentation.
  def get_marker(self):
    return self.marker

  # Returns the marker used to identify lines that contain diagrams.
  def get_diagram(self):
    return self.diagram

  # Is the given line an end-of-line comment within this language?
  def is_comment(self, line):
    stripped = line.strip()
    return stripped.startswith(self.marker)

  def is_diagram(self, line):
    stripped = line.strip()
    return stripped.startswith(self.diagram)

  def is_ignored(self, line):
    if self.ignore is None:
      return False
    return self.ignore.match(line.strip())

  ## ### Determining the language
  ##
  ## The language for a file is determined from the filename using pygments.
  ## As it happens, this is incredibly slow for some reason. The lexer is then
  ## analyzed: we need to know what the end-of-line
  ## {{#DeterminingTheInitial}}(comment syntax) is and which
  ## {{#DeterminingTheMarker}}(marker) to use for literate comments.
  ## Alternatively, if any of these values have been specified explicitly we
  ## don't try to infer them but just use the overrides.

  # Tries to produce a language info for the source file with the given name
  # by querying pygments.
  @staticmethod
  def for_file(filename):
    (root, ext) = os.path.splitext(filename)
    file_override = _FILE_OVERRIDES.get(ext, None)
    if file_override is None:
      lexer = pygments.lexers.get_lexer_for_filename(filename)
      return LanguageInfo.for_lexer(lexer)
    else:
      return LanguageInfo.for_lexer(None, file_override)

  cache = {}

  # Returns a language descriptor for the given pygments lexer. If no language
  # can be constructed None is returned.
  @staticmethod
  def for_lexer(lexer, name=None):
    marker = None
    comments = []
    overrides = None
    ignore = None
    diagram = None
    if (name is None) and hasattr(lexer, 'name'):
      name = lexer.name
      # Now that we have the name check if we've created this language info
      # before.
      if name in LanguageInfo.cache:
        return LanguageInfo.cache[name]
      else:
        LanguageInfo.cache[name] = None
    # Prime the variables with the overrides -- if the values are set we don't
    # do any work below to try to infer them.
    overrides = _LANGUAGE_OVERRIDES.get(name, None)
    if not overrides is None:
      comments += [(c, False) for c in overrides.get("comments", [])]
      marker = overrides.get("marker", None)
      ignore = overrides.get("ignore", None)
      diagram = overrides.get("diagram", None)
    # In addition to the overrides we'll consider the lexer's tokens (if it has
    # any).
    if hasattr(lexer, 'tokens'):
      for matcher in LanguageInfo.flatten_list_dicts(lexer.tokens):
        if not isinstance(matcher, tuple):
          continue
        action = matcher[1]
        if action in pygments.token.Comment:
          comments.append((matcher[0], False))
    comments.append((_FALLBACK_COMMENT, True))
    initial = None
    for (comment, is_fallback) in comments:
      initial = LanguageInfo.initial_from_regexp(comment)
      if not initial is None:
        comments = initial
        if is_fallback:
          logging.info("Comment syntax couldn't be determined for %s. Using fallback '#'." % name)
        break
    if marker is None:
      marker = LanguageInfo.marker_from_initial(comments)
    if ignore is None:
      ignore = LanguageInfo.ignore_from_initial(comments)
    if diagram is None:
      diagram = LanguageInfo.diagram_from_initial(comments)
    result = LanguageInfo(lexer, marker, ignore, diagram)
    if not name is None:
      LanguageInfo.cache[name] = result
    return result

  # Given a dict whose values are lists returns a list that is the concatenation
  # of the list values. If some of the values are themselves list-dicts they are
  # traversed and flattened recursively.
  @staticmethod
  def flatten_list_dicts(root):
    result = []
    def add_values(value):
      if isinstance(value, dict):
        for (k, v) in value.items():
          add_values(v)
      else:
        for elm in value:
          result.append(elm)
    add_values(root)
    return result

  ## ### Determining the initial
  ##
  ## This is the cheesiest part, extracting the string that starts and
  ## end-of-line comment from a regexp that matches it. Generally they look
  ## the same (something like `#.*$` or `^\s*%.*\n` etc.) which can be matched
  ## by a regexp (yes, a meta-regexp). If we can't then it's fine to fall back
  ## on explicit defaults.

  # Given a regexp that is presumed to match end-of-line comments attempts to
  # extract the characters that start the comments. If this is not possible
  # None is returned.
  @staticmethod
  def initial_from_regexp(regexp):
    match = re.match(_INITIAL_REGEXP, regexp)
    if match:
      return match.group(1)
    return None

  ## ### Determining the marker
  ##
  ## This is also cheesy: it just doubles the last initial character. So if
  ## end-of-line comments start at `#` then the literate comment marker will be
  ## `##`. Again these are only defaults, longer term they can be overridden.

  # Given an end-of-line comment initial returns a documentation marker by just
  # doubling the last character.
  @staticmethod
  def marker_from_initial(initial):
    return initial + initial[-1]

  # Given an end-of-line comment initial returns an ignore marker. Just adds a
  # minus at the end.
  @staticmethod
  def ignore_from_initial(initial):
    return initial + "-"

  # Given an end-of-line comment initial returns a diagram marker.
  @staticmethod
  def diagram_from_initial(initial):
    return initial + "%"

# This is just the regexp used above split into separate parts to make it
# easier to read.
_COMMENT_PREFIX = r"^\^?(?:\\s\*)?"
_INITIAL_CHARS = r"((?:#|\\|/|\"|;|@|!|%)+)"
_COMMENT_SUFFIX = r"\.\*\??(?:\$|\\n)?$"
_INITIAL_REGEXP = re.compile(_COMMENT_PREFIX + _INITIAL_CHARS + _COMMENT_SUFFIX)

## ## Main
##
## The main class ties everything together (man). It handles
## {{#build_option_parser}}(flag parsing), runs the
## {{#RunningTheWatchdog}}(watchdog), and handles
## {{#WritingTheOutput}}(writing the output).

class Hemingway(object):

  LOG_FORMAT = "%(levelname)s: %(message)s"

  def __init__(self, args):
    parser = self.build_option_parser()
    self.options = parser.parse_args(args)
    self.initialize_logging()
    self.validate_options()
    self.index = SourceIndex(self.options.root, self.options.pattern)
    self.current_source = None
    self.scheduler = Scheduler(self)
    self.converter = Converter(self)

  # Configure logging appropriately.
  def initialize_logging(self):
    loglevel = self.options.log
    level_value = getattr(logging, loglevel.upper())
    logging.basicConfig(format=Hemingway.LOG_FORMAT, level=level_value)

  # Main entry-point for actually performing the conversion.
  def run(self):
    if self.options.profile:
      self.run_with_profile()
    else:
      self.run_no_profile()

  def run_with_profile(self):
    import cProfile
    import pstats
    profile = cProfile.Profile()
    profile.enable()
    try:
      self.run_no_profile()
    finally:
      profile.disable()
      stats = pstats.Stats(profile)
      stats.sort_stats('cumtime')
      stats.print_stats(32)

  # Does what .run says it does but where run also takes care of profiling this
  # one just does the conversion.
  def run_no_profile(self):
    self.index.load()
    for source in self.index.list_sources():
      self.scheduler.schedule(source)
    self.copy_assets()
    try:
      self.scheduler.start()
      if self.options.watchdog:
        logging.info("Watching for file changes under %s", self.options.root)
        self.run_watchdog()
    finally:
      self.scheduler.finish_up()

  ## ### Running the watchdog
  ##
  ## The watchdog is actually totally straightforward: all the work is done
  ## by the python `watchdog` library. For simplicity watchdog watches the
  ## whole root directory and then filters out files we're not interested in by
  ## asking the index whether it knows about files that have changed.
  ## On any changes the files are dropped into the scheduler's queue which does
  ## the rest.

  def run_watchdog(self):
    import time
    import watchdog
    import watchdog.observers
    import watchdog.events
    context = self
    class EventHandler(watchdog.events.FileSystemEventHandler):
      def on_any_event(self, event):
        context.on_watchdog_event(event)
    observer = watchdog.observers.Observer()
    observer.schedule(EventHandler(), self.options.root, recursive=True)
    observer.start()
    while True:
      time.sleep(1)

  # Fired by watchdog whenever a file changes.
  def on_watchdog_event(self, event):
    path = event.src_path
    source = self.index.get_absolute_source(path)
    if not source is None:
      self.scheduler.schedule(source)

  ## ### Writing the output
  ##
  ## While the conversion itself it delegated to the converter, writing the
  ## output to a file is done here. The source object knows where it will be
  ## written too (it needs to know to know how to resolve cross references)
  ## so all we have to do here is ensure that the file can be written and then
  ## do it.

  # Converts a source file, writing the results into the output file.
  def convert_file(self, source):
    logging.info("Processing %s", source)
    self.current_source = source
    markdown = self.converter.convert_source(source)
    output_file = source.get_absolute_output_path(self.options.out)
    self.ensure_parent_folder(output_file)
    port = open(output_file, "wt")
    try:
      port.write(markdown)
    finally:
      self.current_source = None
      port.close()

  # Given a file, ensures that its parent folder has been created.
  def ensure_parent_folder(self, path):
    parent = os.path.dirname(path)
    if not os.path.exists(parent):
      os.makedirs(parent)

  # Writes the assets into the output directory.
  def copy_assets(self):
    asset_path = os.path.join(self.options.out, 'assets')
    if not os.path.exists(asset_path):
      os.makedirs(asset_path)
    markdown = pkg_resources.resource_string(__name__, "assets/markdown-default.css")
    self.copy_asset(markdown, os.path.join(asset_path, "markdown.css"))
    diagram = pkg_resources.resource_string(__name__, "assets/diagram-default.css")
    self.copy_asset(diagram, os.path.join(asset_path, "diagram.css"))
    # Pygments comes with some style sheets so just copy one of those.
    formatter = pygments.formatters.HtmlFormatter(style=self.options.pygments_style)
    code = formatter.get_style_defs('.codehilite')
    self.copy_asset(code, os.path.join(asset_path, "code.css"))

  def copy_asset(self, source, target):
    logging.info("Creating asset %s", target)
    port = open(target, "wt")
    try:
      port.write(source)
    finally:
      port.close()

  # Returns any extra headers to insert into the HTML output.
  def get_headers(self):
    # Automatically enable refresh in watchdog mode.
    refresh = self.options.refresh
    if refresh is None and self.options.watchdog:
      refresh = "10"
    if refresh is None:
      return ""
    else:
      return "<meta http-equiv=\"refresh\" content=\"%s\"/>" % refresh

  # Returns the source file referred to by the given cross reference.
  def resolve_cross_reference(self, name):
    whos_asking = self.current_source
    return self.index.resolve_cross_reference(name, whos_asking)

  # Builds and returns a new option parser for the flags understood by this
  # script.
  def build_option_parser(self):
    parser = argparse.ArgumentParser()
    parser.add_argument('--out', default='doc',
      help="Root directory of the output. Default 'doc'.")
    parser.add_argument('--root', required=True,
      help="Root directory where the input lives.")
    parser.add_argument('--profile', default=False, action="store_true",
      help="Dump a profile on program exit.")
    parser.add_argument('--watchdog', default=False, action="store_true",
      help="Use watchdog to convert files as they change.")
    parser.add_argument('--refresh', default=None,
      help="If set, inserts a refresh meta-tag in the output html.")
    parser.add_argument('--pygments-style', default="default", dest="pygments_style",
      help="Which formatter style to use for the generated output")
    parser.add_argument('--log', default='INFO', help="Log level to use.")
    parser.add_argument('pattern', nargs='+',
      help="Globs (for instance '**/*.c') used to locate files under the root")
    return parser

  # Checks that the given options are valid.
  def validate_options(self):
    all_styles = list(pygments.styles.get_all_styles())
    if not self.options.pygments_style in all_styles:
      print "Unknown --pygments-style '%s'" % self.options.pygments_style
      print "The available styles are: %s" % ", ".join(all_styles)
      sys.exit(1)

def main():
  main = Hemingway(sys.argv[1:])
  try:
    main.run()
  except KeyboardInterrupt, ki:
    logging.info("Interrupted; exiting.")
    sys.exit(1)

## ## Unit tests.
##
## This function runs all the tests. Everything is wrapped in a function rather
## than at toplevel because the script shouldn't depend on unittest being
## available unless you're actually going to run the tests. This still adds some
## overhead to running script -- whether you run them or not they have to be
## parsed -- but it shouldn't be too bad.
##
## The {{#test_language_infos}} test is more for development than an actual
## test. Also, it takes a long time to run.

def please_run_the_tests():
  import unittest

  # Unit tests for hemingway.
  class HemingwayTest(unittest.TestCase):

    # Generates all the lexers known by pygment.
    def list_pygment_lexers(self):
      for lexer_tuple in pygments.lexers.get_all_lexers():
        long_name = lexer_tuple[0]
        aliases = lexer_tuple[1]
        all_names = [long_name] + list(aliases)
        for name in all_names:
          try:
            lexer = pygments.lexers.get_lexer_by_name(name)
            yield lexer
            break
          except pygments.lexers.ClassNotFound:
            continue

    def test_initial_from_regexp(self):
      def run_test(expected, input):
        found = LanguageInfo.initial_from_regexp(input)
        self.assertEquals(expected, found)
      run_test("#", "#.*$")
      run_test("#", "#.*\\n")
      run_test("#", "#.*?$")
      run_test("#", "#.*?\\n")
      run_test("#", "^#.*$")
      run_test("#", "^\\s*#.*$")
      run_test("#", "\\s*#.*$")
      run_test("##", "##.*$")
      run_test("//", "//.*$")
      run_test("\"", "^\\s*\".*")
      run_test(";", ";.*$")
      run_test("@", "^@.*$")
      run_test("!", "^!.*$")
      run_test("%", "^%.*\\n")

    def test_double_initial(self):
      def run_test(expected, input):
        found = LanguageInfo.marker_from_initial(input)
        self.assertEquals(expected, found)
      run_test("##", "#")
      run_test("///", "//")
      run_test("#//", "#/")

    def test_strip_prefix(self):
      def run_test(expected, marker, line):
        found = CommentBlock.strip_prefix(marker, line)
        self.assertEquals(expected, found)
      run_test("Foo", "#", "#Foo")
      run_test("Foo", "#", "# Foo")
      run_test(" Foo", "#", "#  Foo")
      run_test("#Foo", None, "#Foo")
      run_test("## Foo", "#", "### Foo")
      run_test("# Foo", "##", "### Foo")
      run_test("Foo", "###", "### Foo")
      run_test("## Foo", "###", "## Foo")

    def test_header_to_anchor_name(self):
      def run_test(expected, input):
        found = AutoAnchorProcessor.header_to_anchor_name(input)
        self.assertEquals(expected, found)
      run_test("Foo", "  Foo  ")
      run_test("FooBarBaz", "  Foo bar baz ")
      run_test("FooBar12", "  Foo bar 12 ")
      run_test("FooBarBaz", "Foo.bar.baz!")

    def test_language_infos(self):
      return
      for lexer in self.list_pygment_lexers():
        language = LanguageInfo.for_lexer(lexer)
        if language is None:
          continue
        if len(language.get_comment_regexes()) == 0:
          continue
        if language.initial is None:
          print lexer, language.get_comment_regexes()

  # For crying out loud, why must this be so complicated?!?
  suite = unittest.TestSuite()
  loader = unittest.TestLoader()
  suite.addTests(loader.loadTestsFromTestCase(HemingwayTest))
  suite.addTests(loader.loadTestsFromTestCase(linesboxes.diagram.get_unit_test_suite()))
  unittest.TextTestRunner().run(suite)

## ## Entry-point
##
## The main entry-point uses this cheesy trick to decide whether to run the
## tests. You might imagine a situation where you actually wanted to run the
## script with four files called `please`, `run`, `the`, and `tests` and then
## this wouldn't do what you want -- but even if the hack wasn't there you
## wouldn't get the effect you'd want because no `--root` has been specified so
## the program would fail.

if __name__ == '__main__':
  if sys.argv[1:] == ['please', 'run', 'the', 'tests']:
    please_run_the_tests()
  else:
    main()


text:
	if( mount_file_system_append_${mount_tool_file_system_type}(
	     mount_handle->file_system,
	     ${mount_tool_file_system_type_name},
	     error ) != 1 )
	{
		libcerror_error_set(
		 error,
		 LIBCERROR_ERROR_DOMAIN_RUNTIME,
		 LIBCERROR_RUNTIME_ERROR_APPEND_FAILED,
		 "%s: unable to append ${mount_tool_file_system_type_description} to file system.",
		 function );

		goto on_error;
	}


text:
<?php

$lang['no_changes_to_update']           = "No changes to update";
$lang['bank_detail_added']              = "Bank Details added successfully.";
$lang['bank_detail_updated']            = "Bank Details updated successfully.";
$lang['ssmfileExists']                  = "SSM File already exists. please contcat admin to update new one.";
$lang['idfileExists']                   = "Id Card File already exists. please contcat admin to update new one.";
$lang['company_detail_updated']         = "Company Details updated successfully.";
$lang['employee_created']               = "Employee created successfully";


text:
<?php //-->

use Cradle\Module\Utility\File;

/**
 * Process the Verification Page
 *
 * SIGNUP FLOW:
 * - GET /signup
 * - POST /signup
 * - EMAIL
 * - GET /activate/auth_id/hash
 * - GET /login
 *
 * VERIFY FLOW:
 * - GET /verify
 * - POST /verify
 * - EMAIL
 * - GET /activate/auth_id/hash
 * - GET /login
 *
 * @param Request $request
 * @param Response $response
 */
$cradle->get('/activate/:auth_id/:hash', function ($request, $response) {
    //get the detail
    cradle()->trigger('auth-detail', $request, $response);

    //form hash
    $authId = $response->getResults('auth_id');
    $authUpdated = $response->getResults('auth_updated');
    $hash = md5($authId.$authUpdated);

    //check the verification hash
    if ($hash !== $request->getStage('hash')) {
        cradle('global')->flash('Invalid verification. Try again.', 'danger');
        return cradle('global')->redirect('/verify');
    }

    //activate
    $request->setStage('auth_active', 1);

    //trigger the job
    cradle()->trigger('auth-update', $request, $response);

    if ($response->isError()) {
        cradle('global')->flash('Invalid verification. Try again.', 'danger');
        return cradle('global')->redirect('/verify');
    }

    //it was good
    //add a flash
    cradle('global')->flash('Activation Successful', 'success');

    //redirect
    cradle('global')->redirect('/login');
});

/**
 * Render the Signup Page
 *
 * SIGNUP FLOW:
 * - GET /signup
 * - POST /signup
 * - EMAIL
 * - GET /activate/auth_id/hash
 * - GET /login
 *
 * @param Request $request
 * @param Response $response
 */
$cradle->get('/signup', function ($request, $response) {
    //Prepare body
    $data = ['item' => $request->getPost()];

    //add CSRF
    cradle()->trigger('csrf-load', $request, $response);
    $data['csrf'] = $response->getResults('csrf');

    //add captcha
    cradle()->trigger('captcha-load', $request, $response);
    $data['captcha'] = $response->getResults('captcha');

    if ($response->isError()) {
        if ($response->getValidation('auth_slug')) {
            $message = $response->getValidation('auth_slug');
            $response->addValidation('profile_email', $message);
        }

        $response->setFlash($response->getMessage(), 'danger');
        $data['errors'] = $response->getValidation();
    }

    //Render body
    $class = 'page-auth-signup';
    $title = cradle('global')->translate('Sign Up');
    $body = cradle('/app/www')->template('signup', $data);

    //Set Content
    $response
        ->setPage('title', $title)
        ->setPage('class', $class)
        ->setContent($body);

    //Render blank page
}, 'render-www-blank');

/**
 * Render the Login Page
 *
 * @param Request $request
 * @param Response $response
 */
$cradle->get('/login', function ($request, $response) {
    //Prepare body
    $data = ['item' => $request->getPost()];

    //add CSRF
    cradle()->trigger('csrf-load', $request, $response);
    $data['csrf'] = $response->getResults('csrf');

    if ($response->isError()) {
        $response->setFlash($response->getMessage(), 'danger');

        $data['errors'] = $response->getValidation();
    }

    //Render body
    $class = 'page-auth-login';
    $title = cradle('global')->translate('Log In');
    $body = cradle('/app/www')->template('login', $data);

    //Set Content
    $response
        ->setPage('title', $title)
        ->setPage('class', $class)
        ->setContent($body);

    //Render blank page
}, 'render-www-blank');

/**
 * Process the Logout
 *
 * @param Request $request
 * @param Response $response
 */
$cradle->get('/logout', function ($request, $response) {
    //TODO: Sessions for clusters
    unset($_SESSION['me']);

    //add a flash
    cradle('global')->flash('Log Out Successful', 'success');

    //redirect
    $redirect = '/';
    if ($request->hasGet('redirect_uri')) {
        $redirect = $request->getGet('redirect_uri');
    }

    cradle('global')->redirect($redirect);
});

/**
 * Render the Account Page
 *
 * @param Request $request
 * @param Response $response
 */
$cradle->get('/account', function ($request, $response) {
    //Need to be logged in
    cradle('global')->requireLogin();

    //Prepare body
    $data = ['item' => $request->getPost()];

    //add CDN
    $config = $this->package('global')->service('s3-main');
    $data['cdn_config'] = File::getS3Client($config);

    //add CSRF
    cradle()->trigger('csrf-load', $request, $response);
    $data['csrf'] = $response->getResults('csrf');

    //If no post
    if (!$request->hasPost('profile_name')) {
        //set default data
        $data['item'] = $request->getSession('me');
    }

    if ($response->isError()) {
        $response->setFlash($response->getMessage(), 'danger');
        $data['errors'] = $response->getValidation();
    }

    //Render body
    $class = 'page-auth-account';
    $title = cradle('global')->translate('Account Settings');
    $body = cradle('/app/www')->template('account', $data);

    //Set Content
    $response
        ->setPage('title', $title)
        ->setPage('class', $class)
        ->setContent($body);

    //Render blank page
}, 'render-www-blank');

/**
 * Render the Forgot Page
 *
 * FORGOT FLOW:
 * - GET /forgot
 * - POST /forgot
 * - EMAIL
 * - GET /recover/auth_id/hash
 * - POST /recover/auth_id/hash
 * - GET /login
 *
 * @param Request $request
 * @param Response $response
 */
$cradle->get('/forgot', function ($request, $response) {
    //Prepare body
    $data = ['item' => $request->getPost()];

    //add CSRF
    cradle()->trigger('csrf-load', $request, $response);
    $data['csrf'] = $response->getResults('csrf');

    if ($response->isError()) {
        $response->setFlash($response->getMessage(), 'danger');
        $data['errors'] = $response->getValidation();
    }

    //Render body
    $class = 'page-auth-forgot';
    $title = cradle('global')->translate('Forgot Password');
    $body = cradle('/app/www')->template('forgot', $data);

    //Set Content
    $response
        ->setPage('title', $title)
        ->setPage('class', $class)
        ->setContent($body);

    //Render blank page
}, 'render-www-blank');

/**
 * Render the Recover Page
 *
 * FORGOT FLOW:
 * - GET /forgot
 * - POST /forgot
 * - EMAIL
 * - GET /recover/auth_id/hash
 * - POST /recover/auth_id/hash
 * - GET /login
 *
 * @param Request $request
 * @param Response $response
 */
$cradle->get('/recover/:auth_id/:hash', function ($request, $response) {
    //get the detail
    cradle()->trigger('auth-detail', $request, $response);

    //form hash
    $authId = $response->getResults('auth_id');
    $authUpdated = $response->getResults('auth_updated');
    $hash = md5($authId.$authUpdated);

    //check the verification hash
    if ($hash !== $request->getStage('hash')) {
        cradle('global')->flash('Invalid verification. Try again.', 'danger');
        return cradle('global')->redirect('/verify');
    }

    //Prepare body
    $data = ['item' => $request->getPost()];

    //add CSRF
    cradle()->trigger('csrf-load', $request, $response);
    $data['csrf'] = $response->getResults('csrf');

    if ($response->isError()) {
        $response->setFlash($response->getMessage(), 'danger');
        $data['errors'] = $response->getValidation();
    }

    //Render body
    $class = 'page-auth-recover';
    $title = cradle('global')->translate('Recover Password');
    $body = cradle('/app/www')->template('recover', $data);

    //Set Content
    $response
        ->setPage('title', $title)
        ->setPage('class', $class)
        ->setContent($body);

    //Render blank page
}, 'render-www-blank');

/**
 * Render the Verify Page
 *
 * VERIFY FLOW:
 * - GET /verify
 * - POST /verify
 * - EMAIL
 * - GET /activate/auth_id/hash
 * - GET /login
 *
 * @param Request $request
 * @param Response $response
 */
$cradle->get('/verify', function ($request, $response) {
    //Prepare body
    $data = ['item' => $request->getPost()];

    //add CSRF
    cradle()->trigger('csrf-load', $request, $response);
    $data['csrf'] = $response->getResults('csrf');

    if ($response->isError()) {
        $response->setFlash($response->getMessage(), 'danger');
        $data['errors'] = $response->getValidation();
    }

    //Render body
    $class = 'page-auth-verify';
    $title = cradle('global')->translate('Verify Account');
    $body = cradle('/app/www')->template('verify', $data);

    //Set Content
    $response
        ->setPage('title', $title)
        ->setPage('class', $class)
        ->setContent($body);

    //Render blank page
}, 'render-www-blank');

/**
 * Process the Account Page
 *
 * @param Request $request
 * @param Response $response
 */
$cradle->post('/account', function ($request, $response) {
    //need to be online
    cradle('global')->requireLogin();

    //csrf check
    cradle()->trigger('csrf-validate', $request, $response);

    if ($response->isError()) {
        return cradle()->triggerRoute('get', '/account', $request, $response);
    }

    //set the auth_id and profile_id
    $request->setStage('auth_id', $request->getSession('me', 'auth_id'));
    $request->setStage('profile_id', $request->getSession('me', 'profile_id'));
    $request->setStage('permission', $request->getSession('me', 'profile_id'));

    //remove password if empty
    if (!$request->getStage('auth_password')) {
        $request->removeStage('auth_password');
    }

    if (!$request->getStage('confirm')) {
        $request->removeStage('confirm');
    }

    //trigger the job
    cradle()->trigger('auth-update', $request, $response);

    if ($response->isError()) {
        return cradle()->triggerRoute('get', '/account', $request, $response);
    }

    //it was good
    //update the session
    cradle()->trigger('auth-detail', $request, $response);
    $_SESSION['me'] = $response->getResults();

    //add a flash
    cradle('global')->flash('Update Successful', 'success');

    //redirect
    $redirect = '/';
    if ($request->hasGet('redirect_uri')) {
        $redirect = $request->getGet('redirect_uri');
    }

    cradle('global')->redirect($redirect);
});

/**
 * Process the Login Page
 *
 * @param Request $request
 * @param Response $response
 */
$cradle->post('/login', function ($request, $response) {
    //csrf check
    cradle()->trigger('csrf-validate', $request, $response);

    if ($response->isError()) {
        return cradle()->triggerRoute('get', '/login', $request, $response);
    }

    //call the job
    cradle()->trigger('auth-login', $request, $response);

    if ($response->isError()) {
        return cradle()->triggerRoute('get', '/login', $request, $response);
        ;
    }

    //it was good

    //store to session
    //TODO: Sessions for clusters
    $_SESSION['me'] = $response->getResults();

    //redirect
    if ($request->hasGet('redirect')) {
        return cradle('global')->redirect($request->getGet('redirect'));
    }

    $redirect = '/';
    if ($request->hasGet('redirect_uri')) {
        $redirect = $request->getGet('redirect_uri');
    }

    return cradle('global')->redirect($redirect);
});

$cradle->post('/loginAccount', function ($request, $response) {
    //csrf check
    cradle()->trigger('csrf-validate', $request, $response);

    if ($response->isError()) {
        return cradle()->triggerRoute('get', '/login', $request, $response);
    }

    //call the job
    cradle()->trigger('auth-login', $request, $response);

    if ($response->isError()) {
        return cradle()->triggerRoute('get', '/login', $request, $response);
        ;
    }

    //it was good

    //store to session
    //TODO: Sessions for clusters
    $_SESSION['me'] = $response->getResults();

    //redirect
    if ($request->hasGet('redirect')) {
        return cradle('global')->redirect($request->getGet('redirect'));
    }

    $redirect = '/';
    if ($request->hasGet('redirect_uri')) {
        $redirect = $request->getGet('redirect_uri');
    }

    return cradle('global')->redirect($redirect);
});

/**
 * Process the Forgot Page
 *
 * FORGOT FLOW:
 * - GET /forgot
 * - POST /forgot
 * - EMAIL
 * - GET /recover/auth_id/hash
 * - POST /recover/auth_id/hash
 * - GET /login
 *
 * @param Request $request
 * @param Response $response
 */
$cradle->post('/forgot', function ($request, $response) {
    //csrf check
    cradle()->trigger('csrf-validate', $request, $response);

    if ($response->isError()) {
        return cradle()->triggerRoute('get', '/forgot', $request, $response);
    }

    //trigger the job
    cradle()->trigger('auth-forgot', $request, $response);

    if ($response->isError()) {
        return cradle()->triggerRoute('get', '/forgot', $request, $response);
    }

    //its good
    $response->setFlash('An email with recovery instructions will be sent in a few minutes.', 'success');
    cradle()->triggerRoute('get', '/forgot', $request, $response);
});

/**
 * Process the Recover Page
 *
 * FORGOT FLOW:
 * - GET /forgot
 * - POST /forgot
 * - EMAIL
 * - GET /recover/auth_id/hash
 * - POST /recover/auth_id/hash
 * - GET /login
 *
 * @param Request $request
 * @param Response $response
 */
$cradle->post('/recover/:auth_id/:hash', function ($request, $response) {
    //get the detail
    cradle()->trigger('auth-detail', $request, $response);

    //form hash
    $authId = $response->getResults('auth_id');
    $authUpdated = $response->getResults('auth_updated');
    $hash = md5($authId.$authUpdated);

    //check the recovery hash
    if ($hash !== $request->getStage('hash')) {
        cradle('global')->flash('This recovery page is expired. Please try again.', 'danger');
        return cradle('global')->redirect('/forgot');
    }

    //csrf check
    cradle()->trigger('csrf-validate', $request, $response);

    if ($response->isError()) {
        $redirect = '/recover/' . $authId . '/' . $hash;
        return cradle()->triggerRoute('get', $redirect, $request, $response);
    }

    //trigger the job
    cradle()->trigger('auth-recover', $request, $response);

    if ($response->isError()) {
        $redirect = '/recover/' . $authId . '/' . $hash;
        return cradle()->triggerRoute('get', $redirect, $request, $response);
    }

    //it was good
    //add a flash
    cradle('global')->flash('Recovery Successful', 'success');

    //redirect
    cradle('global')->redirect('/login');
});

/**
 * Process the Signup Page
 *
 * SIGNUP FLOW:
 * - GET /signup
 * - POST /signup
 * - EMAIL
 * - GET /activate/auth_id/hash
 * - GET /login
 *
 * @param Request $request
 * @param Response $response
 */
$cradle->post('/signup', function ($request, $response) {
    //csrf check
    cradle()->trigger('csrf-validate', $request, $response);

    if ($response->isError()) {
        return cradle()->triggerRoute('get', '/signup', $request, $response);
    }

    //captcha check
    cradle()->trigger('captcha-validate', $request, $response);

    if ($response->isError()) {
        return cradle()->triggerRoute('get', '/signup', $request, $response);
    }

    //set defaults
    if (!$request->hasStage('auth_permissions')) {
        $request->setStage('auth_permissions', [
            'public_profile',
            'personal_profile'
        ]);
    }

    //trigger the job
    cradle()->trigger('auth-create', $request, $response);

    if ($response->isError()) {
        return cradle()->triggerRoute('get', '/signup', $request, $response);
    }

    //it was good
    //add a flash
    cradle('global')->flash('Sign Up Successful. Please check your email for verification process.', 'success');

    //redirect
    $query = http_build_query($request->get('get'));
    cradle('global')->redirect('/login?' . $query);
});

/**
 * Process the Verify Page
 *
 * VERIFY FLOW:
 * - GET /verify
 * - POST /verify
 * - EMAIL
 * - GET /activate/auth_id/hash
 * - GET /login
 *
 * @param Request $request
 * @param Response $response
 */
$cradle->post('/verify', function ($request, $response) {
    //csrf check
    cradle()->trigger('csrf-validate', $request, $response);

    if ($response->isError()) {
        return cradle()->triggerRoute('get', '/verify', $request, $response);
    }

    //trigger the job
    cradle()->trigger('auth-verify', $request, $response);

    if ($response->isError()) {
        return cradle()->triggerRoute('get', '/verify', $request, $response);
    }

    //its good
    $response->setFlash('An email with verification instructions will be sent in a few minutes.', 'success');
    cradle()->triggerRoute('get', '/verify', $request, $response);
});

$cradle->post('/register', function ($request, $response) {
    //need to be online
    //cradle('global')->requireLogin();

    //csrf check
    //cradle()->trigger('csrf-validate', $request, $response);

    if ($response->isError()) {
        return cradle()->triggerRoute('get', '/provinces', $request, $response);
        return cradle()->triggerRoute('get', '/cities', $request, $response);
        return cradle()->triggerRoute('get', '/accounttypes', $request, $response);
    }

/*    if (!$request->getStage('confirm')) {
        $request->removeStage('confirm');
    }
*/
    //trigger the job
    cradle()->trigger('provinces', $request, $response);
    cradle()->trigger('cities', $request, $response);
    cradle()->trigger('accounttypes', $request, $response);

    if ($response->isError()) {
        return cradle()->triggerRoute('get', '/register', $request, $response);
    }

    cradle('global')->redirect($redirect);
});

$cradle->post('/supplier', function ($request, $response) {
    //need to be online
    //cradle('global')->requireLogin();

    //csrf check
    //cradle()->trigger('csrf-validate', $request, $response);

    if ($response->isError()) {
        return cradle()->triggerRoute('get', '/supplier', $request, $response);
    }

    //trigger the job
    cradle()->trigger('supplier', $request, $response);

    if ($response->isError()) {
        return cradle()->triggerRoute('get', '/supplier', $request, $response);
    }

    cradle('global')->redirect($redirect);
});

$cradle->post('/sampleretrieve', function ($request, $response) {
    //need to be online
    //cradle('global')->requireLogin();

    //csrf check
    //cradle()->trigger('csrf-validate', $request, $response);

    if ($response->isError()) {
        return cradle()->triggerRoute('get', '/sample', $request, $response);
    }

    //trigger the job
    cradle()->trigger('sample', $request, $response);

    if ($response->isError()) {
        return cradle()->triggerRoute('get', '/sample', $request, $response);
    }

    cradle('global')->redirect($redirect);
});


text:
import {expect} from "chai";
import Sinon from "sinon";
import {JsonMapperContext} from "../domain/JsonMapperContext";
import {SetMapper} from "./SetMapper";

describe("SetMapper", () => {
  describe("deserialize()", () => {
    it("should return value", () => {
      const ctx = new JsonMapperContext({
        type: String,
        collectionType: Set,
        next: Sinon.stub().callsFake((o) => String("testMap")),
        options: {}
      });

      const mapper = new SetMapper();

      const value = mapper.deserialize(["test"], ctx);

      expect(value).to.be.instanceof(Set);
      expect(Array.from(value.values())).to.deep.eq(["testMap"]);
    });
  });
  describe("serialize()", () => {
    it("should return value", () => {
      const ctx = {
        type: String,
        collectionType: Set,
        next: Sinon.stub().callsFake((o) => String("testMap"))
      };

      const arrayMapper = new SetMapper();

      const value = arrayMapper.serialize(new Set(["test"]), ctx);

      expect(value).to.deep.eq(["testMap"]);
    });
  });
});


text:
NaClSrpcError PppUpcallRpcClient::PPB_Core_CallOnMainThread(
    NaClSrpcChannel* channel,
    int32_t closure_number,
    int32_t delay_in_milliseconds
)  {
  NaClSrpcError retval;
  retval = NaClSrpcInvokeBySignature(
      channel,
      "PPB_Core_CallOnMainThread:ii:",
      closure_number,
      delay_in_milliseconds
  );
  return retval;
}

NaClSrpcError PppUpcallRpcClient::PPB_Graphics2D_Flush(
    NaClSrpcChannel* channel,
    int64_t device_context,
    int32_t callback_index,
    int32_t* result
)  {
  NaClSrpcError retval;
  retval = NaClSrpcInvokeBySignature(
      channel,
      "PPB_Graphics2D_Flush:li:i",
      device_context,
      callback_index,
      result
  );
  return retval;
}


text:
package org.apache.druid.segment.data;

import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
import org.apache.druid.segment.ColumnValueSelector;
import org.apache.druid.segment.LongColumnSelector;
import org.apache.druid.segment.historical.HistoricalColumnSelector;

import java.io.Closeable;

/**
 * Resource that provides random access to a packed array of primitive longs. Backs up {@link
 * org.apache.druid.segment.column.LongsColumn}.
 */
public interface ColumnarLongs extends Closeable
{
  int size();

  long get(int index);

  void fill(int index, long[] toFill);

  @Override
  void close();

  default ColumnValueSelector<Long> makeColumnValueSelector(ReadableOffset offset, ImmutableBitmap nullValueBitmap)
  {
    if (nullValueBitmap.isEmpty()) {
      class HistoricalLongColumnSelector implements LongColumnSelector, HistoricalColumnSelector<Long>
      {
        @Override
        public boolean isNull()
        {
          return false;
        }

        @Override
        public long getLong()
        {
          return ColumnarLongs.this.get(offset.getOffset());
        }

        @Override
        public double getDouble(int offset)
        {
          return ColumnarLongs.this.get(offset);
        }

        @Override
        public void inspectRuntimeShape(RuntimeShapeInspector inspector)
        {
          inspector.visit("columnar", ColumnarLongs.this);
          inspector.visit("offset", offset);
        }
      }
      return new HistoricalLongColumnSelector();
    } else {
      class HistoricalLongColumnSelectorWithNulls implements LongColumnSelector, HistoricalColumnSelector<Long>
      {
        @Override
        public boolean isNull()
        {
          return nullValueBitmap.get(offset.getOffset());
        }

        @Override
        public long getLong()
        {
          assert NullHandling.replaceWithDefault() || !isNull();
          return ColumnarLongs.this.get(offset.getOffset());
        }

        @Override
        public double getDouble(int offset)
        {
          assert NullHandling.replaceWithDefault() || !nullValueBitmap.get(offset);
          return ColumnarLongs.this.get(offset);
        }

        @Override
        public void inspectRuntimeShape(RuntimeShapeInspector inspector)
        {
          inspector.visit("columnar", ColumnarLongs.this);
          inspector.visit("offset", offset);
          inspector.visit("nullValueBitmap", nullValueBitmap);
        }
      }
      return new HistoricalLongColumnSelectorWithNulls();
    }
  }
}


text:
using System;
using System.Linq;
using KodiRemote.Uwp.Core;
using Windows.ApplicationModel.Resources;
using Windows.UI.Popups;
using Windows.UI.Xaml;
using Windows.UI.Xaml.Controls;
using Windows.UI.Xaml.Navigation;

namespace KodiRemote.Uwp
{
    public sealed partial class PageSettings : Page
    {
        private bool _newConnection;
        private KodiConnection _connection;

        public PageSettings()
        {
            InitializeComponent();
            NavigationCacheMode = NavigationCacheMode.Required;
        }

        #region Address

        public string Address
        {
            get { return (string)GetValue(AddressProperty); }
            private set { SetValue(AddressProperty, value); }
        }

        public static readonly DependencyProperty AddressProperty =
            DependencyProperty.Register(nameof(Address), typeof(string), typeof(PageSettings), new PropertyMetadata(null));

        #endregion

        #region Port

        public string Port
        {
            get { return (string)GetValue(PortProperty); }
            private set { SetValue(PortProperty, value); }
        }

        public static readonly DependencyProperty PortProperty =
            DependencyProperty.Register(nameof(Port), typeof(string), typeof(PageSettings), new PropertyMetadata(null));

        #endregion

        #region Login

        public string Login
        {
            get { return (string)GetValue(LoginProperty); }
            private set { SetValue(LoginProperty, value); }
        }

        public static readonly DependencyProperty LoginProperty =
            DependencyProperty.Register(nameof(Login), typeof(string), typeof(PageSettings), new PropertyMetadata(null));

        #endregion

        #region Password

        public string Password
        {
            get { return (string)GetValue(PasswordProperty); }
            private set { SetValue(PasswordProperty, value); }
        }

        public static readonly DependencyProperty PasswordProperty =
            DependencyProperty.Register(nameof(Password), typeof(string), typeof(PageSettings), new PropertyMetadata(null));

        #endregion

        #region MacAddress

        public string MacAddress
        {
            get { return (string)GetValue(MacAddressProperty); }
            private set { SetValue(MacAddressProperty, value); }
        }

        public static readonly DependencyProperty MacAddressProperty =
            DependencyProperty.Register(nameof(MacAddress), typeof(string), typeof(PageSettings), new PropertyMetadata(null));

        #endregion

        #region IsLoading

        public bool IsLoading
        {
            get { return (bool)GetValue(IsLoadingProperty); }
            private set { SetValue(IsLoadingProperty, value); }
        }

        public static readonly DependencyProperty IsLoadingProperty =
            DependencyProperty.Register(nameof(IsLoading), typeof(bool), typeof(PageSettings), new PropertyMetadata(false));

        #endregion

        protected override void OnNavigatedTo(NavigationEventArgs e)
        {
            if (Windows.Foundation.Metadata.ApiInformation.IsTypePresent("Windows.UI.ViewManagement.StatusBar"))
            {
                var statusbar = Windows.UI.ViewManagement.StatusBar.GetForCurrentView();
                statusbar.BackgroundColor = new Windows.UI.Color() { R = 40, G = 42, B = 43 };
                statusbar.BackgroundOpacity = 1;
                statusbar.ForegroundColor = Windows.UI.Colors.White;
            }

            string parameter = e.Parameter.ToString();
            if ("new".Equals(parameter, StringComparison.OrdinalIgnoreCase))
            {
                _newConnection = true;
                _connection = new KodiConnection { Kodi = KodiRemote.Core.Connection.Default() };
            }
            else
            {
                _newConnection = false;
                _connection = App.Context.Connections.FirstOrDefault(c => c.Id.Equals(parameter, StringComparison.OrdinalIgnoreCase));
            }

            Address = _connection.Kodi.Address;
            Port = _connection.Kodi.Port;
            Login = _connection.Kodi.Login;
            Password = _connection.Kodi.Password;
            MacAddress = _connection.Kodi.MacAddress;

            DataContext = this;
        }

        private bool AreInformationValid()
        {
            if (string.IsNullOrWhiteSpace(Address) || string.IsNullOrWhiteSpace(Port))
                return false;

            _connection.Kodi.Address = Address;
            _connection.Kodi.Port = Port;
            _connection.Kodi.Login = Login;
            _connection.Kodi.Password = Password;
            _connection.Kodi.MacAddress = MacAddress;

            return true;
        }

        private async void TestButtonClick(object sender, RoutedEventArgs e)
        {
            if (!AreInformationValid())
                return;

            IsLoading = true;

            try
            {
                var result = await _connection.TestConnectionAsync();

                var resourceLoader = ResourceLoader.GetForCurrentView();
                string message = result ? resourceLoader.GetString("/settings/TestGood") : resourceLoader.GetString("/settings/TestBad");
                var dialog = new MessageDialog(message, resourceLoader.GetString("/settings/ConnectivityTest.Content"));
                await dialog.ShowAsync();

                App.Context.Save();
            }
            finally
            {
                IsLoading = false;
            }
        }

        private void SaveButtonClick(object sender, RoutedEventArgs e)
        {
            if (!AreInformationValid())
                return;

            if (_newConnection)
                App.Context.Connections.Add(_connection);

            App.Context.Save();

            if (Frame.CanGoBack)
                Frame.GoBack();
        }
    }
}