text:
package org.apache.derbyBuild;

import java.io.*;
import java.net.URL;
import java.util.ArrayList;
import javax.xml.parsers.*;
import javax.xml.transform.*;
import javax.xml.transform.dom.*;
import javax.xml.transform.stream.*;
import org.w3c.dom.*;

import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.Project;
import org.apache.tools.ant.Task;
import org.apache.tools.ant.taskdefs.Echo;

/**
 *

 * This tool generates Release Notes for a Derby release. See the USAGE
 * constant for details on how to run this tool standalone. It is recommended
 * that you freshly regenerate your BUG_LIST and NOTES_LIST just before
 * you run this tool.
 *

 *
 *

 * The tool is designed to be run from Derby's ant build scripts. To run under
 * ant, do the following:
 *

 *
 *

 *
Define the "relnotes.src.reports" variable in your ant.properties. This
 * variable points at the directory which holds your xml JIRA reports.

 *
Put your xml JIRA reports in that directory. They should have the
 * following names:
 *

 *
fixedBugsList.xml - This is the list of issues addressed by the release.

 *
releaseNotesList.xml - This is the list of issues which have detailed
 *  release notes.

 *

 *

 *
Then cd to tools/release and run ant thusly: "ant genrelnotes"

 *

 *
 *
For more information on this tool, please see the JIRA which introduced it:
 *

 *
 *

 * DERBY-2570
 *

 *
 *

 * The format of the html reports changes over time. You may need to adjust this
 * tool for each release.
 *

 *
 */
public class ReleaseNotesGenerator extends Task
{
    /////////////////////////////////////////////////////////////////////////
    //
    //  CONSTANTS
    //
    /////////////////////////////////////////////////////////////////////////

    private static  final   String  USAGE =
        "Usage:\n" +
        "\n" +
        "  java org.apache.derbyBuild.ReleaseNotesGenerator SUMMARY BUG_LIST NOTES_LIST OUTPUT_PAMPHLET\n" +
        "\n" +
        "    where\n" +
        "                  SUMMARY                    Summary, a filled-in copy of releaseSummaryTemplate.xml.\n" +
        "                  BUG_LIST                     An xml JIRA report of issues addressed by this release.\n" +
        "                  NOTES_LIST                An xml JIRA report listing issues which have detailed releaseNotes.html attachments.\n" +
        "                  OUTPUT_PAMPHLET  The output file to generate, typically RELEASE-NOTES.html.\n" +
        "\n" +
        "The ReleaseNoteGenerator attempts to connect to issues.apache.org in\n" +
        "order to read the detailed release notes that have been clipped to\n" +
        "individual JIRAs. Before running this program, make sure that you can\n" +
        "ping issues.apache.org.\n" +
        "\n" +
        "The ReleaseNoteGenerator assumes that the two JIRA reports contain\n" +
        "key and title information for each Derby issue. For each\n" +
        "issue in NOTES_LIST, the ReleaseNotesGenerator looks through the\n" +
        "attachments clipped to that issue and grabs the latest reported\n" +
        "releaseNote.html.\n" +
        "\n" +
        "In order to parse the xml files, you may need to remove the\n" +
        "introductory DOCTYPE preamble because it refers to a DTD which does\n" +
        "not live in your local file system.\n" +
        "\n" +
        "For this reason, it is recommended that you freshly generate BUG_LIST\n" +
        "and NOTES_LIST just before you run this tool.\n"
        ;

    // header levels
    private static  final   int     BANNER_LEVEL = 1;
    private static  final   int     MAIN_SECTION_LEVEL = BANNER_LEVEL + 1;
    private static  final   int     ISSUE_DETAIL_LEVEL = MAIN_SECTION_LEVEL + 1;

    // major sections
    private static  final   String  OVERVIEW_SECTION = "Overview";
    private static  final   String  NEW_FEATURES_SECTION = "New Features";
    private static  final   String  BUG_FIXES_SECTION = "Bug Fixes";
    private static  final   String  ISSUES_SECTION = "Issues";
    private static  final   String  BUILD_ENVIRONMENT_SECTION = "Build Environment";

    // headlines
    private static  final   String  ANT_HEADLINE = "Ant";
    private static  final   String  BRANCH_HEADLINE = "Branch";
    private static  final   String  COMPILER_HEADLINE = "Compiler";
    private static  final   String  DESCRIPTION_HEADLINE = "Description";
    private static  final   String  ISSUE_ID_HEADLINE = "Issue Id";
    private static  final   String  JAVA6_HEADLINE = "Java 6";
    private static  final   String  JDK14_HEADLINE = "JDK 1.4";
    private static  final   String  JSR169_HEADLINE = "JSR 169";
    private static  final   String  MACHINE_HEADLINE = "Machine";
    private static  final   String  OSGI_HEADLINE = "OSGi";

    // formatting tags
    private static  final   String  ANCHOR = "a";
    private static  final   String  BODY = "body";
    private static  final   String  BOLD = "b";
    private static  final   String  BORDER = "border";
    private static  final   String  COLUMN = "td";
    private static  final   String  HORIZONTAL_LINE = "hr";
    private static  final   String  HTML = "html";
    private static  final   String  INDENT = "blockquote";
    private static  final   String  LIST = "ul";
    private static  final   String  LIST_ELEMENT = "li";
    private static  final   String  NAME = "name";
    private static  final   String  PARAGRAPH = "p";
    private static  final   String  ROW = "tr";
    private static  final   String  TABLE = "table";

    // tags in summary xml
    private static  final   String  SUM_ANT_VERSION = "antVersion";
    private static  final   String  SUM_BRANCH = "branch";
    private static  final   String  SUM_COMPILER = "compilers";
    private static  final   String  SUM_JAVA6 = "java6";
    private static  final   String  SUM_JDK14 = "jdk1.4";
    private static  final   String  SUM_JSR169 = "jsr169";
    private static  final   String  SUM_MACHINE = "machine";
    private static  final   String  SUM_NEW_FEATURES = "newFeatures";
    private static  final   String  SUM_OSGI = "osgi";
    private static  final   String  SUM_OVERVIEW = "overview";
    private static  final   String  SUM_PREVIOUS_RELEASE_ID = "previousReleaseID";
    private static  final   String  SUM_RELEASE_ID = "releaseID";

    // tags in JIRA reports
    private static  final   String  JIRA_ITEM = "h3";

    // positioning strings inside a JIRA
    private static  final   String  ATTACHMENT_START = "/jira/secure/attachment/";
    private static  final   String  ATTACHMENT_VERSION_END = "/";
    private static  final   String  ATTACHMENT_END = "\"";
    private static  final   String  ATTACHMENT_PREFIX = "https://issues.apache.org";

    // managing releaseNote.html
    private static  final   String  RN_SUMMARY_OF_CHANGE = "Summary of Change";

    // other JIRA control
    private static  final   String  RELEASE_NOTE_NAME = "releaseNote.html";
    private static  final   String  RN_H4 = "h4";

    // other html control
    private static  final   int     DEFAULT_TABLE_BORDER_WIDTH = 2;
    private static  final   String  XML_DECLARATION = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n";

    /////////////////////////////////////////////////////////////////////////
    //
    //  STATE
    //
    /////////////////////////////////////////////////////////////////////////

    // set on the command line or by ant
    private String  _summaryFileName;
    private String  _bugListFileName;
    private String  _releaseNotesListFileName;
    private String  _pamphletFileName;

    // computed at run time
    private String  _releaseID;
    private String  _previousReleaseID;

    // misc
    private boolean _invokedByAnt = true;

    /////////////////////////////////////////////////////////////////////////
    //
    //  INNER CLASSES
    //
    /////////////////////////////////////////////////////////////////////////

    /**
     *

     * State used by the generator
     *

     */
    public  static  class   GeneratorState
    {
        private DocumentBuilder _documentBuilder;
        private Document    _pamphlet;
        private Document    _summary;
        private Document    _bugList;
        private Document    _releaseNotesList;
        private ReleaseNoteReader   _releaseNoteReader;

        private ArrayList       _missingReleaseNotes;
        private ArrayList       _errors;

        public  GeneratorState
            (
             DocumentBuilder    documentBuilder,
             Document   pamphlet,
             Document   summary,
             Document   bugList,
             Document   releaseNotesList
             )
        {
            _documentBuilder = documentBuilder;
            _pamphlet = pamphlet;
            _summary = summary;
            _bugList = bugList;
            _releaseNotesList = releaseNotesList;

            _missingReleaseNotes = new ArrayList();
            _errors = new ArrayList();

            _releaseNoteReader = new ReleaseNoteReader( documentBuilder );
        }

        public  void    addMissingReleaseNote( JiraIssue issue )
        {
            _missingReleaseNotes.add( issue );
        }

        public  void    addError( String message )
        {
            _errors.add( message );
        }

        public  DocumentBuilder getDocumentBuilder() { return _documentBuilder; }
        public  Document    getPamphlet() { return _pamphlet; }
        public  Document    getSummary() { return _summary; }
        public  Document    getBugList() { return _bugList; }
        public  Document    getReleaseNotesList() { return _releaseNotesList; }
        public  ReleaseNoteReader   getReleaseNoteReader() { return _releaseNoteReader; }

        public  JiraIssue[]     getMissingReleaseNotes()
        {
            JiraIssue[]     missingNotes = new JiraIssue[ _missingReleaseNotes.size() ];

            _missingReleaseNotes.toArray( missingNotes );

            return missingNotes;
        }

        public  String[]     getErrors()
        {
            String[]        squeezed = new String[ _errors.size() ];

            _errors.toArray( squeezed );

            return squeezed;
        }
    }

    /**
     *

     * An issue from a JIRA report.
     *

     */
    public  static  class   JiraIssue
    {
        private String  _key;
        private String  _title;

        public  JiraIssue
            (
             String key,
             String title
             )
        {
            _key = key;
            _title = title;
        }

        public  String  getKey() { return _key; }
        public  String  getTitle() { return _title; }

        public  String  getJiraAddress()
        {
            return "http://issues.apache.org/jira/browse/" + _key;
        }
    }

    /////////////////////////////////////////////////////////////////////////
    //
    //  CONSTRUCTORS
    //
    /////////////////////////////////////////////////////////////////////////

    /////////////////////////////////////////////////////////////////////////
    //
    //  ENTRY POINT
    //
    /////////////////////////////////////////////////////////////////////////

    /**
     *

     * Generate the release notes (for details on how to invoke this tool, see
     * the header comment on this class).
     *

     */
    public  static  void    main( String[] args )
        throws Exception
    {
        ReleaseNotesGenerator   me = new ReleaseNotesGenerator();

        me._invokedByAnt = false;

        if ( me.parseArgs( args ) ) { me.execute(); }
        else { me.printUsage(); }
    }

    /////////////////////////////////////////////////////////////////////////
    //
    //  ANT Task BEHAVIOR
    //
    /////////////////////////////////////////////////////////////////////////

    /** Ant accessor to set the name of the summary file prepared by the Release Manager */
    public   void    setSummaryFileName( String summaryFileName ) { _summaryFileName = summaryFileName; }

    /** Ant accessor to set the name of the JIRA-generated list of bugs addressed by this release */
    public   void    setBugListFileName( String bugListFileName ) { _bugListFileName = bugListFileName; }

    /** Ant accessor to set the name of the JIRA-generated list of bugs which have release notes */
    public   void    setReleaseNotesListFileName( String releaseNotesListFileName ) { _releaseNotesListFileName = releaseNotesListFileName; }

    /** Ant accessor to set the name of the generated output file */
    public   void    setPamphletFileName( String pamphletFileName ) { _pamphletFileName = pamphletFileName; }

    /**
     *

     * This is Ant's entry point into this task.
     *

     */
    public  void    execute()
        throws BuildException
    {
        try {
            GeneratorState                  gs = initialize();

            beginPamphlet( gs );
            buildOverview( gs );
            buildNewFeatures( gs );
            buildBugList( gs );
            buildIssuesList( gs );
            buildEnvironment( gs );
            replaceVariables( gs );
            endPamphlet( gs );

            printPamphlet( gs );

            printMissingReleaseNotes( gs );
            printErrors( gs );
        }
        catch ( Throwable t )
        {
            t.printStackTrace();

            throw new BuildException( "Error running ReleaseNotesGenerator: " + t.getMessage(), t );
        }
    }

    /////////////////////////////////////////////////////////////////////////
    //
    //  OTHER ACCESSORS
    //
    /////////////////////////////////////////////////////////////////////////

    /**
     *

     * Get the release ID from the summary file.
     *

     */
    private String getReleaseID( GeneratorState gs)
        throws Exception
    {
        if ( _releaseID == null )
        {
            Document    summary = gs.getSummary();
            Element     summaryRoot = summary.getDocumentElement();

            _releaseID = squeezeText( getFirstChild( summaryRoot, SUM_RELEASE_ID ) );
        }

        return _releaseID;
    }

    /**
     *

     * Get the previous release ID from the summary file.
     *

     */
    private String getPreviousReleaseID( GeneratorState gs)
        throws Exception
    {
        if ( _previousReleaseID == null )
        {
            Document    summary = gs.getSummary();
            Element     summaryRoot = summary.getDocumentElement();

            _previousReleaseID = squeezeText( getFirstChild( summaryRoot, SUM_PREVIOUS_RELEASE_ID ) );
        }

        return _previousReleaseID;
    }

    /////////////////////////////////////////////////////////////////////////
    //
    //  MINIONS
    //
    /////////////////////////////////////////////////////////////////////////

    //////////////////////////////////
    //
    //  Boilerplate
    //
    //////////////////////////////////

    /**
     *

     * Start an html docment. Returns the body element.
     *

     */
    private void beginPamphlet( GeneratorState gs )
        throws Exception
    {
        Document    pamphlet = gs.getPamphlet();
        String          releaseID = getReleaseID( gs );
        String          titleText = "Release Notes for Derby " + releaseID;
        Element     html = pamphlet.createElement( HTML );
        Element     title = createTextElement( pamphlet, "title", titleText );
        Element     body = pamphlet.createElement( BODY );

        pamphlet.appendChild( html );
        html.appendChild( title );
        html.appendChild( body );

        Element     bannerBlock = createHeader( body, BANNER_LEVEL, titleText );
        buildDelta( gs, bannerBlock );

        Element     toc = createList( body );

        createSection( body, MAIN_SECTION_LEVEL, toc, OVERVIEW_SECTION, OVERVIEW_SECTION );
        createSection( body, MAIN_SECTION_LEVEL, toc, NEW_FEATURES_SECTION, NEW_FEATURES_SECTION );
        createSection( body, MAIN_SECTION_LEVEL, toc, BUG_FIXES_SECTION, BUG_FIXES_SECTION );
        createSection( body, MAIN_SECTION_LEVEL, toc, ISSUES_SECTION, ISSUES_SECTION );
        createSection( body, MAIN_SECTION_LEVEL, toc, BUILD_ENVIRONMENT_SECTION, BUILD_ENVIRONMENT_SECTION );
    }

    /**
     *

     * Finish the html document.
     *

     */
    private void    endPamphlet( GeneratorState gs )
        throws Exception
    {
    }

    //////////////////////////////////
    //
    //  Delta SECTION
    //
    //////////////////////////////////

    /**
     *

     * Note that this release is a delta from the previous one.
     *

     */
    private void buildDelta( GeneratorState gs, Element parent )
        throws Exception
    {
        String          releaseID = getReleaseID( gs );
        String          previousReleaseID = getPreviousReleaseID( gs );
        String          deltaStatement =
            "These notes describe the difference between Derby release " + releaseID +
            " and the preceding release " + previousReleaseID + ".";

        addParagraph( parent, deltaStatement );
    }

    //////////////////////////////////
    //
    //  Overview SECTION
    //
    //////////////////////////////////

    /**
     *

     * Build the Overview section.
     *

     */
    private void buildOverview( GeneratorState gs )
        throws Exception
    {
        Document    pamphlet = gs.getPamphlet();
        Element     overviewSection = getSection( pamphlet, MAIN_SECTION_LEVEL, OVERVIEW_SECTION );
        Document    summary = gs.getSummary();
        Element     summaryRoot = summary.getDocumentElement();
        Element     details = getFirstChild( summaryRoot, SUM_OVERVIEW );

        // copy the details out of the summary file into the overview section
        cloneChildren( details, overviewSection );
    }

    //////////////////////////////////
    //
    //  New Features SECTION
    //
    //////////////////////////////////

    /**
     *

     * Build the New Features section.
     *

     */
    private void buildNewFeatures( GeneratorState gs )
        throws Exception
    {
        Document    pamphlet = gs.getPamphlet();
        Element     newFeaturesSection = getSection( pamphlet, MAIN_SECTION_LEVEL, NEW_FEATURES_SECTION );
        Document    summary = gs.getSummary();
        Element     summaryRoot = summary.getDocumentElement();
        Element     details = getFirstChild( summaryRoot, SUM_NEW_FEATURES );

        // copy the details out of the summary file into the overview section
        cloneChildren( details, newFeaturesSection );
    }

    //////////////////////////////////
    //
    //  Bug List SECTION
    //
    //////////////////////////////////

    /**
     *

     * Build the Bug List section.
     *

     */
    private void buildBugList( GeneratorState gs )
        throws Exception
    {
        Document    pamphlet = gs.getPamphlet();
        Element     bugListSection = getSection( pamphlet, MAIN_SECTION_LEVEL, BUG_FIXES_SECTION );
        Document    bugList = gs.getBugList();
        JiraIssue[]    bugs = getJiraIssues( bugList );
        int                 count = bugs.length;
        String          releaseID = getReleaseID( gs );
        String          previousReleaseID = getPreviousReleaseID( gs );
        String          deltaStatement =
            "The following issues are addressed by Derby release " + releaseID +
            ". These issues are not addressed in the preceding " + previousReleaseID + " release.";

        addParagraph( bugListSection, deltaStatement );

        Element     table = createTable
            ( bugListSection, DEFAULT_TABLE_BORDER_WIDTH, new String[] { ISSUE_ID_HEADLINE, DESCRIPTION_HEADLINE } );

        for ( int i = 0; i < count; i++ )
        {
            JiraIssue   issue = bugs[ i ];
            Element     row = insertRow( table );
            Element     linkColumn = insertColumn( row );
            Element     descriptionColumn = insertColumn( row );
            Element     hotlink = createLink( pamphlet, issue.getJiraAddress(), issue.getKey() );
            Text            title = pamphlet.createTextNode( issue.getTitle() );

            linkColumn.appendChild( hotlink );
            descriptionColumn.appendChild( title );
        }
    }

    //////////////////////////////////
    //
    //  Issues SECTION
    //
    //////////////////////////////////

    /**
     *

     * Build the Issues section.
     *

     */
    private void buildIssuesList( GeneratorState gs )
        throws Exception
    {
        Document    pamphlet = gs.getPamphlet();
        Element     issuesSection = getSection( pamphlet, MAIN_SECTION_LEVEL, ISSUES_SECTION );
        Document    issuesList = gs.getReleaseNotesList();
        ReleaseNoteReader   releaseNoteReader = gs.getReleaseNoteReader();
        JiraIssue[]    bugs = getJiraIssues( issuesList );
        int                 count = bugs.length;
        String          releaseID = getReleaseID( gs );
        String          previousReleaseID = getPreviousReleaseID( gs );
        String          deltaStatement =
            "Compared with the previous release (" + previousReleaseID +
            "), Derby release " + releaseID + " introduces the following new features " +
            "and incompatibilities. These merit your special attention.";

        addParagraph( issuesSection, deltaStatement );

        Element     toc = createList( issuesSection );

        for ( int i = 0; i < count; i++ )
        {
            JiraIssue       issue = bugs[ i ];

            Document    releaseNote = null;

            try {
                releaseNote = getReleaseNote( gs, issue );
            }
            catch (Throwable t)
            {
                gs.addError( formatError( "Unable to read or parse release note for " + issue.getKey(), t ) );
            }

            // skip this note if we were unable to read it
            if ( releaseNote == null )
            {
                gs.addMissingReleaseNote( issue );
                continue;
            }

            String          key = "Note for " + issue.getKey();
            Node     summaryText = null;

            try {
                summaryText = releaseNoteReader.getReleaseNoteSummary( releaseNote );
            }
            catch (Throwable t)
            {
                gs.addError( formatError( "Badly formatted summary for " + issue.getKey(), t ) );
                summaryText = pamphlet.createTextNode( "Unreadable summary line" );
            }

            Element         paragraph = pamphlet.createElement( PARAGRAPH );

            paragraph.appendChild(  pamphlet.createTextNode( key + ": ") );
            cloneChildren( summaryText, paragraph );

            insertLine( issuesSection );

            Element     issueSection = createSection( issuesSection, ISSUE_DETAIL_LEVEL, toc, key, paragraph );

            try {
                Element     details = releaseNoteReader.getReleaseNoteDetails( releaseNote );

                // copy the details out of the release note into this section of the
                // pamphlet
                cloneChildren( details, issueSection );
            }
            catch (Throwable t)
            {
                gs.addError( formatError( "Could not read required sections out of issue " + issue.getKey(), t ) );
            }

        }
    }

    /**
     *

     * Get the release note for an issue.
     *

     */
    private Document   getReleaseNote( GeneratorState gs, JiraIssue issue )
        throws Exception
    {
        // first get the issue
        InputStream issueStream = grabDocument( gs, issue.getJiraAddress() );
        if ( issueStream == null ) { return null; }

        String          issueDocument = readJira( issueStream );
        String          releaseNoteAddress = getReleaseNoteAddress( issueDocument );
        InputStream releaseNoteStream = grabDocument( gs, releaseNoteAddress );
        if ( releaseNoteStream == null ) { return null; }

        Document        releaseNoteDoc = gs.getReleaseNoteReader().getReleaseNote( releaseNoteStream );

        return releaseNoteDoc;
    }

    /**
     *

     * We don't even bother trying to parse an individual jira since it is html
     * and not likely to be syntactically correct xml. We just siphon the jira
     * into a big string.
     *

     */
    private String  readJira( InputStream is )
        throws Exception
    {
        InputStreamReader   isr = new InputStreamReader( is, "UTF-8" );
        LineNumberReader    lr = new LineNumberReader( isr );
        StringBuilder            buffer = new StringBuilder();

        while( true )
        {
            String  nextLine = lr.readLine();

            if ( nextLine == null ) { break; }
            else { buffer.append( nextLine ); }
        }

        lr.close();
        isr.close();
        is.close();

        String  result = buffer.toString();

        return result;
    }

    /**
     *

     * Loop through the attachment pointers in a JIRA, looking for the highest
     * numbered release note.
     *

     */
    private String  getReleaseNoteAddress( String jira )
        throws Exception
    {
        long    highestReleaseNoteVersion = 0;
        String  releaseNoteAddress = null;
        int         idx = 0;

        while ( true )
        {
            int start = jira.indexOf( ATTACHMENT_START, idx );
            if ( start < 0 ) { break; }
            int end = jira.indexOf( ATTACHMENT_END, start );
            if ( end < 0 ) { break; }

            idx = end;

            String  attachmentAddress = jira.substring( start, end );

            // skip attachments that aren't release notes
            if ( !attachmentAddress.endsWith( RELEASE_NOTE_NAME ) ) { continue; }

            int         versionNumberStart = ATTACHMENT_START.length();
            int         versionNumberEnd = attachmentAddress.indexOf( ATTACHMENT_VERSION_END, versionNumberStart );
            String  versionString = attachmentAddress.substring( versionNumberStart, versionNumberEnd );
            long    versionNumber = Long.parseLong( versionString);

            if ( versionNumber >= highestReleaseNoteVersion)
            {
                highestReleaseNoteVersion = versionNumber;
                releaseNoteAddress = ATTACHMENT_PREFIX + attachmentAddress;
            }
        }

        System.out.println( "Found release note address: " + releaseNoteAddress );

        return releaseNoteAddress;
    }

    /**
     *

     * Grab a document off the web.
     *

     */
    private InputStream   grabDocument( GeneratorState gs, String urlString )
        throws Exception
    {
        // first get the issue
        URL             url = null;
        InputStream is = null;

        try {
            url = new URL( urlString );
            is = url.openStream();

            return is;
        }
        catch (Throwable t)
        {
            processThrowable( t );
            return null;
        }
    }

    //////////////////////////////////
    //
    //  Build Environment SECTION
    //
    //////////////////////////////////

    /**
     *

     * Build the section describing the build environment.
     *

     */
    private void buildEnvironment( GeneratorState gs )
        throws Exception
    {
        Document    pamphlet = gs.getPamphlet();
        Element         environmentSection = getSection( pamphlet, MAIN_SECTION_LEVEL, BUILD_ENVIRONMENT_SECTION );
        Document    summary = gs.getSummary();
        Element     summaryRoot = summary.getDocumentElement();
        String          releaseID = getReleaseID( gs );
        String          desc = "Derby release " + releaseID + " was built using the following environment:";

        addParagraph( environmentSection, desc );

        Element     list = createList( environmentSection );
        String          branchName = squeezeText( getFirstChild( summaryRoot, SUM_BRANCH ) );
        String          branchText = "Source code came from the " + branchName + " branch.";
        addHeadlinedItem( list, BRANCH_HEADLINE, branchText );

        String          machine = squeezeText( getFirstChild( summaryRoot, SUM_MACHINE ) );
        addHeadlinedItem( list, MACHINE_HEADLINE, machine );

        String          antVersion = squeezeText( getFirstChild( summaryRoot, SUM_ANT_VERSION ) );
        addHeadlinedItem( list, ANT_HEADLINE, antVersion );

        String          jdk14Version = squeezeText( getFirstChild( summaryRoot, SUM_JDK14 ) );
        addHeadlinedItem( list, JDK14_HEADLINE, jdk14Version );

        String          java6Version = squeezeText( getFirstChild( summaryRoot, SUM_JAVA6 ) );
        addHeadlinedItem( list, JAVA6_HEADLINE, java6Version );

        String          osgiText = squeezeText( getFirstChild( summaryRoot, SUM_OSGI ) );
        addHeadlinedItem( list, OSGI_HEADLINE, osgiText );

        String          compilerText = squeezeText( getFirstChild( summaryRoot, SUM_COMPILER ) );
        addHeadlinedItem( list, COMPILER_HEADLINE, compilerText );

        String          jsr169Text = squeezeText( getFirstChild( summaryRoot, SUM_JSR169 ) );
        addHeadlinedItem( list, JSR169_HEADLINE, jsr169Text );
    }

    //////////////////////////////////
    //
    //  REPLACE VARIABLES
    //
    //////////////////////////////////

    /**
     *

     * Replace the known parameters with their corresponding text values.
     *

     */
    private void replaceVariables( GeneratorState gs )
        throws Exception
    {
        Document    pamphlet = gs.getPamphlet();

        replaceTag( pamphlet, SUM_RELEASE_ID, getReleaseID( gs ) );
        replaceTag( pamphlet, SUM_PREVIOUS_RELEASE_ID, getPreviousReleaseID( gs ) );
    }

   //////////////////////////////////
    //
    //  Print the generated document.
    //
    //////////////////////////////////

    /**
     *

     * Print pamphlet to output file.
     *

     */
    private void    printPamphlet( GeneratorState gs )
        throws Exception
    {
        Document    pamphlet = gs.getPamphlet();
        Source            source = new DOMSource( pamphlet );
        File                  outputFile = new File( _pamphletFileName );
        Result            result = new StreamResult( outputFile );
        Transformer   transformer = TransformerFactory.newInstance().newTransformer();

        transformer.transform( source, result );
    }

    //////////////////////////////////
    //
    //  Print errors
    //
    //////////////////////////////////

    /**
     *

     * Print missing release notes
     *

     */
    private void printMissingReleaseNotes( GeneratorState gs )
        throws Exception
    {
        JiraIssue[]     missingReleaseNotes = gs.getMissingReleaseNotes();
        int                 count = missingReleaseNotes.length;

        if ( count > 0 )
        {
            println( "The following JIRA issues still need release notes or the release notes provided are unreadable:" );

            for ( int i = 0; i < count; i++ )
            {
                JiraIssue   issue = missingReleaseNotes[ i ];

                println( "\t" + issue.getKey() + "\t" + issue.getTitle() );
            }
        }
    }

    /**
     *

     * Print errors.
     *

     */
    private void printErrors( GeneratorState gs )
        throws Exception
    {
        String[]     errors = gs.getErrors();
        int                 count = errors.length;

        if ( count > 0 )
        {
            println( "The following other errors occurred:" );

            for ( int i = 0; i < count; i++ )
            {
                String  error = errors[ i ];

                println( "\n" + error );
            }
        }
    }

    //////////////////////////////////
    //
    //  ARGUMENT MINIONS
    //
    //////////////////////////////////

    /**
     *

     * Returns true if arguments parse successfully, false otherwise.
     *

     */
    private boolean    parseArgs( String[] args )
        throws Exception
    {
        if ( (args == null) || (args.length != 4) ) { return false; }

        int     idx = 0;

        setSummaryFileName( args[ idx++ ] );
        setBugListFileName( args[ idx++ ] );
        setReleaseNotesListFileName( args[ idx++ ] );
        setPamphletFileName( args[ idx++ ] );

        return true;
    }

    private void    printUsage()
    {
        println( USAGE );
    }

    /**
     *

     * Make sure that the input files all exist. Returns a state variable for the generator.
     *

     */
    private GeneratorState  initialize()
        throws Exception
    {
        DocumentBuilderFactory  factory = DocumentBuilderFactory.newInstance();
        DocumentBuilder              builder = factory.newDocumentBuilder();
        Document                          pamphlet = builder.newDocument();
        File                                    summaryFile = new File( _summaryFileName );
        File                                    bugListFile = new File( _bugListFileName );
        File                                    releaseNotesListFile = new File( _releaseNotesListFileName);
        Document                        summary = builder.parse( summaryFile );
        Document                        bugList = builder.parse( bugListFile );
        Document                        releaseNotesList = builder.parse( releaseNotesListFile );

        return new GeneratorState
            (
             builder,
             pamphlet,
             summary,
             bugList,
             releaseNotesList
             );
    }

    ////////////////////////////////////////////////////////
    //
    // HTML MINIONS
    //
    ////////////////////////////////////////////////////////

    /**
     *

     * Create a section at the end of a parent element and link to it from a
     * table of contents.
     *

     */
    private Element createSection( Element parent, int sectionLevel, Element toc, String sectionName, String tocEntry )
        throws Exception
    {
        Document        doc = parent.getOwnerDocument();
        Text                textNode = doc.createTextNode( tocEntry );

        return createSection( parent, sectionLevel, toc, sectionName, textNode );
    }

    /**
     *

     * Create a section at the end of a parent element and link to it from a
     * table of contents.
     *

     */
    private Element createSection( Element parent, int sectionLevel, Element toc, String sectionName, Node visibleText )
        throws Exception
    {
        Document        doc = parent.getOwnerDocument();
        Element             link = createLocalLink( doc, sectionName, visibleText );

        addListItem( toc, link );

        return createHeader( parent, sectionLevel, sectionName );
    }

    /**
     *

     * Create a header at the end of the parent node. Return the block created
     * to hold the text following this header.
     *

     */
    private Element createHeader( Element parent, int headerLevel, String text )
        throws Exception
    {
        Document        doc = parent.getOwnerDocument();
        Text            textNode = doc.createTextNode( text );
        Element         header = doc.createElement( makeHeaderTag( headerLevel ) );
        Element         anchor = doc.createElement( ANCHOR );
        Element         block =doc.createElement( INDENT );

        parent.appendChild( header );
        anchor.setAttribute( NAME, text );
        header.appendChild( anchor );
        header.appendChild( textNode );
        parent.appendChild( block );

        return block;
    }

    /**
     *

     * Create an html text element.
     *

     */
    private Element createTextElement( Document doc, String tag, String text )
        throws Exception
    {
        Element     retval = doc.createElement( tag );
        Text            textNode = doc.createTextNode( text );

        retval.appendChild( textNode );

        return retval;
    }

    /**
     *

     * Create a standard link to a local label.
     *

     */
    private Element createLocalLink( Document doc, String anchor, String text )
        throws Exception
    {
        Text        textNode = doc.createTextNode( text );

        return createLocalLink( doc, anchor, textNode );
    }

    /**
     *

     * Create a standard link to a local label.
     *

     */
    private Element createLocalLink( Document doc, String anchor, Node visibleText )
        throws Exception
    {
        return createLink( doc, "#" + anchor, visibleText );
    }

    /**
     *

     * Create a hotlink.
     *

     */
    private Element createLink( Document doc, String label, String text )
        throws Exception
    {
        Text        textNode = doc.createTextNode( text );

        return createLink( doc, label, textNode );
    }

    /**
     *

     * Create a hotlink.
     *

     */
    private Element createLink( Document doc, String label, Node visibleText )
        throws Exception
    {
        Element hotlink = doc.createElement( ANCHOR );

        hotlink.setAttribute( "href", label );
        hotlink.appendChild( visibleText );

        return hotlink;
    }

    /**
     *

     * Insert a list at the end of the parent element.
     *

     */
    private Element createList( Element parent )
        throws Exception
    {
        Document        doc = parent.getOwnerDocument();
        Element list = doc.createElement( LIST );

        parent.appendChild( list );

        return list;
    }

    /**
     *

     * Add an item with a bold name to the end of a list.
     *

     */
    private Element addHeadlinedItem( Element list, String headline, String text )
        throws Exception
    {
        Document        doc = list.getOwnerDocument();
        Element         itemElement = doc.createElement( LIST_ELEMENT );
        Element         boldText = boldText( doc, headline );
        Text            textNode = doc.createTextNode( " - " + text );

        list.appendChild( itemElement );
        itemElement.appendChild( boldText );
        itemElement.appendChild( textNode );

        return itemElement;
    }

    /**
     *

     * Make some bold text.
     *

     */
    private Element boldText( Document doc, String text)
        throws Exception
    {
        Element bold = createTextElement( doc, BOLD, text );

        return bold;
    }

    /**
     *

     * Add an item to the end of a list.
     *

     */
    private Element addListItem( Element list, Node item )
        throws Exception
    {
        Document        doc = list.getOwnerDocument();
        Element         itemElement = doc.createElement( LIST_ELEMENT );

        list.appendChild( itemElement );
        itemElement.appendChild( item );

        return itemElement;
    }

    /**
     *

     * Retrieve the indented block inside a section
     *

     */
    private Element getSection( Document doc, int sectionLevel, String sectionName )
        throws Exception
    {
        String          headerTag = makeHeaderTag( sectionLevel );
        Element     root = doc.getDocumentElement();
        NodeList    sectionList = root.getElementsByTagName( headerTag );
        int             count = sectionList.getLength();

        for ( int i = 0; i < count; i++ )
        {
            Element     section = (Element) sectionList.item( i );
            Element     sectionAnchor = getFirstChild( section, ANCHOR );

            if ( sectionName.equals( sectionAnchor.getAttribute( NAME ) ) )
            {
                // the next item after the section header, is the indented block

                return (Element) section.getNextSibling();
            }
        }

        return null;
    }

    /**
     *

     * Make the tag for a header, given its level
     *

     */
    private String makeHeaderTag( int headerLevel )
    {
        return "h" + Integer.toString( headerLevel );
    }

    /**
     *

     * Add a paragraph to the end of a parent element.
     *

     */
    private Element addParagraph( Element parent, String text )
        throws Exception
    {
        Document        doc = parent.getOwnerDocument();
        Text            textNode = doc.createTextNode( text );
        Element         paragraph = doc.createElement( PARAGRAPH );

        parent.appendChild( paragraph );
        paragraph.appendChild( textNode );

        return paragraph;
    }

    /**
     *

     * Insert a table at the end of the parent element.
     *

     */
    private Element createTable( Element parent, int borderWidth, String[] columnHeadings )
        throws Exception
    {
        Document        doc = parent.getOwnerDocument();
        Element         table = doc.createElement( TABLE );
        Element         headingRow = insertRow( table );
        int                     count = columnHeadings.length;

        parent.appendChild( table );
        table.setAttribute( BORDER, Integer.toString( borderWidth ) );

        for ( int i = 0; i < count; i++ )
        {
            Element     headingColumn = insertColumn( headingRow );
            Element     boldText = boldText( doc, columnHeadings[ i ] );

            headingColumn.appendChild( boldText );
        }

        return table;
    }

    /**
     *

     * Insert a row at the end of a table
     *

     */
    private Element insertRow( Element table )
        throws Exception
    {
        Document        doc = table.getOwnerDocument();
        Element         row = doc.createElement( ROW );

        table.appendChild( row );

        return row;
    }

    /**
     *

     * Insert a column at the end of a row
     *

     */
    private Element insertColumn( Element row )
        throws Exception
    {
        Document        doc = row.getOwnerDocument();
        Element         column = doc.createElement( COLUMN );

        row.appendChild( column );

        return column;
    }

    /**
     *

     * Insert a horizontal line at the end of the parent element.
     *

     */
    private Element insertLine( Element parent )
        throws Exception
    {
        Document        doc = parent.getOwnerDocument();
        Element         line = doc.createElement( HORIZONTAL_LINE );

        parent.appendChild( line );

        return line;
    }

    ////////////////////////////////////////////////////////
    //
    // XML MINIONS
    //
    ////////////////////////////////////////////////////////

    private Element getFirstChild( Element node, String childName )
        throws Exception
    {
        Element retval = getOptionalChild( node, childName );

        if ( retval == null )
        {
            throw new BuildException( "Could not find child element '" + childName + "' in parent element '" + node.getNodeName() + "'." );
        }

        return retval;
    }

    private Element getOptionalChild( Element node, String childName )
        throws Exception
    {
        return (Element) node.getElementsByTagName( childName ).item( 0 );
    }

    /**
     *

     * Squeeze the text out of an Element.
     *

     */
    private String squeezeText( Element node )
        throws Exception
    {
        Node        textChild = node.getFirstChild();
        String      text = textChild.getNodeValue();

        return text;
    }

    /**
     *

     * Clone all of the children of a source node and add them as children
     * of a target node.
     *

     */
    private void cloneChildren( Node source, Node target )
        throws Exception
    {
        Document    targetDoc = target.getOwnerDocument();
        NodeList        sourceChildren = source.getChildNodes();
        int                 count = sourceChildren.getLength();

        for ( int i = 0; i < count; i++ )
        {
            Node    sourceChild = sourceChildren.item( i );
            Node    targetChild = targetDoc.importNode( sourceChild, true );

            target.appendChild( targetChild );
        }
    }

    /**
     *

     * Replace all instances of the tag with the indicated text.
     *

     */
    private void replaceTag( Document doc, String tag, String replacementText )
        throws Exception
    {
        NodeList        sourceChildren = doc.getElementsByTagName( tag );
        int                 count = sourceChildren.getLength();

        for ( int i = 0; i < count; i++ )
        {
            Node    oldChild = sourceChildren.item( i );
            Node    newChild = doc.createTextNode( replacementText );

            if ( oldChild != null )
            {
                Node    parent = oldChild.getParentNode();

                if ( parent != null ) { parent.insertBefore( newChild, oldChild ); }
            }
        }

        for ( int i = count-1; i > -1; i-- )
        {
            Node    oldChild = sourceChildren.item( i );

            if ( oldChild != null )
            {
                Node    parent = oldChild.getParentNode();

                if ( parent != null ) { parent.removeChild( oldChild ); }
            }
        }
    }

    ////////////////////////////////////////////////////////
    //
    // JIRA MINIONS
    //
    ////////////////////////////////////////////////////////

    /**
     *

     * Get an array of JiraIssues from a JIRA report.
     *

     */
    private JiraIssue[]   getJiraIssues( Document report )
        throws Exception
    {
        Element         reportRoot = report.getDocumentElement();
        NodeList        itemList = reportRoot.getElementsByTagName( JIRA_ITEM );
        int                 count = itemList.getLength();
        JiraIssue[]     issues = new JiraIssue[ count ];

        for ( int i = 0; i < count; i++ ) { issues[ i ] = makeJiraIssue( (Element) itemList.item( i ) ); }

        return issues;
    }

    /**
     *

     * Create a JiraIssue from an <item> element in a JIRA report.
     *

     */
    private JiraIssue   makeJiraIssue( Element itemElement )
        throws Exception
    {
        // the text lives in an anchor
        Element anchorElement = getFirstChild( itemElement, ANCHOR );
        String  contents = squeezeText( anchorElement );

        String  key = parseKey( contents );
        String  title;

        //
        // A JIRA title has the following form:
        //
        //  "[DERBY-2598] new upgrade  test failures after change 528033"
        //
        // We strip off the leading JIRA id because that information already
        // lives in the key.
        //
        title = contents.substring( contents.indexOf( ']' ) + 2, contents.length() );

        return new JiraIssue( key, title );
    }

    /**
     *

     * Parse a string, looking for an id of the form "[DERBY-2598]". Extract the
     * bit between the brackets.
     *

     */
    private String parseKey( String raw )
    {
        int  start = raw.indexOf( '[' );
        int  end = raw.indexOf( ']' );

        return raw.substring( start + 1, end );
    }

    ////////////////////////////////////////////////////////
    //
    // EXCEPTION PROCESSING MINIONS
    //
    ////////////////////////////////////////////////////////

    /**
     *

     * Special processing for special exceptions.
     *

     */
    private  void    processThrowable( Throwable t )
    {
        if ( t instanceof java.net.UnknownHostException )
        {
            println( "Unknown host '" + t.getMessage() + "'. Can you ping this host from a shell window?" );
        }
    }

    /**
     *

     * Format an error for later reporting.
     *

     */
    private  String    formatError( String text, Throwable t )
    {
        text = text + ": " + t.toString() + "\n" + stringifyStackTrace( t );

        return text;
    }

    /**
     *

     * Print a stack trace as a string.
     *

     */
    private  String stringifyStackTrace( Throwable t )
    {
        StringWriter    sw = new StringWriter();
        PrintWriter     pw = new PrintWriter( sw, true );

        t.printStackTrace( pw );
        pw.flush();
        sw.flush();

        return sw.toString();
    }

     ////////////////////////////////////////////////////////
    //
    // MISC MINIONS
    //
    ////////////////////////////////////////////////////////

    private  void    println( String text )
    {
        if ( _invokedByAnt )
        {
            log( text, Project.MSG_WARN );
        }
        else
        {
            System.out.println( text );
        }
    }

}


text:
The simple but complete ruby library for the TVDB apis. Created by Nathan Esquenazi for use at Miso.

Yes, there are a lot of libraries out there for this. I needed a few things that were different:

  * Easy way to 'verify' the correctness of a match beyond just title
  * An easy way to get all updates since a certain time, to keep our database synced
  * Easy helpers for accessing episodes from a series and the objects conceptually mapped to our existing schema.

Please reference the [TVDB APIs](http://thetvdb.com/wiki/index.php?title=Programmers_API) to
become familiar with the available actions and properties.

## Installation

    $ sudo gem install tvdbr

### Usage

Create the tdvbr client:

    tvdb = Tvdbr::Client.new('API_KEY')

Get a series based on the id:

    series = tvdb.find_series_by_id('1234')

Get a series based on a remote (IMDB) id:

    series = tvdb.find_series_by_remote_id('tt0290978')

Get an episode based on the id:

    episode = tvdb.find_episode_by_id('5678')

Get an episode based on the seriesid and airdate:

    episode = tvdb.find_episode_by_airdate(80348, '2007-09-24')

Fetch a series based on the title:

    dexter = tvdb.fetch_series_from_data(:title => "Dexter")
    # => <#Tvbdr::Series>
    dexter.title # => "Dexter"
    dexter.actors # => ["Michael C. Hall", ...]
    dexter.genres # => ["Drama", ...]
    dexter.poster # => "http://thetvdb.com/some/poster.jpg"

Fetch a series based on the title and list of actors:

    dexter = tvdb.fetch_series_from_data(:title => "Dexter", :starring => "Michael C. Hall, Jennifer Carpenter")

Get the episodes for a series:

    series.episodes.each { |e| puts "#{e.name} - S#{e.season_num}E#{e.episode_num}" }

Get the updated series since a given timestamp:

    tvdb.each_updated_series(:since => 1.day.ago) do |series|
      puts "#{series.id} - #{series.title}"
    end

Get the updated episodes since a given timestamp:

    tvdb.each_updated_episode(:since => 1.day.ago) do |episode|
      puts "#{episode.id} - #{episode.name} - #{episode.seriesid}"
    end

You can also query using the newer static updates, by providing a 'period':

    tvdb.each_updated_episode(:period => :day) do |episode|
      puts "#{episode.id} - #{episode.name} - #{episode.seriesid}"
    end

You can provide `day`, `week`, or `month` to get complete data for that range of time.

## Known Issues

* None so far, but the gem is very young.
* ...except I wrote this in a day or two and didn't write specs :(
* ...but this will soon be corrected!

redpajama_set_name: RedPajamaGithub


text:
Web Processing Services (WPS) Tube Select
=========================================

This tutorial will show you how to:

1. Ingest a set of Twitter data.
2. Use GeoServer Web Processing Services (WPS) to generate
   time-interpolated queries.
3. Query from and save results to GeoServer layers.

Introduction
------------

A time-interpolated query finds features that intersect in both **space
and time**. Normal spatial or temporal queries select features that
fall within a specified geometry, time range, or both. A
time-interpolated query has a changing geometry as you move forward in
time.

Imagine you are traveling from Washington, DC to New York City and are
interested in people traveling the same route at the same time. These
travelers would be within a given distance from you at any point in the
trip. They wouldn't be in DC while you were arriving in New York. Thus
the geometry of results of your query would be **a function of time**.

Time-interpolated queries are more complex than normal spatial or
temporal queries, as the input is a sequence of points or lines with an
increasing time attribute. Time-interpolated queries are colloquially
known as **tube selects**.

Prerequisites
-------------

Make sure that you've worked through the steps described in the
:ref:`installation` section of the GeoMesa User
Manual, particularly the tasks relevant to Accumulo. Afterwards, it may be
necessary to change the versions of Accumulo and Hadoop that the tutorial
uses. After completing the deployment tutorial, you should have:

-  an instance of Accumulo |accumulo_version| running on Hadoop
   |hadoop_version|,
-  access to an Accumulo user that has both create-table and write
   permissions, and
-  an instance of GeoServer 2.5.2 with the GeoMesa plugin installed.

Because this tutorial requires building custom Java code, development
tools should also be installed and configured:

-  `Java JDK 8 <http://www.oracle.com/technetwork/java/javase/downloads/index.html>`__,
-  `Apache Maven <http://maven.apache.org/>`__ |maven_version|, and
-  a `git <http://git-scm.com/>`__ client.

Using the GeoServer Plugin
--------------------------

Once everything is installed you should see "geomesa:TubeSelect" appear
in the `WPS Request
Builder <http://docs.geoserver.org/stable/en/user/services/wps/requestbuilder.html>`__,
which is found under the *Demos* link in the menu bar on the left side
of the GeoServer web administration interface.

.. figure:: _static/img/tutorials/2014-05-16-geomesa-tubeselect/wps_builder1.png
   :alt: "geomesa:TubeSelect WPS plugin"

   "geomesa:TubeSelect WPS plugin"

Downloading and Building the Tutorial Code
------------------------------------------

Clone the **geomesa** project and build it, if you haven't already:

.. code-block:: bash

    $ git clone https://github.com/locationtech/geomesa.git
    $ cd geomesa
    $ mvn clean install

This is needed to install the GeoMesa JAR files in your local Maven
repository. For more information see the `GeoMesa Accumulo Quick
Start </geomesa-quickstart/>`__ tutorial. We'll be using Twitter data
from the public Twitter API for this tutorial and ingesting it using a
single node ingester that uses a
`GeoTools <http://www.geotools.org/>`__-based ``FeatureSource`` to
ingest into GeoMesa. First, check out the code and take a look at it:

.. code-block:: bash

    $ git clone https://github.com/geomesa/geomesa-twitter
    $ cd geomesa-twitter

To build the project, run the following command:

.. code-block:: bash

    $ mvn clean install

Collecting Twitter Data
-----------------------

.. warning::

    It will take several hours to collect a large enough set of Twitter data for the analysis below.

The **twitter-collect** module collects
`Twitter <http://twitter.com/>`__ data in JSON format and writes it a
local file. To run the code, you will need a Twitter account, as well as
access tokens. Go to the Twitter development app management console at
https://apps.twitter.com/ and create a new application, then go to the
"Keys and Access Tokens" page to generate an "Access Token." The
consumer API key, the consumer secret, the access token, and the access
token secret are all needed to run the collection process.

Run the program:

.. code-block:: bash

    $ java -jar twitter-collect/target/twitter-collect-accumulo1.5-1.0-SNAPSHOT.jar \
        --consumerKey <CONSUMER_KEY> \
        --consumerSecret <CONSUMER_SECRET> \
        --outputDir ./tmp \
        --secret <ACCESS_SECRET> \
        --token <ACCESS_TOKEN>

This will collect live Twitter data and save it in JSON files in the
``tmp`` subdirectory; this will collect data indefinitely until it is
interrupted.

Ingesting Twitter Data
----------------------

The **twitter-ingest** module takes a set of JSON files.

There are three classes in the module:

-  ``Runner``: Provides the main method to handle command line argument
   parsing
-  ``TwitterFeatureIngester``: Inserts features into GeoMesa using a
   GeoTools ``FeatureSource``
-  ``TwitterParser``: Parses Twitter JSON files and turns them into
   ``SimpleFeature``\ s

Run the ingest program:

.. code-block:: bash

    $ java -jar twitter-ingest/target/twitter-ingest-accumulo1.5-1.0-SNAPSHOT.jar \
        --featureName <featureName> \
        --tableName <tableName> \
        --instanceId <instance> \
        --user <user> \
        --password <password> \
        --zookeepers <zookeepers> \
        tmp/20151201/20151201-090959.txt

where ``<featureName>`` is the name of the SimpleFeatureType to create,
``<tableName>`` is the name of the GeoMesa catalog table to use, and
``<instance>``, ``<user>``, ``<password>``, and ``<zookeepers>``\ are
needed to connect to Accumulo. The final argument is the name of one of
the Twitter JSON files created by the **twitter-collect** module.

After ingesting the data, create a data store and layer in GeoServer
from the data ingested into GeoMesa (see the **VISUALIZE DATA WITH
GEOSERVER** section of the `GeoMesa Accumulo Quick Start
tutorial </geomesa-quickstart/>`__ for more details about how to do
this).

More about parsing Twitter JSON data
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

The JSON object returned from the tweets stream is documented on the
`Twitter developer
site <https://dev.twitter.com/docs/platform-objects/tweets>`__. For this
tutorial we'll be interested in the following fields:

-  ``coordinates``: a `geoJSON <http://geojson.org/>`__ object with
   lat/lon (do not use the deprecated ``geo`` field)
-  ``user``: the user object
-  ``id``: the user id
-  ``name``: the user name
-  ``id``: the tweet id
-  ``created_at``: the time when the tweet was created
-  ``text``: the text of the tweet

The Twitter JSON looks something like this:

.. code-block:: json

    {
      "retweeted" : false,
      "source" : "web",
      "favorited" : false,
      "coordinates" : {
        "coordinates" : [
          -75.14310264,
          40.05701649
        ],
        "type" : "Point"
      },
      "place" : null,
      "retweet_count" : 0,
      "entities" : {
        "hashtags" : [],
        "user_mentions" : [
          {
            "name" : "Twitter API",
            "indices" : [
              19,
              30
            ],
            "id" : 6253282,
            "id_str" : "6253282",
            "screen_name" : "twitterapi"
          }
        ],
        "urls" : []
      },
      "truncated" : false,
      "in_reply_to_status_id_str" : null,
      "created_at" : "Wed Feb 29 19:42:02 +0000 2012",
      "contributors" : null,
      "text" : "Man I like me some @twitterapi",
      "in_reply_to_user_id" : null,
      "user" : {
        "friends_count" : 5,
        "follow_request_sent" : null,
        "profile_sidebar_fill_color" : "ffffff",
        "profile_image_url" : "http://a1.twimg.com/profile_images/1540298033/phatkicks_normal.jpg",
        "profile_background_image_url_https" : "https://si0.twimg.com/profile_background_images/365782739/doof.jpg",
        "profile_background_color" : "C0DEED",
        "notifications" : null,
        "url" : "http://blog.roomanna.com",
        "id" : 370773112,
        "is_translator" : false,
        "following" : null,
        "screen_name" : "fakekurrik",
        "lang" : "en",
        "location" : "",
        "followers_count" : 8,
        "statuses_count" : 142,
        "name" : "fakekurrik",
        "description" : "I am just a testing account, following me probably won't gain you very much",
        "favourites_count" : 1,
        "profile_background_tile" : true,
        "listed_count" : 0,
        "contributors_enabled" : false,
        "profile_link_color" : "0084B4",
        "profile_image_url_https" : "https://si0.twimg.com/profile_images/1540298033/phatkicks_normal.jpg",
        "profile_sidebar_border_color" : "C0DEED",
        "created_at" : "Fri Sep 09 16:13:20 +0000 2011",
        "utc_offset" : -28800,
        "verified" : false,
        "show_all_inline_media" : false,
        "profile_background_image_url" : "http://a3.twimg.com/profile_background_images/365782739/doof.jpg",
        "default_profile" : false,
        "protected" : false,
        "id_str" : "370773112",
        "profile_text_color" : "333333",
        "default_profile_image" : false,
        "time_zone" : "Pacific Time (US & Canada)",
        "geo_enabled" : true,
        "profile_use_background_image" : true
      },
      "id" : 174942523154894848,
      "in_reply_to_status_id" : null,
      "geo" : null,
      "in_reply_to_user_id_str" : null,
      "id_str" : "174942523154894848",
      "in_reply_to_screen_name" : null
    }

We parse this object manually with `GSON
(google-json) <https://github.com/google/gson>`__. Optionally,
you can create Java Object bindings for GSON and parse the entire tweet
into an object. For more information about connecting to the twitter
public stream check out the `Twitter Public Stream
website <https://dev.twitter.com/docs/streaming-apis/streams/public>`__.

Creating a Web Processing Service (WPS) Request
-----------------------------------------------

The WPS builder will provide us with a sample XML document as a starting
point to perform a tube select. Entering the values we want and clicking
the "Generate XML" button produces the request document that we'll save
as ``tube-select.xml`` for use later.

.. figure:: _static/img/tutorials/2014-05-16-geomesa-tubeselect/generate_xml.png
   :alt: "Generating XML from inputs"

   "Generating XML from inputs"

Chaining Processes
~~~~~~~~~~~~~~~~~~

WPS supports chaining other WPS processes. This is useful when selecting
data from an existing layer or storing data back into GeoServer to
create a new layer. The GeoServer User Guide has a section on `Process
Chaining <http://docs.geoserver.org/stable/en/user/services/wps/processes/chaining.html>`__.

Tube Selections
---------------

Using a Layer as Input
~~~~~~~~~~~~~~~~~~~~~~

For this demonstration, let's assume someone is traveling from
Wilmington, DE to north of Philadelphia, PA along the New Jersey
Turnpike and is tweeting. Each tweet would have a GPS coordinate and a
timestamp allowing us to infer that the person was traveling somewhere
around 55mph (25 m/s) on the Turnpike. This track is the input that
defines the "tube" geometry for our query. Using the ``maxSpeed`` of 25
m/s and a ``maxTime`` of 120s we will create a space/time query tube.
Here's a GeoServer rendering of a possible track:

.. figure:: _static/img/tutorials/2014-05-16-geomesa-tubeselect/tube_features.png
   :alt: "Tube Features"

   "Tube Features"

Using JSON as Input
~~~~~~~~~~~~~~~~~~~

Instead of using a layer as input, you can manually define an input
``FeatureCollection`` track with JSON using
`GeoJSON <http://geojson.org/>`__. The features need a unique **id**,
a **geometry** object, and a property named **dtg** representing the
observation time of the feature. A sample of two of our input track
points is shown below:

.. code-block:: xml

     <wps:Input>
        <ows:Identifier>tubeFeatures</ows:Identifier>
        <wps:Data>
             <wps:ComplexData mimeType="application/json">
                <![CDATA[
                {
                   "features" : [
                        {
                            "id" : "0",
                            "geometry" : {
                                "coordinates": [-75.6002273 , 39.669456],
                                "type" : "Point"
                            },
                            "type" : "Feature",
                            "properties" : {
                                "dtg" : "2014-05-17T15:33:16.000+0000",
                            }
                        }, {
                            "id" : "1",
                            "geometry" : {
                                "coordinates": [-75.5102991 , 39.6857254],
                                "type" : "Point"
                            },
                            "type" : "Feature",
                            "properties" : {
                                "dtg" : "2014-05-17T15:39:28.000+0000",
                            }
                        }
                   ],
                   "type" : "FeatureCollection"
                }
                ]]>
            </wps:ComplexData>
        </wps:Data>
    </wps:Input>

Posting a Query
~~~~~~~~~~~~~~~

After creating our tube query XML we can post it to GeoServer with
``curl``:

.. code-block:: bash

    $ curl -u admin:geoserver -H 'Content-type: xml' -XPOST -d@'/tmp/query.xml' http://localhost:8081/geoserver/wps | json_pp

In this request, the MIME type of the result variable is set to ``json``
in our file ``query.xml``:

.. code-block:: xml

    <wps:ResponseForm>
      <wps:RawDataOutput mimeType="application/json">
        <ows:Identifier>result</ows:Identifier>
      </wps:RawDataOutput>
    </wps:ResponseForm>

We piped the output into
`json\_pp <http://search.cpan.org/~makamaka/JSON-PP-2.27103/bin/json_pp>`__,
which will pretty print the output (native packages are available for
most Linux distributions).

Saving The Output as a Layer
----------------------------

Chaining your result with a **gs:import** process allows you to store
your layer back into GeoMesa, allowing you to render a layer. Below we
have created a layer combination with a map, our original tube track,
and our results with ``maxSpeed`` 25 m/s and ``maxTime`` 120 s. These
layers are **mapproxy:globe.osm.toner**, **accumulo:njtrack**, and
**accumulo:nj120**:

.. code-block:: bash

    http://localhost:8081/geoserver/wms?service=WMS&version=1.1.0&request=GetMap&layers=mapproxy:globe.osm.toner,accumulo:njtrack,accumulo:nj120&styles=&bbox=-75.62390877,39.65103897,-74.81563784,40.04023423&width=1440&height=960&srs=EPSG:4326&format=application/openlayers

.. figure:: _static/img/tutorials/2014-05-16-geomesa-tubeselect/tube-25-120.png
   :alt: "Tube Result at 25m/s 120s"

   "Tube Result at 25m/s 120s"

Tweaking Parameters
-------------------

maxSpeed, maxTime, bufferSize
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Adjusting the ``maxSpeed`` and ``maxTime`` parameters will increase the
size of the tube from both a spatial and temporal aspect. Features that
are further away in either time or geometry could possibly be reached by
the input track by increasing these parameters. Below, we have doubled
our ``maxTime`` to 240 s creating a wider tube. Perhaps our friend
strayed off the highway and visited a restaurant or gas station. We'd
like to account for that possibility. Once again, a chained
**gs:import** command has stored the layer as **nj240**.

.. code-block:: bash

    http://localhost:8081/geoserver/wms?service=WMS&version=1.1.0&request=GetMap&layers=mapproxy:globe.osm.toner,accumulo:njtrack,accumulo:nj240&styles=&bbox=-75.62390877,39.65103897,-74.81563784,40.04023423&width=1440&height=960&srs=EPSG:4326&format=application/openlayers

.. figure:: _static/img/tutorials/2014-05-16-geomesa-tubeselect/tube-25-240.png
   :alt: "Tube Result at 25m/s 240s"

   "Tube Result at 25m/s 240s"

gapFill
~~~~~~~

Many input tracks can be sparse. Our friend driving the Turnpike didn't
tweet every 5 seconds. Selecting a ``gapFill`` method fills in those
missing tracks. In this tutorial we used the **line** option for gap
filling. The options available for the ``gapFill`` method are:

-  ``none`` - use when you have a dense track
-  ``line`` - draws a line between points two points and uses the time
   range of both points

Beyond Curl
-----------

WPS services can also be accessed by a variety of clients including
`OpenLayers <http://openlayers.org/>`__, which has implemented a
`JavaScript WPS
Builder <http://dev.openlayers.org/releases/OpenLayers-2.13.1/examples/wps.html>`__
tutorial similar to the one built into GeoServer.


text:
package openblocks.common.block;

import openmods.infobook.BookDocumentation;

@BookDocumentation
public class BlockBlockBreaker extends BlockBlockManpulatorBase {}


text:
var group___p_w_r___private___macros =
[
    [ "PWR Private macros to check input parameters", "group___p_w_r___i_s___p_w_r___definitions.html", null ]
];


text:
var spawn = require('child_process').spawn
var through = require('through2')
var platform = require('os').platform()
var util = require('util')
var eol = require('os').EOL

/**
 * Spawner - a wrapper to promised spawn
 * @param  {Object}  options        modifiers {out: Function|string, err: Function|string}
 * @param  {Object}  spawn_options  options to spawn process (nodejs.org/api/child_process.html)
 * @return {Object}  Wrapper        {Promise sp = spawn, Stream out, Stream err}
 */
var Spawner = function(options, spawn_options) {
  var opt = {}
  var self = this

  if(typeof options == 'object') {
    opt.out = options.out !== undefined ? options.out : opt.out
    opt.err = options.err !== undefined ? options.err : opt.err
  }

  //those are global streams to allow piping of the current running spawn
  this.out = through.obj(function (chunk, enc, callback) {
    this.push(chunk)
    self.data.out.push(chunk)
    return callback()
  })

  this.err = through.obj(function (chunk, enc, callback) {
    this.push(chunk)
    self.data.err.push(chunk)
    return callback()
  })

  var spawn = function() {
    var args = [].slice.call(arguments)
    var last = args.length - 1

    if(!util.isArray(args[last]) && typeof args[last] == 'object') {
      var options = args.pop()
    }

    args = [].concat.apply([], args) //flatten

    var num_commands = args.length
    var commands = []
    var i = 0

    for(i; i < num_commands; i++) {
      commands.push({
        command: this.command,
        args: this.args.concat([args[i]]),
        options: options || spawn_options || {}
      })
    }

    //reset previous spawn datas
    this.data = {out: [], err: []}

    return new Promise((resolve, reject) => {
      var j = 0

      //function to loop through promises commands
      //This handles the catch promise when a command fail
      //Then, we can reject the spawner global promise
      var loop = (array) => {
        var command = array.shift()

        this
          .promise_spawn(command, opt)
          .then(function(code) {
            if(j < num_commands - 1) {
              j++
              return loop(array)
            } else {
              resolve(code)
            }

          }).catch(function(code) {
            reject(code)
          })
      }

      loop(commands)

    })
  }

  //main function wrapper
  return {
    sp: spawn.bind(this),
    spawn: spawn.bind(this),
    out: this.out,
    err: this.err
  }
}

Spawner.prototype = {
  command: platform == 'win32' ? 'cmd.exe' : 'sh',
  args: platform == 'win32' ? ['/s', '/c'] : ['-c'],
  /**
   * Promisify Spawn
   * @param  {Object} command {command, args, spawn_options}
   * @param  {Object} options modifiers from Spawner
   * @return {Promise}         Promise resolve on error code = 0 or reject
   */
  promise_spawn: function(command, options) {
    return new Promise((resolve, reject) => {

      var s = spawn(command.command, command.args, command.options)

      //spawn stdout to the Stream modifier
      //writes data back to global Stream
      s.stdout && s.stdout
        .pipe(this.pipe(options.out))
        .on('data', (d) => {
          this.out.write(d)
        })

      s.stderr && s.stderr
        .pipe(this.pipe(options.err))
        .on('data', (d) => {
          this.err.write(d)
        })

      s.on('close', (code) => {
        if(code === 0) {
          resolve({code: code, data: this.data})
        } else {
          reject({code: code, data: this.data})
        }
      })
    })
  },
  pipe: function(modifier) {
    modifier = modifier === undefined ? '' : modifier

    return through.obj(function (chunk, enc, callback) {
      chunk = typeof modifier == 'function' ? modifier(chunk) : modifier + chunk.toString().replace(eol, '')
      this.push(chunk)
      callback()
    })
  }
}

module.exports = Spawner


text:
import { HttpClient, HttpErrorResponse } from '@angular/common/http';
import { Injectable } from '@angular/core';
import { Observable } from 'rxjs/Observable';
import { _throw as observableThrow } from 'rxjs/observable/throw';
import { catchError, map } from 'rxjs/operators';

import { UserInfo, UserInfoServerResponse, UserService } from '../core/user/user.service';

export type LoginServerResponse = UserInfoServerResponse;

export class InvalidLoginCredentialError { }

@Injectable()
export class LoginService {
  constructor(private http: HttpClient, private userService: UserService) { }

  login(email: string, password: string): Observable<UserInfo> {
    return this.http
      .post<LoginServerResponse>('/api/login-username/', {
        username: email,
        password
      })
      .pipe(
        map(result => {
          const userInfo: UserInfo = {
            firstName: result.first_name,
            lastName: result.last_name,
            email: result.email,
            isLessor: result.is_lessor,
            phone: result.phone,
            profileImage: result.profile_image
          };
          this.userService.setLoginData(userInfo);
          return userInfo;
        }),
        catchError(err => {
          if (err instanceof HttpErrorResponse) {
            if (err.status >= 400 && err.status < 500) {
              return observableThrow(new InvalidLoginCredentialError());
            }
          }
          return observableThrow(err);
        })
      );
  }
}


text:
//
// DbValue.h
//

#ifndef _H_APPLEDL_DBVALUE
#define _H_APPLEDL_DBVALUE

#include "ReadWriteSection.h"

#include <security_cdsa_utilities/cssmdata.h>
#include <security_cdsa_utilities/cssmdb.h>
#include <Security/cssmerr.h>
#include <map>
#include <vector>

namespace Security
{

//
// DbValue -- A base class for all types of database values.
//
class DbValue
{
public:
	virtual ~DbValue();
};

// A collection of subclasses of DbValue that work for simple
// data types, e.g. uint32, sint32, and double, that have
// the usual C comparison and sizeof operations. Defining this
// template saves typing below.

template <class T>
class BasicValue : public DbValue
{
public:
	BasicValue() {}
	BasicValue(T value) : mValue(value) {}

	bool evaluate(const BasicValue<T> &other, CSSM_DB_OPERATOR op) const
	{
		switch (op) {

		case CSSM_DB_EQUAL:
			return mValue == other.mValue;

		case CSSM_DB_NOT_EQUAL:
			return mValue != other.mValue;

		case CSSM_DB_LESS_THAN:
			return mValue < other.mValue;

		case CSSM_DB_GREATER_THAN:
			return mValue > other.mValue;

		default:
			CssmError::throwMe(CSSMERR_DL_UNSUPPORTED_QUERY);
			return false;
		}
	}

	size_t size() const { return sizeof(T); }
	size_t size(const ReadSection &rs, uint32 offset) const { return size(); }
	const uint8 *bytes() const { return reinterpret_cast<const uint8 *>(&mValue); }

protected:
	T mValue;
};

// Actual useful subclasses of DbValue as instances of BasicValue.
// Note that all of these require a constructor of the form
// (const ReadSection &, uint32 &offset) that advances the offset
// to just after the value.

class UInt32Value : public BasicValue<uint32>
{
public:
	UInt32Value(const ReadSection &rs, uint32 &offset);
	UInt32Value(const CSSM_DATA &data);
	virtual ~UInt32Value();
	void pack(WriteSection &ws, uint32 &offset) const;
};

class SInt32Value : public BasicValue<sint32>
{
public:
	SInt32Value(const ReadSection &rs, uint32 &offset);
	SInt32Value(const CSSM_DATA &data);
	virtual ~SInt32Value();
	void pack(WriteSection &ws, uint32 &offset) const;
};

class DoubleValue : public BasicValue<double>
{
public:
	DoubleValue(const ReadSection &rs, uint32 &offset);
	DoubleValue(const CSSM_DATA &data);
	virtual ~DoubleValue();
	void pack(WriteSection &ws, uint32 &offset) const;
};

// Subclasses of Value for more complex types.

class BlobValue : public DbValue, public CssmData
{
public:
	BlobValue() {}
	BlobValue(const ReadSection &rs, uint32 &offset);
	BlobValue(const CSSM_DATA &data);
	virtual ~BlobValue();
	void pack(WriteSection &ws, uint32 &offset) const;
	bool evaluate(const BlobValue &other, CSSM_DB_OPERATOR op) const;

	size_t size() const { return Length; }
	const uint8 *bytes() const { return Data; }

protected:
	class Comparator {
	public:
		virtual ~Comparator();
		virtual int operator () (const uint8 *ptr1, const uint8 *ptr2, uint32 length);
	};

	static bool evaluate(const CssmData &data1, const CssmData &data2, CSSM_DB_OPERATOR op,
		Comparator compare);
};

class TimeDateValue : public BlobValue
{
public:
	enum { kTimeDateSize = 16 };

	TimeDateValue(const ReadSection &rs, uint32 &offset);
	TimeDateValue(const CSSM_DATA &data);
	virtual ~TimeDateValue();
	void pack(WriteSection &ws, uint32 &offset) const;

	bool isValidDate() const;

private:
	uint32 rangeValue(uint32 start, uint32 length) const;
};

class StringValue : public BlobValue
{
public:
	StringValue(const ReadSection &rs, uint32 &offset);
	StringValue(const CSSM_DATA &data);
	virtual ~StringValue();
	bool evaluate(const StringValue &other, CSSM_DB_OPERATOR op) const;

private:
	class Comparator : public BlobValue::Comparator {
	public:
		virtual int operator () (const uint8 *ptr1, const uint8 *ptr2, uint32 length);
	};

};

class BigNumValue : public BlobValue
{
public:
	static const uint8 kSignBit = 0x80;

	BigNumValue(const ReadSection &rs, uint32 &offset);
	BigNumValue(const CSSM_DATA &data);
	virtual ~BigNumValue();
	bool evaluate(const BigNumValue &other, CSSM_DB_OPERATOR op) const;

private:
	static int compare(const uint8 *a, const uint8 *b, int length);
};

class MultiUInt32Value : public DbValue
{
public:
	MultiUInt32Value(const ReadSection &rs, uint32 &offset);
	MultiUInt32Value(const CSSM_DATA &data);
	virtual ~MultiUInt32Value();
	void pack(WriteSection &ws, uint32 &offset) const;
	bool evaluate(const MultiUInt32Value &other, CSSM_DB_OPERATOR op) const;

	size_t size() const { return mNumValues * sizeof(uint32); }
	const uint8 *bytes() const { return reinterpret_cast<uint8 *>(mValues); }

private:
	uint32 mNumValues;
	uint32 *mValues;
	bool mOwnsValues;
};

} // end namespace Security

#endif // _H_APPLEDL_DBVALUE


text:
<?php
/**
 * Drupal_Sniffs_Formatting_SpaceInlineIfSniff.
 *
 * PHP version 5
 *
 * @category PHP
 * @package  PHP_CodeSniffer
 * @link     http://pear.php.net/package/PHP_CodeSniffer
 */

/**
 * Checks that there is no space between "?" and ":" inline if/else statements.
 *
 * @category PHP
 * @package  PHP_CodeSniffer
 * @link     http://pear.php.net/package/PHP_CodeSniffer
 */
class Drupal_Sniffs_Formatting_SpaceInlineIfSniff implements PHP_CodeSniffer_Sniff
{

    /**
     * Returns an array of tokens this test wants to listen for.
     *
     * @return array
     */
    public function register()
    {
        return array(T_INLINE_ELSE);

    }//end register()

    /**
     * Processes this test, when one of its tokens is encountered.
     *
     * @param PHP_CodeSniffer_File $phpcsFile The file being scanned.
     * @param int                  $stackPtr  The position of the current token in
     *                                        the stack passed in $tokens.
     *
     * @return void
     */
    public function process(PHP_CodeSniffer_File $phpcsFile, $stackPtr)
    {
        $tokens = $phpcsFile->getTokens();

        // Handle the short ternary operator (?:) introduced in PHP 5.3.
        $previous = $phpcsFile->findPrevious(T_WHITESPACE, ($stackPtr - 1), null, true);
        if ($tokens[$previous]['code'] === T_INLINE_THEN) {
            if ($previous !== ($stackPtr - 1)) {
                $error = 'There must be no space between ? and :';
                $phpcsFile->addError($error, $stackPtr, 'SpaceInlineElse');
            }
        }//end if

    }//end process()

}//end class


text:
package com.github.heuermh.adam.commands

import grizzled.slf4j.Logging
import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.spark.rdd.RDD
import org.bdgenomics.adam.ds.ADAMContext._
import org.bdgenomics.adam.ds.read.AlignmentDataset
import org.bdgenomics.formats.avro.Alignment
import org.bdgenomics.utils.cli._
import org.kohsuke.args4j.Argument

object CountAlignmentsPerRead extends BDGCommandCompanion {
  val commandName = "count_alignments_per_read"
  val commandDescription = "Counts the alignments per read in a read dataset."

  def apply(cmdLine: Array[String]) = {
    new CountAlignmentsPerRead(Args4j[CountAlignmentsPerReadArgs](cmdLine))
  }
}

class CountAlignmentsPerReadArgs extends Args4jBase {
  @Argument(required = true, metaVar = "INPUT", usage = "The ADAM, BAM or SAM file to count alignments per read from", index = 0)
  var inputPath: String = null
}

/**
 * Count alignments per read ADAM command.
 *
 * @author  Michael Heuer
 */
class CountAlignmentsPerRead(protected val args: CountAlignmentsPerReadArgs) extends BDGSparkCommand[CountAlignmentsPerReadArgs] with Logging {
  val companion = CountAlignmentsPerRead

  def run(sc: SparkContext) {
    var alignments: AlignmentDataset = sc.loadAlignments(args.inputPath)
    var rdd: RDD[Alignment] = alignments.rdd

    rdd.map(rec => if (rec.getReadMapped) rec.getReadName else "unmapped")
      .map(readName => (readName, 1))
      .reduceByKey(_ + _)
      .foreach(println)
  }
}


text:
<?php

/**
 * Author: imsamurai <im.samuray@gmail.com>
 * Date: Jan 14, 2014
 * Time: 6:36:02 PM
 * Format: http://book.cakephp.org/2.0/en/models.html
 */
App::uses('LocalizationAppModel', 'Localization.Model');

/**
 * Localization Model
 *
 * @property Language $Language Localization language model
 * @property Message $Message Localization message model
 * @property MessageReference $MessageReference Localization message reference model
 * @property Translation $Translation Localization translation model
 *
 * @package Localization
 * @subpackage Model
 */
class Localization extends LocalizationAppModel {

	/**
	 * {@inheritdoc}
	 *
	 * @var array
	 */
	public $useTable = false;

	/**
	 * {@inheritdoc}
	 *
	 * @param bool|int|string|array $id Set this ID for this model on startup,
	 *   can also be an array of options, see above.
	 * @param string $table Name of database table to use.
	 * @param string $ds DataSource connection name.
	 */
	public function __construct($id = false, $table = null, $ds = null) {
		parent::__construct($id, $table, $ds);
		$this->Language = ClassRegistry::init('Localization.Language');
		$this->Message = ClassRegistry::init('Localization.Message');
		$this->MessageReference = ClassRegistry::init('Localization.MessageReference');
		$this->Translation = ClassRegistry::init('Localization.Translation');
	}

	/**
	 * Export data
	 *
	 * @return array
	 */
	public function export() {
		$Created = new DateTime();
		return array(
			'Export' => array(
				'created_timestamp' => $Created->getTimestamp(),
				'created' => $Created->format('Y-m-d H:i:s')
			),
			'Languages' => Hash::extract((array)$this->Language->find('all'), '{n}.{s}'),
			'Messages' => Hash::extract((array)$this->Message->find('all'), '{n}.{s}'),
			'References' => Hash::extract((array)$this->MessageReference->find('all'), '{n}.{s}'),
			'Translations' => Hash::extract((array)$this->Translation->find('all'), '{n}.{s}')
		);
	}

	/**
	 * Import data
	 *
	 * @param array $data
	 * @return bool
	 */
	public function import(array $data) {
		$success = $this->clearAll();
		$success = (bool)$this->Language->saveAll($data['Languages']) && $success;
		$success = (bool)$this->Message->saveAll($data['Messages']) && $success;
		$success = (bool)$this->MessageReference->saveAll($data['References']) && $success;
		$success = (bool)$this->Translation->saveAll($data['Translations']) && $success;
		return $success;
	}

	/**
	 * Delete all localization records
	 *
	 * @return bool
	 */
	public function clearAll() {
		$success = true;
		$success = (bool)$this->Language->deleteAll(array(1 => 1), false) && $success;
		$success = (bool)$this->Message->deleteAll(array(1 => 1), false) && $success;
		$success = (bool)$this->MessageReference->deleteAll(array(1 => 1), false) && $success;
		$success = (bool)$this->Translation->deleteAll(array(1 => 1), false) && $success;
		return $success;
	}

}


text:
package com.grietenenknapen.sithandroid.model.database.relations;

import java.util.Arrays;

public class GamePlayerRelations {
    public static final String PLAYED_GAME_ID = "playedGameId";
    public static final String GAME_PLAYER_ID = "gamePlayerId";
    public static final String SITH_CARD_IDS = "sithCardIds";

    private long _id;
    private long playedGameId;
    private long gamePlayerId;
    private String sithCardIds;

    private GamePlayerRelations(Builder builder) {
        _id = builder._id;
        playedGameId = builder.playedGameId;
        gamePlayerId = builder.gamePlayerId;
        sithCardIds = Arrays.toString(builder.sithCardIds);
    }

    public static Builder newBuilder() {
        return new Builder();
    }

    public static Builder newBuilder(GamePlayerRelations copy) {
        Builder builder = new Builder();
        builder._id = copy._id;
        builder.playedGameId = copy.playedGameId;
        builder.gamePlayerId = copy.gamePlayerId;
        builder.sithCardIds = copy.getSithCardIds();
        return builder;
    }

    public long getId() {
        return _id;
    }

    public long getPlayedGameId() {
        return playedGameId;
    }

    public long getGamePlayerId() {
        return gamePlayerId;
    }

    public long[] getSithCardIds() {
        String[] s = sithCardIds.split(",");
        long[] ids = new long[s.length];
        for (int curr = 0; curr < s.length; curr++)
            ids[curr] = Long.parseLong(s[curr]);
        return ids;
    }

    /**
     * {@code GamePlayerRelations} builder static inner class.
     */
    public static final class Builder {
        private long _id;
        private long playedGameId;
        private long gamePlayerId;
        private long[] sithCardIds;

        private Builder() {
        }

        /**
         * Sets the {@code _id} and returns a reference to this Builder so that the methods can be chained together.
         *
         * @param val the {@code _id} to set
         * @return a reference to this Builder
         */
        public Builder _id(long val) {
            _id = val;
            return this;
        }

        /**
         * Sets the {@code playedGameId} and returns a reference to this Builder so that the methods can be chained together.
         *
         * @param val the {@code playedGameId} to set
         * @return a reference to this Builder
         */
        public Builder playedGameId(long val) {
            playedGameId = val;
            return this;
        }

        /**
         * Sets the {@code gamePlayerId} and returns a reference to this Builder so that the methods can be chained together.
         *
         * @param val the {@code gamePlayerId} to set
         * @return a reference to this Builder
         */
        public Builder gamePlayerId(long val) {
            gamePlayerId = val;
            return this;
        }

        /**
         * Sets the {@code sithCardIds} and returns a reference to this Builder so that the methods can be chained together.
         *
         * @param val the {@code sithCardIds} to set
         * @return a reference to this Builder
         */
        public Builder sithCardIds(long[] val) {
            sithCardIds = val;
            return this;
        }

        /**
         * Returns a {@code GamePlayerRelations} built from the parameters previously set.
         *
         * @return a {@code GamePlayerRelations} built with parameters of this {@code GamePlayerRelations.Builder}
         */
        public GamePlayerRelations build() {
            return new GamePlayerRelations(this);
        }
    }
}


text:
import axios from 'axios';
import request from '../utils/request';

export function getCars(type) {
  const url = `/api/e_bike_model?category=${type}`;
  return axios.get(url);
}


text:
module.exports = {
	EnglishLexicon: require('./english-lexicon/');
}


text:
import { Component, OnInit } from '@angular/core';

@Component({
  selector: 'app-customer-detail',
  templateUrl: './customer-detail.component.html',
  styleUrls: ['./customer-detail.component.css']
})
export class CustomerDetailComponent implements OnInit {

  constructor() { }

  ngOnInit() {
  }

}


text:
package libbuildpack_test

import (
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/cloudfoundry/libbuildpack"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("JSON", func() {
	var (
		json   *libbuildpack.JSON
		tmpDir string
		err    error
	)

	BeforeEach(func() {
		tmpDir, err = ioutil.TempDir("", "json")
		Expect(err).To(BeNil())

		json = &libbuildpack.JSON{}
	})

	AfterEach(func() {
		err = os.RemoveAll(tmpDir)
		Expect(err).To(BeNil())
	})

	Describe("Load", func() {
		Context("file is valid json", func() {
			BeforeEach(func() {
				ioutil.WriteFile(filepath.Join(tmpDir, "valid.json"), []byte(`{"key": "value"}`), 0666)
			})
			It("returns an error", func() {
				obj := make(map[string]string)
				err = json.Load(filepath.Join(tmpDir, "valid.json"), &obj)

				Expect(err).To(BeNil())
				Expect(obj["key"]).To(Equal("value"))
			})
		})

		Context("file is NOT valid json", func() {
			BeforeEach(func() {
				ioutil.WriteFile(filepath.Join(tmpDir, "invalid.json"), []byte("not valid json"), 0666)
			})
			It("returns an error", func() {
				obj := make(map[string]string)
				err = json.Load(filepath.Join(tmpDir, "invalid.json"), &obj)
				Expect(err).ToNot(BeNil())
			})
		})

		Context("file does not exist", func() {
			It("returns an error", func() {
				obj := make(map[string]string)
				err = json.Load(filepath.Join(tmpDir, "does_not_exist.json"), &obj)
				Expect(err).ToNot(BeNil())
			})
		})
	})

	Describe("Write", func() {
		Context("directory exists", func() {
			It("writes the json to a file ", func() {
				obj := map[string]string{
					"key": "val",
				}
				err = json.Write(filepath.Join(tmpDir, "file.json"), obj)
				Expect(err).To(BeNil())

				Expect(ioutil.ReadFile(filepath.Join(tmpDir, "file.json"))).To(Equal([]byte(`{"key":"val"}`)))
			})
		})

		Context("directory does not exist", func() {
			It("creates the directory", func() {
				obj := map[string]string{
					"key": "val",
				}
				err = json.Write(filepath.Join(tmpDir, "extradir", "file.json"), obj)
				Expect(err).To(BeNil())

				Expect(ioutil.ReadFile(filepath.Join(tmpDir, "extradir", "file.json"))).To(Equal([]byte(`{"key":"val"}`)))
			})
		})
	})
})


text:
package org.jivesoftware.smackx.provider;

import org.jivesoftware.smack.packet.PacketExtension;
import org.jivesoftware.smack.provider.PacketExtensionProvider;
import org.jivesoftware.smackx.FormField;
import org.jivesoftware.smackx.packet.DataForm;
import org.xmlpull.v1.XmlPullParser;

import java.util.ArrayList;
import java.util.List;

/**
 * The DataFormProvider parses DataForm packets.
 *
 * @author Gaston Dombiak
 */
public class DataFormProvider implements PacketExtensionProvider {

    /**
     * Creates a new DataFormProvider.
     * ProviderManager requires that every PacketExtensionProvider has a public, no-argument constructor
     */
    public DataFormProvider() {
    }

    public PacketExtension parseExtension(XmlPullParser parser) throws Exception {
        boolean done = false;
        StringBuilder buffer = null;
        DataForm dataForm = new DataForm(parser.getAttributeValue("", "type"));
        while (!done) {
            int eventType = parser.next();
            if (eventType == XmlPullParser.START_TAG) {
                if (parser.getName().equals("instructions")) {
                    dataForm.addInstruction(parser.nextText());
                }
                else if (parser.getName().equals("title")) {
                    dataForm.setTitle(parser.nextText());
                }
                else if (parser.getName().equals("field")) {
                    dataForm.addField(parseField(parser));
                }
                else if (parser.getName().equals("item")) {
                    dataForm.addItem(parseItem(parser));
                }
                else if (parser.getName().equals("reported")) {
                    dataForm.setReportedData(parseReported(parser));
                }
            } else if (eventType == XmlPullParser.END_TAG) {
                if (parser.getName().equals(dataForm.getElementName())) {
                    done = true;
                }
            }
        }
        return dataForm;
    }

    private FormField parseField(XmlPullParser parser) throws Exception {
        boolean done = false;
        FormField formField = new FormField(parser.getAttributeValue("", "var"));
        formField.setLabel(parser.getAttributeValue("", "label"));
        formField.setType(parser.getAttributeValue("", "type"));
        while (!done) {
            int eventType = parser.next();
            if (eventType == XmlPullParser.START_TAG) {
                if (parser.getName().equals("desc")) {
                    formField.setDescription(parser.nextText());
                }
                else if (parser.getName().equals("value")) {
                    formField.addValue(parser.nextText());
                }
                else if (parser.getName().equals("required")) {
                    formField.setRequired(true);
                }
                else if (parser.getName().equals("option")) {
                    formField.addOption(parseOption(parser));
                }
            } else if (eventType == XmlPullParser.END_TAG) {
                if (parser.getName().equals("field")) {
                    done = true;
                }
            }
        }
        return formField;
    }

    private DataForm.Item parseItem(XmlPullParser parser) throws Exception {
        boolean done = false;
        List<FormField> fields = new ArrayList<FormField>();
        while (!done) {
            int eventType = parser.next();
            if (eventType == XmlPullParser.START_TAG) {
                if (parser.getName().equals("field")) {
                    fields.add(parseField(parser));
                }
            } else if (eventType == XmlPullParser.END_TAG) {
                if (parser.getName().equals("item")) {
                    done = true;
                }
            }
        }
        return new DataForm.Item(fields);
    }

    private DataForm.ReportedData parseReported(XmlPullParser parser) throws Exception {
        boolean done = false;
        List<FormField> fields = new ArrayList<FormField>();
        while (!done) {
            int eventType = parser.next();
            if (eventType == XmlPullParser.START_TAG) {
                if (parser.getName().equals("field")) {
                    fields.add(parseField(parser));
                }
            } else if (eventType == XmlPullParser.END_TAG) {
                if (parser.getName().equals("reported")) {
                    done = true;
                }
            }
        }
        return new DataForm.ReportedData(fields);
    }

    private FormField.Option parseOption(XmlPullParser parser) throws Exception {
        boolean done = false;
        FormField.Option option = null;
        String label = parser.getAttributeValue("", "label");
        while (!done) {
            int eventType = parser.next();
            if (eventType == XmlPullParser.START_TAG) {
                if (parser.getName().equals("value")) {
                    option = new FormField.Option(label, parser.nextText());
                }
            } else if (eventType == XmlPullParser.END_TAG) {
                if (parser.getName().equals("option")) {
                    done = true;
                }
            }
        }
        return option;
    }
}


text:
'use strict';

const fs = require('fs');
const path = require('path');

module.exports = function() {
    const packageJsonPath = path.resolve(__dirname, '../../package.json');
    if (!fs.existsSync(packageJsonPath)) {
        console.error(`Something went wrong; couldn't get the version.`);
        console.error(`If the problem persists, try reinstalling swagen.`);
        return;
    }
    const packageJson = require(packageJsonPath);
    console.log(packageJson.version);
};