text:
import {Box, BoxView, BoxData} from "./box"
import {Arrayable} from "core/types"
import * as p from "core/properties"
import {SpatialIndex} from "core/util/spatial"

export interface HBarData extends BoxData {
  _left: Arrayable<number>
  _y: Arrayable<number>
  _height: Arrayable<number>
  _right: Arrayable<number>

  sy: Arrayable<number>
  sh: Arrayable<number>
  sleft: Arrayable<number>
  sright: Arrayable<number>
  stop: Arrayable<number>
  sbottom: Arrayable<number>

  max_height: number
}

export interface HBarView extends HBarData {}

export class HBarView extends BoxView {
  model: HBar
  visuals: HBar.Visuals

  scenterx(i: number): number {
    return (this.sleft[i] + this.sright[i])/2
  }

  scentery(i: number): number {
    return this.sy[i]
  }

  protected _index_data(): SpatialIndex {
    return this._index_box(this._y.length)
  }

  protected _lrtb(i: number): [number, number, number, number] {
    const l = Math.min(this._left[i], this._right[i])
    const r = Math.max(this._left[i], this._right[i])
    const t = this._y[i] + 0.5*this._height[i]
    const b = this._y[i] - 0.5*this._height[i]
    return [l, r, t, b]
  }

  protected _map_data(): void {
    this.sy = this.renderer.yscale.v_compute(this._y)
    this.sh = this.sdist(this.renderer.yscale, this._y, this._height, "center")
    this.sleft = this.renderer.xscale.v_compute(this._left)
    this.sright = this.renderer.xscale.v_compute(this._right)

    const n = this.sy.length
    this.stop = new Float64Array(n)
    this.sbottom = new Float64Array(n)
    for (let i = 0; i < n; i++) {
      this.stop[i] = this.sy[i] - this.sh[i]/2
      this.sbottom[i] = this.sy[i] + this.sh[i]/2
    }

    this._clamp_viewport()
  }
}

export namespace HBar {
  export type Attrs = p.AttrsOf<Props>

  export type Props = Box.Props & {
    left: p.CoordinateSpec
    y: p.CoordinateSpec
    height: p.NumberSpec
    right: p.CoordinateSpec
  }

  export type Visuals = Box.Visuals
}

export interface HBar extends HBar.Attrs {}

export class HBar extends Box {
  properties: HBar.Props

  constructor(attrs?: Partial<HBar.Attrs>) {
    super(attrs)
  }

  static init_HBar(): void {
    this.prototype.default_view = HBarView

    this.coords([['left', 'y']])
    this.define<HBar.Props>({
      height: [ p.NumberSpec     ],
      right:  [ p.CoordinateSpec ],
    })
    this.override({ left: 0 })
  }
}


text:
package name.valery1707.test.serialization.util.spi;

import java.io.IOException;
import java.io.Reader;
import java.io.Writer;

public interface TypeProcessor<T> {
	boolean canProcess(Class<?> type);

	void write(Writer dst, T value) throws IOException;

	<R extends T> R read(Reader src, Class<R> type) throws IOException;
}

redpajama_set_name: RedPajamaGithub


text:
/**
 * SidebarEmptyPaneView.tsx
 */

import * as React from "react"
import styled from "styled-components"
import { boxShadow, withProps } from "./common"

const ButtonWrapper = styled.button`
    background-color: ${props => props.theme.background};
    color: ${props => props.theme.foreground};
    padding: 1em;
    border: 2px solid transparent;
    width: 100%;
    outline: none;
    cursor: pointer;
    transition: all 0.1s ease-in;

    pointer-events: all;

    &:hover {
        ${boxShadow} transform: translateY(-1px);
    }
`

export interface ButtonContainerProps {
    selected: boolean
}

const ButtonContainer = withProps<ButtonContainerProps>(styled.div)`
    padding-left: 32px;
    padding-right: 32px;

    transition: all 0.1s ease-in;

    background-color: ${props => (props.selected ? "rgba(0, 0, 0, 0.1)" : "transparent")};
    border-left: 2px solid ${props =>
        props.selected ? props.theme["highlight.mode.normal.background"] : "transparent"};
`

import { Sneakable } from "./Sneakable"

export interface ISidebarButtonProps {
    focused: boolean
    text: string | JSX.Element
    onClick: () => void
}

export class SidebarButton extends React.PureComponent<ISidebarButtonProps, {}> {
    public render(): JSX.Element {
        return (
            <ButtonContainer selected={this.props.focused}>
                <Sneakable callback={this.props.onClick}>
                    <ButtonWrapper onClick={this.props.onClick}>
                        <span>{this.props.text}</span>
                    </ButtonWrapper>
                </Sneakable>
            </ButtonContainer>
        )
    }
}


text:
flavored-markdown.css
=====================

Style your rendered markdown exactly like Github does


text:
namespace extensions {

namespace {

base::LazyInstance<std::set<const ExtensionFrameHelper*>> g_frame_helpers =
    LAZY_INSTANCE_INITIALIZER;

// Returns true if the render frame corresponding with |frame_helper| matches
// the given criteria.
bool RenderFrameMatches(const ExtensionFrameHelper* frame_helper,
                        ViewType match_view_type,
                        int match_window_id,
                        const std::string& match_extension_id) {
  if (match_view_type != VIEW_TYPE_INVALID &&
      frame_helper->view_type() != match_view_type)
    return false;

  // Not all frames have a valid ViewType, e.g. devtools, most GuestViews, and
  // unclassified detached WebContents.
  if (frame_helper->view_type() == VIEW_TYPE_INVALID)
    return false;

  // This logic matches ExtensionWebContentsObserver::GetExtensionFromFrame.
  blink::WebSecurityOrigin origin =
      frame_helper->render_frame()->GetWebFrame()->getSecurityOrigin();
  if (origin.isUnique() ||
      !base::EqualsASCII(base::StringPiece16(origin.protocol()),
                         kExtensionScheme) || (!match_extension_id.empty() &&
      !base::EqualsASCII(base::StringPiece16(origin.host()),
                         match_extension_id.c_str())))
    return false;

  if (match_window_id != extension_misc::kUnknownWindowId &&
      frame_helper->browser_window_id() != match_window_id)
    return false;
  return true;
}

// Runs every callback in |callbacks_to_be_run_and_cleared| while |frame_helper|
// is valid, and clears |callbacks_to_be_run_and_cleared|.
void RunCallbacksWhileFrameIsValid(
    base::WeakPtr<ExtensionFrameHelper> frame_helper,
    std::vector<base::Closure>* callbacks_to_be_run_and_cleared) {
  // The JavaScript code can cause re-entrancy. To avoid a deadlock, don't run
  // callbacks that are added during the iteration.
  std::vector<base::Closure> callbacks;
  callbacks_to_be_run_and_cleared->swap(callbacks);
  for (auto& callback : callbacks) {
    callback.Run();
    if (!frame_helper.get())
      return;  // Frame and ExtensionFrameHelper invalidated by callback.
  }
}

}  // namespace

ExtensionFrameHelper::ExtensionFrameHelper(content::RenderFrame* render_frame,
                                           Dispatcher* extension_dispatcher)
    : content::RenderFrameObserver(render_frame),
      content::RenderFrameObserverTracker<ExtensionFrameHelper>(render_frame),
      view_type_(VIEW_TYPE_INVALID),
      tab_id_(-1),
      browser_window_id_(-1),
      extension_dispatcher_(extension_dispatcher),
      did_create_current_document_element_(false),
      weak_ptr_factory_(this) {
  g_frame_helpers.Get().insert(this);
}

ExtensionFrameHelper::~ExtensionFrameHelper() {
  g_frame_helpers.Get().erase(this);
}

// static
std::vector<content::RenderFrame*> ExtensionFrameHelper::GetExtensionFrames(
    const std::string& extension_id,
    int browser_window_id,
    ViewType view_type) {
  std::vector<content::RenderFrame*> render_frames;
  for (const ExtensionFrameHelper* helper : g_frame_helpers.Get()) {
    if (RenderFrameMatches(helper, view_type, browser_window_id, extension_id))
      render_frames.push_back(helper->render_frame());
  }
  return render_frames;
}

// static
content::RenderFrame* ExtensionFrameHelper::GetBackgroundPageFrame(
    const std::string& extension_id) {
  for (const ExtensionFrameHelper* helper : g_frame_helpers.Get()) {
    if (RenderFrameMatches(helper, VIEW_TYPE_EXTENSION_BACKGROUND_PAGE,
                           extension_misc::kUnknownWindowId, extension_id)) {
      blink::WebLocalFrame* web_frame = helper->render_frame()->GetWebFrame();
      // Check if this is the top frame.
      if (web_frame->top() == web_frame)
        return helper->render_frame();
    }
  }
  return nullptr;
}

// static
bool ExtensionFrameHelper::IsContextForEventPage(const ScriptContext* context) {
  content::RenderFrame* render_frame = context->GetRenderFrame();
  return context->extension() && render_frame &&
         BackgroundInfo::HasLazyBackgroundPage(context->extension()) &&
         ExtensionFrameHelper::Get(render_frame)->view_type() ==
              VIEW_TYPE_EXTENSION_BACKGROUND_PAGE;
}

void ExtensionFrameHelper::DidCreateDocumentElement() {
  did_create_current_document_element_ = true;
  extension_dispatcher_->DidCreateDocumentElement(
      render_frame()->GetWebFrame());
  nw::DocumentHook2(true, render_frame(), extension_dispatcher_);
}

void ExtensionFrameHelper::DidFinishDocumentLoad() {
  extension_dispatcher_->DidFinishDocumentLoad(
      render_frame()->GetWebFrame());
  nw::DocumentHook2(false, render_frame(), extension_dispatcher_);
}

void ExtensionFrameHelper::DidCreateNewDocument() {
  did_create_current_document_element_ = false;
}

void ExtensionFrameHelper::RunScriptsAtDocumentStart() {
  DCHECK(did_create_current_document_element_);
  RunCallbacksWhileFrameIsValid(weak_ptr_factory_.GetWeakPtr(),
                                &document_element_created_callbacks_);
  // |this| might be dead by now.
}

void ExtensionFrameHelper::RunScriptsAtDocumentEnd() {
  RunCallbacksWhileFrameIsValid(weak_ptr_factory_.GetWeakPtr(),
                                &document_load_finished_callbacks_);
  // |this| might be dead by now.
}

void ExtensionFrameHelper::ScheduleAtDocumentStart(
    const base::Closure& callback) {
  document_element_created_callbacks_.push_back(callback);
}

void ExtensionFrameHelper::ScheduleAtDocumentEnd(
    const base::Closure& callback) {
  document_load_finished_callbacks_.push_back(callback);
}

void ExtensionFrameHelper::DidMatchCSS(
    const blink::WebVector<blink::WebString>& newly_matching_selectors,
    const blink::WebVector<blink::WebString>& stopped_matching_selectors) {
  extension_dispatcher_->content_watcher()->DidMatchCSS(
      render_frame()->GetWebFrame(), newly_matching_selectors,
      stopped_matching_selectors);
}

void ExtensionFrameHelper::DidCreateScriptContext(
    v8::Local<v8::Context> context,
    int extension_group,
    int world_id) {
  extension_dispatcher_->DidCreateScriptContext(
      render_frame()->GetWebFrame(), context, extension_group, world_id);
}

void ExtensionFrameHelper::WillReleaseScriptContext(
    v8::Local<v8::Context> context,
    int world_id) {
  extension_dispatcher_->WillReleaseScriptContext(
      render_frame()->GetWebFrame(), context, world_id);
}

bool ExtensionFrameHelper::OnMessageReceived(const IPC::Message& message) {
  bool handled = true;
  IPC_BEGIN_MESSAGE_MAP(ExtensionFrameHelper, message)
    IPC_MESSAGE_HANDLER(ExtensionMsg_DispatchOnConnect,
                        OnExtensionDispatchOnConnect)
    IPC_MESSAGE_HANDLER(ExtensionMsg_DeliverMessage, OnExtensionDeliverMessage)
    IPC_MESSAGE_HANDLER(ExtensionMsg_DispatchOnDisconnect,
                        OnExtensionDispatchOnDisconnect)
    IPC_MESSAGE_HANDLER(ExtensionMsg_SetTabId, OnExtensionSetTabId)
    IPC_MESSAGE_HANDLER(ExtensionMsg_UpdateBrowserWindowId,
                        OnUpdateBrowserWindowId)
    IPC_MESSAGE_HANDLER(ExtensionMsg_NotifyRenderViewType,
                        OnNotifyRendererViewType)
    IPC_MESSAGE_HANDLER(ExtensionMsg_Response, OnExtensionResponse)
    IPC_MESSAGE_HANDLER(ExtensionMsg_MessageInvoke, OnExtensionMessageInvoke)
    IPC_MESSAGE_UNHANDLED(handled = false)
  IPC_END_MESSAGE_MAP()
  return handled;
}

void ExtensionFrameHelper::OnExtensionDispatchOnConnect(
    int target_port_id,
    const std::string& channel_name,
    const ExtensionMsg_TabConnectionInfo& source,
    const ExtensionMsg_ExternalConnectionInfo& info,
    const std::string& tls_channel_id) {
  MessagingBindings::DispatchOnConnect(
      extension_dispatcher_->script_context_set(),
      target_port_id,
      channel_name,
      source,
      info,
      tls_channel_id,
      render_frame());
}

void ExtensionFrameHelper::OnExtensionDeliverMessage(int target_id,
                                                     const Message& message) {
  MessagingBindings::DeliverMessage(
      extension_dispatcher_->script_context_set(), target_id, message,
      render_frame());
}

void ExtensionFrameHelper::OnExtensionDispatchOnDisconnect(
    int port_id,
    const std::string& error_message) {
  MessagingBindings::DispatchOnDisconnect(
      extension_dispatcher_->script_context_set(), port_id, error_message,
      render_frame());
}

void ExtensionFrameHelper::OnExtensionSetTabId(int tab_id) {
  CHECK_EQ(tab_id_, -1);
  CHECK_GE(tab_id, 0);
  tab_id_ = tab_id;
}

void ExtensionFrameHelper::OnUpdateBrowserWindowId(int browser_window_id) {
  browser_window_id_ = browser_window_id;
}

void ExtensionFrameHelper::OnNotifyRendererViewType(ViewType type) {
  // TODO(devlin): It'd be really nice to be able to
  // DCHECK_EQ(VIEW_TYPE_INVALID, view_type_) here.
  view_type_ = type;
}

void ExtensionFrameHelper::OnExtensionResponse(int request_id,
                                               bool success,
                                               const base::ListValue& response,
                                               const std::string& error) {
  extension_dispatcher_->OnExtensionResponse(request_id,
                                             success,
                                             response,
                                             error);
}

void ExtensionFrameHelper::OnExtensionMessageInvoke(
    const std::string& extension_id,
    const std::string& module_name,
    const std::string& function_name,
    const base::ListValue& args,
    bool user_gesture) {
  extension_dispatcher_->InvokeModuleSystemMethod(render_frame(), extension_id,
                                                  module_name, function_name,
                                                  args, user_gesture);
}

}  // namespace extensions


text:
"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
    return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var core_1 = require('@angular/core');
var ErroComponent = (function () {
    function ErroComponent() {
    }
    ErroComponent.prototype.ngOnInit = function () {
    };
    ErroComponent = __decorate([
        core_1.Component({
            selector: 'app-erro',
            templateUrl: 'app/erro/erro.component.html',
            styleUrls: ['app/erro/erro.component.css']
        }),
        __metadata('design:paramtypes', [])
    ], ErroComponent);
    return ErroComponent;
}());
exports.ErroComponent = ErroComponent;
//# sourceMappingURL=erro.component.js.map


text:
didiCAT
=======

didiCAT is a Distributed request combinator and dispatcher.  It offers a unified view on a set of
underlying services.

Topology
--------

didiCAT dispatches requests to peers and children.  Each instance specifies which requests are
supported and how they are handled.

A didiCAT instance lives in a group of peer didiCAT instances (known as friends).  Each instance
governs a set of services over which it offers an abstract interface.  didiCAT instances find their
friends by contacting a peer_cnnctr at a previously known location.  As each didiCAT instance
governs a set of services, the instance needs to be configured.  The implementation specifying how
services are queried and combined is done in Ruby.  The specification of which handlers to use for
the request is done through triples (Turtle and RDF/XML are supported).

Using didiCAT
-------------

didiCAT uses the `peer_cnnctr` service to discover instances.  It has a Rails JSON backend and an
ember-cli frontend.  The backend uses sqlite and Virtuoso as a store.

### Necessary services

didiCAT uses a `peer_cnnctr`, Ruby on Rails, an Sqlite database and Virtuoso.

In order to use didiCAT, you need to have a didiCAT instance and a `peer_cnnctr`.  The `peer_cnnctr`
allows didiCAT instances to discover their peers.  You can find the `peer_cnnctr` at
`tenforce/peer_cnnctr`.

didiCAT stores its configuration, peers and children in a database.  Both a SPARQL1.1 endpoint and a
Relation database is used.  We used Virtuoso 7.1 as SPARQL endpoint, but other SPARQL1.1 compliant
stores should work also.  The SPARQL endpoint is configured in
`rails/config/didicat/sparql.yaml`. For a relational database Sqlite is used, but that too can be
swapped out for a different database in typical Rails fashion, see `rails/config/db.yaml`.

### Configuring didiCAT

The implementation of didiCAT's operations is handled by plugins.  A plugin configuration is a graph
which describes (and possibly further configures) a set of plugins.  Examples of these configurations
can be found in the `examples` folder.

Each configuration specifies the following:

- A uri for the new plugin, of type Plugin.
- A regex for the path to operate on.
- The filter for queries of this path, which is a selector which allows you to query only portions of the peers and children.
- The dispatcher, which specifies which specific requests should be sent to peers and children.
- The extractor, which extracts the required contents from the requests of the peers and children.
- The combinator, which combines the results of the children and peers to what the user would expect.

This architecture allows for reuse between the components.  Once a JSON list combinator is built once, it can be reused across installations.

The implementation of each of these objects is done through classes.  Each of the components has a base class:

- Filters extend `rails/app/models/node_filter.rb`
- Dispatchers extend `rails/app/models/dispatcher.rb`
- Extractors extend `rails/app/models/information_extractor.rb`
- Combinators extend `rails/app/models/combinator.rb`

The extentions are placed in their relevant subfolder.  Documentation regarding the intended operation can be found in the ruby files (or through RDoc).


text:
package org.broadinstitute.dsde.rawls.metrics

import org.broadinstitute.dsde.rawls.model.SubmissionStatuses.SubmissionStatus
import org.broadinstitute.dsde.rawls.model.Subsystems.Subsystem
import org.broadinstitute.dsde.rawls.model.WorkflowStatuses.WorkflowStatus
import org.broadinstitute.dsde.rawls.model.{Submission, Workspace, WorkspaceName}
import org.broadinstitute.dsde.rawls.util.MockitoTestUtils
import org.scalatest.concurrent.Eventually

/**
  * Created by rtitle on 7/14/17.
  */
trait RawlsStatsDTestUtils extends StatsDTestUtils { this: Eventually with MockitoTestUtils =>

  protected def expectedHttpRequestMetrics(method: String,
                                           path: String,
                                           statusCode: Int,
                                           expectedTimes: Int,
                                           subsystem: Option[Subsystem] = None
  ): Set[(String, String)] = {
    val prefix =
      s"test.${subsystem.map(s => s"subsystem.${s.toString}.").getOrElse("")}httpRequestMethod.$method.httpRequestUri.$path.httpResponseStatusCode.$statusCode"
    val expectedTimesStr = expectedTimes.toString
    Set(
      (s"$prefix.request", expectedTimesStr),
      (s"$prefix.latency.samples", expectedTimesStr)
    )
  }

  private def expectedWorkflowStatusBase(workspaceName: WorkspaceName,
                                         submissionId: String,
                                         workflowStatus: WorkflowStatus
  ): String =
    s"${workbenchMetricBaseName}.transient.workspace.${workspaceName.toString.replace('/', '.')}.submission.$submissionId.workflowStatus.${workflowStatus.toString}"

  protected def expectedWorkflowStatusMetric(workspaceName: WorkspaceName,
                                             submissionId: String,
                                             workflowStatus: WorkflowStatus,
                                             expectedTimes: Int
  ): (String, String) =
    (s"${expectedWorkflowStatusBase(workspaceName, submissionId, workflowStatus)}.count", expectedTimes.toString)

  protected def expectedWorkflowStatusGauge(workspaceName: WorkspaceName,
                                            submissionId: String,
                                            workflowStatus: WorkflowStatus,
                                            expectedRate: Int
  ): (String, String) =
    (s"${expectedWorkflowStatusBase(workspaceName, submissionId, workflowStatus)}.current", expectedRate.toString)

  protected def expectedWorkflowStatusMetric(workspace: Workspace,
                                             submission: Submission,
                                             workflowStatus: WorkflowStatus,
                                             expectedTimes: Int
  ): (String, String) =
    expectedWorkflowStatusMetric(workspace.toWorkspaceName, submission.submissionId, workflowStatus, expectedTimes)

  protected def expectedWorkflowStatusMetric(workspace: Workspace,
                                             submission: Submission,
                                             workflowStatus: WorkflowStatus
  ): (String, String) =
    expectedWorkflowStatusMetric(workspace, submission, workflowStatus, submission.workflows.size)

  private def expectedSubmissionStatusBase(workspaceName: WorkspaceName, submissionStatus: SubmissionStatus): String =
    s"${workbenchMetricBaseName}.transient.workspace.${workspaceName.toString.replace('/', '.')}.submissionStatus.${submissionStatus.toString}"

  protected def expectedSubmissionStatusMetric(workspace: Workspace,
                                               submissionStatus: SubmissionStatus,
                                               expectedTimes: Int = 1
  ): (String, String) =
    expectedSubmissionStatusMetric(workspace.toWorkspaceName, submissionStatus, expectedTimes)

  protected def expectedSubmissionStatusMetric(workspaceName: WorkspaceName,
                                               submissionStatus: SubmissionStatus,
                                               expectedTimes: Int
  ): (String, String) =
    (s"${expectedSubmissionStatusBase(workspaceName, submissionStatus)}.count", expectedTimes.toString)

  protected def expectedSubmissionStatusGauge(workspaceName: WorkspaceName,
                                              submissionStatus: SubmissionStatus,
                                              expectedRate: Int
  ): (String, String) =
    (s"${expectedSubmissionStatusBase(workspaceName, submissionStatus)}.current", expectedRate.toString)

  protected def expectedGlobalWorkflowStatusGauge(workflowStatus: WorkflowStatus, expectedRate: Int): (String, String) =
    (s"${workbenchMetricBaseName}.workflowStatus.${workflowStatus.toString}.current", expectedRate.toString)

  protected def expectedGlobalSubmissionStatusGauge(submissionStatus: SubmissionStatus,
                                                    expectedRate: Int
  ): (String, String) =
    (s"${workbenchMetricBaseName}.submissionStatus.${submissionStatus.toString}.current", expectedRate.toString)
}


text:
class AppOfflineApplication: public PollingAppOfflineApplication
{
public:
    AppOfflineApplication(const IHttpApplication& pApplication)
        : PollingAppOfflineApplication(pApplication, StopWhenRemoved)
    {
        CheckAppOffline();
    }

    HRESULT CreateHandler(IHttpContext* pHttpContext, IREQUEST_HANDLER** pRequestHandler) override;

    HRESULT OnAppOfflineFound() override;

    static bool ShouldBeStarted(const IHttpApplication& pApplication);

private:
    std::string m_strAppOfflineContent;
};


text:
from __future__ import unicode_literals

from django.db import migrations, models

def cleanup_tokens(apps, schema_editor):
    AuthToken = apps.get_model('knox', 'AuthToken')
    AuthToken.objects.filter(token_key__isnull=True).delete()

class Migration(migrations.Migration):

    dependencies = [
        ('knox', '0005_authtoken_token_key'),
    ]

    operations = [
        migrations.RunPython(cleanup_tokens),
        migrations.AlterField(
            model_name='authtoken',
            name='token_key',
            field=models.CharField(db_index=True, default='', max_length=8),
            preserve_default=False,
        ),
    ]


text:
import sys, os, stat, re, optparse, multiprocessing, cStringIO, fnmatch, time, hashlib, errno
import glob, logging, traceback, Queue, types, threading, random, ConfigParser

import boto
import boto.s3
import boto.s3.key
import boto.exception

# Global constants

SINGLEPART_UPLOAD_MAX = 4500 * 1024 * 1024 # Max file size to upload without S3 multipart upload
SINGLEPART_DOWNLOAD_MAX = 50 * 1024 * 1024 # Max file size to download with single thread
DEFAULT_SPLIT = 50 * 1024 * 1024
DEFAULT_RETRY = 3
RETRY_DELAY = 10
PATH_SEP = '/'
DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S UTC'
TIMESTAMP_REGEX = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2}).(\d{3})Z')
TIMESTAMP_FORMAT = '%4s-%2s-%2s %2s:%2s'
INDENT = "  "

# Temp/buffer globals.
TEMP_FILES = set()
BUFFER_KEYS = ["dirs", "files", "buckets"]

# Environment variable names for S3 credentials.
S3_ACCESS_KEY_NAME = "S3_ACCESS_KEY"
S3_SECRET_KEY_NAME = "S3_SECRET_KEY"

##
## Utility classes
##

class Options:
  '''Default wrapper for running options
     This class contains most running options with default values. They can
     be overwritten by command line options.
  '''
  def __init__(self, opt=None):

    self.opt = None
    self.recursive = False
    self.force = False
    self.sync_check = False
    self.dry_run = False
    self.verbose = False
    self.debug = False
    self.use_ssl = False
    self.show_dir = False
    self.ignore_empty = False
    self.num_threads = 2
    self.retry = False

    if opt:

      '''Assign Argparse NameSpace to Class.dict'''
      [setattr(self, key, opt[key]) for key in opt]

      '''Setup multiprocessing'''

      if opt["num_threads"]:
        self.num_threads = opt["num_threads"]
      else:
        self.num_threads = int(
          os.getenv('S4CMD_NUM_THREADS',
            multiprocessing.cpu_count() * 4)
          )

class Failure(RuntimeError):
  '''Exception for runtime failures'''
  pass

class InvalidArgument(RuntimeError):
  '''Exception for invalid input parameters'''
  pass

class RetryFailure(Exception):
  '''Runtime failure that can be retried'''
  pass

def initialize(opt = None):
  '''Initialization of this tool. Setup loggers.'''

  if not opt:
    opt = Options()

  global log
  log = logging.Logger("s4cmd")

  if opt.debug:
    verbosity = 3
  elif opt.verbose:
    verbosity = 2
  else:
    verbosity = 1

  log.verbosity = verbosity
  log.stream = sys.stderr

  log_handler = logging.StreamHandler(log.stream)
  log.addHandler(log_handler)

  if verbosity == 3:
    log_handler.setFormatter(logging.Formatter('  (%(levelname).1s)%(filename)s:%(lineno)-4d %(message)s', DATETIME_FORMAT))
    log.setLevel(logging.DEBUG)
  elif verbosity == 2:
    log_handler.setFormatter(logging.Formatter('%(message)s', DATETIME_FORMAT))
    log.setLevel(logging.INFO)
  else:
    log_handler.setFormatter(logging.Formatter('%(message)s', DATETIME_FORMAT))
    log.setLevel(logging.ERROR)

# Call initalize() by default.
initialize()

def log_calls(func):
  '''Decorator to log function calls.'''
  def wrapper(*args, **kwargs):
    callStr = "%s(%s)" % (func.__name__, ", ".join([repr(p) for p in args] + ["%s=%s" % (k, repr(v)) for (k, v) in kwargs.items()]))
    debug(">> %s", callStr)
    ret = func(*args, **kwargs)
    debug("<< %s: %s", callStr, repr(ret))
    return ret
  return wrapper

##
## Utility functions
##

def synchronized(func):
  '''Decorator to synchronize function.'''
  func.__lock__ = threading.Lock()
  def synced_func(*args, **kargs):
    with func.__lock__:
      return func(*args, **kargs)
  return synced_func

def debug(msg, *args):
  '''Debug output wrapper.'''
  log.debug(msg, *args)

def info(msg, *args):
  '''Program verbose information output.'''
  log.info(msg, *args)

def warn(msg, *args):
  '''Program warnings.'''
  log.warn(msg, *args)

def error(msg, *args):
  '''Program error output.'''
  log.error(msg, *args)

def clear_progress():
  '''Clear previous progress message, if any.'''
  progress('')

@synchronized
def progress(msg, *args):
  '''Show current progress message to stderr.
     This function will remember the previous message so that next time,
     it will clear the previous message before showing next one.
  '''
  # Don't show any progress if the output is directed to a file.
  if not sys.stderr.isatty():
    return

  text = (msg % args)
  if progress.prev_message:
    sys.stderr.write(' ' * len(progress.prev_message) + '\r')
  sys.stderr.write(text + '\r')
  progress.prev_message = text

progress.prev_message = None

@synchronized
def message(msg, *args):
  '''Program message output.'''
  clear_progress()
  text = (msg % args)
  sys.stdout.write(text + '\n')

def fail(message, exc_info = None, status = 1, stacktrace = False):
  '''Utility function to handle runtime failures gracefully.
     Show concise information if possible, then terminate program.
  '''
  text = message
  if exc_info:
    text += str(exc_info)
  error(text)
  if stacktrace:
    error(traceback.format_exc())
  clean_tempfiles()
  sys.exit(status)

@synchronized
def tempfile_get(target):
  '''Get a temp filename for atomic download.'''
  fn = '%s-%s.tmp' % (target, ''.join(random.Random().sample("0123456789abcdefghijklmnopqrstuvwxyz", 15)))
  TEMP_FILES.add(fn)
  return fn

@synchronized
def tempfile_set(tempfile, target):
  '''Atomically rename and clean tempfile'''
  if target:
    os.rename(tempfile, target)
  else:
    os.unlink(tempfile)

  if target in TEMP_FILES:
    TEMP_FILES.remove(tempfile)

def clean_tempfiles():
  '''Clean up temp files'''
  for fn in TEMP_FILES:
    if os.path.exists(fn):
      os.unlink(fn)

class S3URL:
  '''Simple wrapper for S3 URL.
     This class parses a S3 URL and provides accessors to each component.
  '''
  S3URL_PATTERN = re.compile(r'(s3[n]?)://([^/]+)[/]?(.*)')

  def __init__(self, uri):
    '''Initialization, parse S3 URL'''
    try:
      self.proto, self.bucket, self.path = S3URL.S3URL_PATTERN.match(uri).groups()
      self.proto = 's3' # normalize s3n => s3
    except:
      raise InvalidArgument('Invalid S3 URI: %s' % uri)

  def __str__(self):
    '''Return the original S3 URL'''
    return S3URL.combine(self.proto, self.bucket, self.path)

  def get_fixed_path(self):
    '''Get the fixed part of the path without wildcard'''
    pi = self.path.split(PATH_SEP)
    fi = []
    for p in pi:
      if '*' in p or '?' in p:
        break
      fi.append(p)
    return PATH_SEP.join(fi)

  @staticmethod
  def combine(proto, bucket, path):
    '''Combine each component and general a S3 url string, no path normalization
       here. The path should not start with slash.
    '''
    return '%s://%s/%s' % (proto, bucket, path)

  @staticmethod
  def is_valid(uri):
    '''Check if given uri is a valid S3 URL'''
    return S3URL.S3URL_PATTERN.match(uri) != None

class TaskQueue(Queue.Queue):
  '''Wrapper class to Queue.
     Since we need to ensure that main thread is not blocked by child threads
     and cannot be wake up by Ctrl-C interrupt, we have to override join()
     method.
  '''
  def __init__(self):
    Queue.Queue.__init__(self)
    self.exc_info = None

  def join(self):
    '''Override original join() with a timeout and handle keyboard interrupt.'''
    self.all_tasks_done.acquire()
    try:
      while self.unfinished_tasks:
        self.all_tasks_done.wait(1000)

        # Child thread has exceptions, fail main thread too.
        if self.exc_info:
          fail(INDENT + '[Thread Failure] ', exc_info = self.exc_info)
    except KeyboardInterrupt:
      raise Failure('Interrupted by user')
    finally:
      self.all_tasks_done.release()

  def terminate(self, exc_info = None):
    '''Terminate all threads by deleting the queue and forcing the child threads
       to quit.
    '''
    if exc_info:
      self.exc_info = exc_info
    try:
      while self.get_nowait():
        self.task_done()
    except Queue.Empty:
      pass

class ThreadPool(object):
  '''Utility class for thread pool.
     This class needs to work with a utility class, which is derived from Worker.
  '''

  class Worker(threading.Thread):
    '''Utility thread worker class.
       This class handles all items in task queue and execute them. It also
       handles runtime errors gracefully, and provides automatic retry.
    '''
    def __init__(self, pool):
      '''Thread worker initalization.
         Setup values and start threads right away.
      '''
      threading.Thread.__init__(self)
      self.pool = pool
      self.opt = pool.opt
      self.daemon = True
      self.start()

    def run(self):
      '''Main thread worker execution.
         This function extract items from task queue and execute them accordingly.
         It will retry tasks when encounter exceptions by putting the same item
         back to the work queue.
      '''
      while True:
        item = self.pool.tasks.get()
        if not item:
          break

        try:
          func_name, retry, args, kargs = item
          self.__class__.__dict__[func_name](self, *args, **kargs)
        except InvalidArgument, e:
          self.pool.tasks.terminate(e)
          fail('[Invalid Argument] ', exc_info = e)
        except Failure, e:
          self.pool.tasks.terminate(e)
          fail(INDENT + '[Runtime Failure] ', exc_info = e)
        # Also retry known S3ResponseError since S3 has transient
        # errors from time to time.
        # except boto.exception.S3ResponseError, e:
        #   self.pool.tasks.terminate(e)
        #   fail('[S3ResponseError] %s: %s' % (e.error_code, e.error_message))
        except OSError, e:
          self.pool.tasks.terminate(e)
          fail(INDENT + '[OSError] %d: %s' % (e.errno, e.strerror))
        except Exception, e:
          # XXX Should we retry on all unknown exceptions?
          if retry >= self.opt.retry:
            self.pool.tasks.terminate(e)
            fail(INDENT + '[Runtime Exception] ', exc_info = e, stacktrace = True)
          else:
            # Show content of exceptions.
            error(e)

          time.sleep(RETRY_DELAY)
          self.pool.tasks.put((func_name, retry + 1, args, kargs))
        finally:
          self.pool.processed()
          self.pool.tasks.task_done()

  def __init__(self, thread_class, opt):
    '''Constructor of ThreadPool.
       Create workers and pool will automatically inherit all methods from
       thread_class by redirecting calls through __getattribute__().
    '''
    self.opt = opt
    self.tasks = TaskQueue()
    self.processed_tasks = 0
    self.thread_class = thread_class
    self.workers = []
    for i in range(opt.num_threads):
      self.workers.append(thread_class(self))

  def __enter__(self):
    '''Utility function for with statement'''
    return self

  def __exit__(self, exc_type, exc_value, traceback):
    '''Utility function for with statement, wait for completion'''
    self.join()
    return isinstance(exc_value, TypeError)

  def __getattribute__(self, name):
    '''Special attribute accessor to add tasks into task queue.
       Here if we found a function not in ThreadPool, we will try
       to see if we have a function in the utility class. If so, we
       add the function call into task queue.
    '''
    try:
      attr = super(ThreadPool, self).__getattribute__(name)
    except AttributeError, e:
      if name in self.thread_class.__dict__:
        # Here we masquerade the original function with add_task(). So the
        # function call will be put into task queue.
        def deferred_task(*args, **kargs):
          self.add_task(name, *args, **kargs)
        attr = deferred_task
      else:
        raise AttributeError('Unable to resolve %s' % name)
    return attr

  def add_task(self, func_name, *args, **kargs):
    '''Utility function to add a single task into task queue'''
    self.tasks.put((func_name, 0, args, kargs))

  def join(self):
    '''Utility function to wait all tasks to complete'''
    self.tasks.join()

    # Force each thread to break loop.
    for worker in self.workers:
      self.tasks.put(None)

    # Wait for all thread to terminate.
    for worker in self.workers:
      worker.join()
      worker.s3 = None

  @synchronized
  def processed(self):
    '''Increase the processed task counter and show progress message'''
    self.processed_tasks += 1
    qsize = self.tasks.qsize()
    if qsize > 0:
      progress(INDENT + '[%d task(s) completed, %d remaining, %d thread(s)]', self.processed_tasks, qsize, len(self.workers))
    else:
      progress(INDENT + '[%d task(s) completed, %d thread(s)]', self.processed_tasks, len(self.workers))

class S3Handler(object):
  '''Core S3 class.
     This class provide the functions for all operations. It will start thread
     pool to execute tasks generated by each operation. See ThreadUtil for
     more details about the tasks.
  '''

  S3_KEYS = None

  @staticmethod
  def s3_keys_from_env():
    '''Retrieve S3 access keys from the environment, or None if not present.'''
    env = os.environ
    if S3_ACCESS_KEY_NAME in env and S3_SECRET_KEY_NAME in env:
      keys = (env[S3_ACCESS_KEY_NAME], env[S3_SECRET_KEY_NAME])
      log.debug("read S3 keys from environment")
      return keys
    else:
      return None

  @staticmethod
  def s3_keys_from_s3cfg():
    '''Retrieve S3 access key settings from s3cmd's config file, if present; otherwise return None.'''
    s3cfg_path = None

    try:
      s3cfg_path = "%s/.s3cfg" % os.environ["HOME"]
      if not os.path.exists(s3cfg_path):
        return None
      config = ConfigParser.ConfigParser()
      config.read(s3cfg_path)
      keys = config.get("default", "access_key"), config.get("default", "secret_key")
      log.debug("read S3 keys from $HOME/.s3cfg file")
      return keys

    except Exception, e:
      log.info("could not read S3 keys from $HOME/.s3cfg file; skipping (%s)", e)
      return None

  @staticmethod
  def init_s3_keys():
    '''Initialize s3 access keys from environment varialbe or s3cfg config file.'''
    S3Handler.S3_KEYS = S3Handler.s3_keys_from_env() or S3Handler.s3_keys_from_s3cfg()

  def __init__(self, opt = Options()):
    '''Constructor, connect to S3 store'''

    self.buffer = {
      k: [] for k in BUFFER_KEYS
      }

    self.s3 = None
    self.opt = opt
    self.connect()

  def __del__(self):
    '''Destructor, stop s3 connection'''
    if self.s3 is not None:
      self.s3.close()
      self.s3 = None

  def connect(self):
    '''Connect to S3 storage'''
    try:
      if S3Handler.S3_KEYS:
        self.s3 = boto.connect_s3(S3Handler.S3_KEYS[0],
                                  S3Handler.S3_KEYS[1],
                                  is_secure = self.opt.use_ssl,
                                  suppress_consec_slashes = False)

      else:
        self.s3 = boto.connect_s3(is_secure = self.opt.use_ssl,
                                  suppress_consec_slashes = False)
    except Exception, e:
      raise RetryFailure('Unable to connect to s3: %s' % e)

  @log_calls
  def list_buckets(self):
    '''List all buckets'''
    for bucket in self.s3.get_all_buckets():
      self.buffer["dirs"].append(S3URL.combine('s3', bucket.name, ''))
      self.buffer["buckets"].append({
          'name': S3URL.combine('s3', bucket.name, ''),
          'last_modified': bucket.creation_date,
          'is_dir': True,
          'size': 0
        })

    return self.buffer

  @log_calls
  def s3walk(self, basedir, show_dir = None):
    '''Walk through a S3 directory. This function initiates a walk with a basedir.
       It also supports multiple wildcards.
    '''
    # Provide the default value from command line if no override.
    if not show_dir:
      show_dir = self.opt.show_dir

    # trailing slash normalization, this is for the reason that we want
    # ls 's3://foo/bar/' has the same result as 's3://foo/bar'. Since we
    # call partial_match() to check wildcards, we need to ensure the number
    # of slashes stays the same when we do this.
    if basedir[-1] == PATH_SEP:
      basedir = basedir[0:-1]

    s3url = S3URL(basedir)
    result = []

    pool = ThreadPool(ThreadUtil, self.opt)
    pool.s3walk(s3url, s3url.get_fixed_path(), s3url.path, result)
    pool.join()

    # automatic directory detection
    if not show_dir and len(result) == 1 and result[0]['is_dir']:
      path = result[0]['name']
      s3url = S3URL(path)
      result = []
      pool = ThreadPool(ThreadUtil, self.opt)
      pool.s3walk(s3url, s3url.get_fixed_path(), s3url.path, result)
      pool.join()

    def compare(x, y):
      '''Comparator for ls output'''
      result = -cmp(x['is_dir'], y['is_dir'])
      if result != 0:
        return result
      return cmp(x['name'], y['name'])

    self.sorted = sorted(result, compare)
    for x in self.sorted:
      if x['is_dir']: self.buffer["dirs"].append(x['name'])
      else: self.buffer["files"].append(x['name'])
    return self.sorted

  @log_calls
  def local_walk(self, basedir):
    '''Walk through local directories from root basedir'''
    result = []

    for root, dirs, files in os.walk(basedir):
      for f in files:
        result.append(os.path.join(root, f))
    return result

  @log_calls
  def get_basename(self, path):
    '''Unix style basename.
       This fuction will return 'bar' for '/foo/bar/' instead of empty string.
       It is used to normalize the input trailing slash.
    '''
    if path[-1] == PATH_SEP:
      path = path[0:-1]
    return os.path.basename(path)

  def source_expand(self, source):
    '''Expand the wildcards for an S3 path. This emulates the shall expansion
       for wildcards if the input is local path.
    '''
    result = []

    if not isinstance(source, types.ListType):
      source = [source]

    for src in source:
      # XXX Hacky: We need to disable recursive when we expand the input
      #            parameters, need to pass this as an override parameter if
      #            provided.
      tmp = self.opt.recursive
      self.opt.recursive = False
      result += [f['name'] for f in self.s3walk(src, True)]
      self.opt.recursive = tmp

    if (len(result) == 0) and (not self.opt.ignore_empty_source):
      fail(INDENT + "[Runtime Failure] Source doesn't exist.")

    return result

  @log_calls
  def put_single_file(self, pool, source, target):
    '''Upload a single file or a directory by adding a task into queue'''

    if os.path.isdir(source):
      if self.opt.recursive:
        for f in filter(lambda f: not os.path.isdir(f), self.local_walk(source)):
          target_url = S3URL(target)
          # deal with ./ or ../ here by normalizing the path.
          joined_path = os.path.normpath(os.path.join(target_url.path, os.path.relpath(f, source)))
          pool.upload(None, f, S3URL.combine('s3', target_url.bucket, joined_path))
      else:
        message('omitting directory "%s".' % source)
    else:
      pool.upload(None, source, target)

  @log_calls
  def put_files(self, source, target):

    '''
      Upload files to S3.
      This function can handle multiple file upload if source is a list.
      Also, it works for recursive mode which copies all files and keeps the
      directory structure under the given source directory.
    '''

    pool = ThreadPool(ThreadUtil, self.opt)
    if not isinstance(source, types.ListType):
      source = [source]

    if target[-1] == PATH_SEP:
      for src in source:
        self.put_single_file(pool, src, os.path.join(target, self.get_basename(src)))
    else:
      if len(source) == 1:
        self.put_single_file(pool, source[0], target)
      else:
        raise Failure('Target "%s" is not a directory (with a trailing slash).' % target)

    pool.join()

  @log_calls
  def update_privilege(self, source, target):
    '''Get privileges from metadata of the source in s3, and apply them to target'''
    s3url = S3URL(source)
    bucket = self.s3.lookup(s3url.bucket)
    remoteKey = bucket.get_key(s3url.path)
    if 'privilege' in remoteKey.metadata:
      os.chmod(target, int(remoteKey.metadata['privilege'], 8))

  @log_calls
  def get_single_file(self, pool, source, target):
    '''Download a single file or a directory by adding a task into queue'''
    if source[-1] == PATH_SEP:
      if self.opt.recursive:
        basepath = S3URL(source).path
        for f in filter(lambda f: not f['is_dir'], self.s3walk(source)):
          pool.download(None, f['name'], os.path.join(target, os.path.relpath(S3URL(f['name']).path, basepath)))
      else:
        message('omitting directory "%s".' % source)
    else:
      pool.download(None, source, target)

  @log_calls
  def get_files(self, source, target):
    '''Download files.
       This function can handle multiple files if source S3 URL has wildcard
       characters. It also handles recursive mode by download all files and
       keep the directory structure.
    '''
    pool = ThreadPool(ThreadUtil, self.opt)
    source = self.source_expand(source)

    if os.path.isdir(target):
      for src in source:
        self.get_single_file(pool, src, os.path.join(target, self.get_basename(S3URL(src).path)))
    else:
      if len(source) > 1:
        raise Failure('Target "%s" is not a directory.' % target)
        # Get file if it exists on s3 otherwise do nothing
      elif len(source) == 1:
        self.get_single_file(pool, source[0], target)
      else:
        #Source expand may return empty list only if ignore-empty-source is set to true
        pass

    pool.join()

  @log_calls
  def cp_single_file(self, pool, source, target, delete_source):
    '''Copy a single file or a directory by adding a task into queue'''
    if source[-1] == PATH_SEP:
      if self.opt.recursive:
        basepath = S3URL(source).path
        for f in filter(lambda f: not f['is_dir'], self.s3walk(source)):
          pool.copy(f['name'], os.path.join(target, os.path.relpath(S3URL(f['name']).path, basepath)), delete_source)
      else:
        message('omitting directory "%s".' % source)
    else:
      pool.copy(source, target, delete_source)

  @log_calls
  def cp_files(self, source, target, delete_source = False):
    '''Copy files
       This function can handle multiple files if source S3 URL has wildcard
       characters. It also handles recursive mode by copying all files and
       keep the directory structure.
    '''
    pool = ThreadPool(ThreadUtil, self.opt)
    source = self.source_expand(source)

    if target[-1] == PATH_SEP:
      for src in source:
        self.cp_single_file(pool, src, os.path.join(target, self.get_basename(S3URL(src).path)), delete_source)
    else:
      if len(source) > 1:
        raise Failure('Target "%s" is not a directory (with a trailing slash).' % target)
        # Copy file if it exists otherwise do nothing
      elif len(source) == 1:
        self.cp_single_file(pool, source[0], target, delete_source)
      else:
        # Source expand may return empty list only if ignore-empty-source is set to true
        pass

    pool.join()

  @log_calls
  def del_files(self, source):
    '''Delete files on S3'''
    src_files = []
    for key in self.s3walk(source):
      if not key['is_dir']: # ignore directories
        src_files.append(key['name'])
    pool = ThreadPool(ThreadUtil, self.opt)

    for src_file in src_files:
      pool.delete(src_file)

    pool.join()

  @log_calls
  def sync_files(self, source, target):
    '''Sync files to S3. Does not implement deletions.
       Currently identical to get/put -r -f --sync-check.
    '''
    src_s3_url = S3URL.is_valid(source)
    dst_s3_url = S3URL.is_valid(target)

    if src_s3_url and not dst_s3_url:
      self.get_files(source, target)
    elif not src_s3_url and dst_s3_url:
      self.put_files(source, target)
    elif src_s3_url and dst_s3_url:
      self.cp_files(source, target)
    else:
      raise InvalidArgument('No S3 URI provided')

  @log_calls
  def size(self, source):
    '''Get the size component of the given s3url. If it is a
       directory, combine the sizes of all the files under
       that directory. Subdirectories will not be counted unless
       --recursive option is set.
    '''
    result = []
    for src in self.source_expand(source):
      size = 0
      for f in self.s3walk(src):
        size += f['size']
      result.append((src, size))

    return result

class ThreadUtil(S3Handler, ThreadPool.Worker):
  '''Thread workers for S3 operations.
     This class contains all thread workers for S3 operations.

     1) Expand source into [source] list if it contains wildcard characters '*' or '?'.
        This is done by shell, but we need to do this ourselves for S3 path.
        Basically we see [source] as the first-class source list.

     2) Run the following algorithm:
        if target is directory? (S3 path uses trailing slash to determine this)
          for src in source:
            copy src to target/src.basename
        else
          if source has only one element?
            copy src to target
          else
            error "target should be a directory"!

     3) Copy operations should work for both single file and directory:
        def copy(src, target)
          if src is a directory?
            copy the whole directory recursively to target
          else
            copy the file src to target
  '''

  def __init__(self, pool):
    '''Constructor'''
    S3Handler.__init__(self, pool.opt)
    ThreadPool.Worker.__init__(self, pool)

  def reset(self):
    '''Reset connection for retry.'''
    if self.s3:
      self.s3.close()
    self.connect()

  def auto_reset(func):
    '''Simple decorator for connection reset.
       This is necessary to have a clean connection if s3 connection failed.
    '''
    def wrapper(self, *args, **kwargs):
      self.reset()
      return func(self, *args, **kwargs)
    return wrapper

  @log_calls
  def mkdirs(self, target):
    '''Ensure all directories are created for a given target file.'''
    path = os.path.dirname(target)
    if path and path != '/' and not os.path.isdir(path):
      # Multi-threading means there will be intervleaved execution
      # between the check and creation of the directory.
      try:
        os.makedirs(path)
      except OSError, ose:
        if ose.errno != errno.EEXIST:
          raise Failure('Unable to create directory (%s)' % (path,))

  @log_calls
  def file_hash(self, filename, block_size = None):
    '''Calculate MD5 hash code for a local file'''
    if not block_size:
      block_size = 2**20
    m = hashlib.md5()
    with open(filename, 'rb') as f:
      while True:
        data = f.read(block_size)
        if not data:
          break
        m.update(data)
    return m.hexdigest()

  @log_calls
  def sync_check(self, localFilename, remoteKey):
    '''Check MD5 for a local file and a remote file.
       Return True if they have the same md5 hash, otherwise False.
    '''
    if not remoteKey:
      return False
    if not os.path.exists(localFilename):
      return False
    localmd5 = self.file_hash(localFilename)
    # check multiple md5 locations
    return (remoteKey.etag and remoteKey.etag == '"%s"' % localmd5) or \
           (remoteKey.md5 and remoteKey.md5 == localmd5) or \
           ('md5' in remoteKey.metadata and remoteKey.metadata['md5'] == localmd5)

  @log_calls
  def partial_match(self, path, filter_path):
    '''Partially match a path and a filter_path with wildcards.
       This function will return True if this path partially match a filter path.
       This is used for walking through directories with multiple level wildcard.
    '''
    if not path or not filter_path:
      return True

    # trailing slash normalization
    if path[-1] == PATH_SEP:
      path = path[0:-1]
    if filter_path[-1] == PATH_SEP:
      filter_path += '*'

    pi = path.split(PATH_SEP)
    fi = filter_path.split(PATH_SEP)

    # Here, if we are in recursive mode, we allow the pi to be longer than fi.
    # Otherwise, length of pi should be equal or less than the lenght of fi.
    min_len = min(len(pi), len(fi))
    matched = fnmatch.fnmatch(PATH_SEP.join(pi[0:min_len]), PATH_SEP.join(fi[0:min_len]))
    return matched and (self.opt.recursive or len(pi) <= len(fi))

  @log_calls
  @auto_reset
  def s3walk(self, s3url, s3dir, filter_path, result):

    '''
      Thread worker for s3walk.
      Recursively walk into all subdirectories if they still match the filter
      path partially.
    '''

    bucket = self.s3.lookup(s3url.bucket)
    for key in bucket.list(s3dir, PATH_SEP):
      if not self.partial_match(key.name, filter_path):
        continue

      # determine if it is a leaf node.
      is_dir = (key.name[-1] == PATH_SEP)
      if is_dir:
        is_leaf = (key.name.count(PATH_SEP) == filter_path.count(PATH_SEP) + 1)
      else:
        is_leaf = (key.name.count(PATH_SEP) == filter_path.count(PATH_SEP))

      if self.opt.recursive:
        is_leaf = not is_dir

      if is_leaf:
        result.append({
          'name': S3URL.combine(s3url.proto, s3url.bucket, key.name),
          'is_dir': is_dir,
          'size': key.size if not is_dir else 0,
          'last_modified': key.last_modified if not is_dir else None
        })
      elif is_dir and key.name != s3dir: # bug?
        self.pool.s3walk(s3url, key.name, filter_path, result)

  class MultipartItem:
    '''Utility class for multiple part upload/download.
       This class is used to keep track of a single upload/download file, so
       that we can initialize/finalize a file when needed.
    '''
    def __init__(self, id):
      '''Constructor.
         An unique identify for a single donwload/upload file.
           - Download: the temporary file name.
           - Upload: the id of multipart upload provided by S3.
      '''
      self.id = id
      self.processed = 0
      self.total = -1

    @synchronized
    def complete(self):
      '''Increase the processed counter, and see if the file is completely
         uploaded or downloaded.
      '''
      self.processed += 1
      return self.processed == self.total

  @log_calls
  def get_file_splits(self, id, source, target, fsize, splitsize):
    '''Get file splits for upload/download.'''
    pos = 0
    part = 1 # S3 part id starts from 1
    mpi = ThreadUtil.MultipartItem(id)
    splits = []

    while pos < fsize:
      chunk = min(splitsize, fsize - pos)
      assert(chunk > 0)
      splits.append((mpi, source, target, pos, chunk, part))
      part += 1
      pos += chunk
    mpi.total = len(splits)

    return splits

  @log_calls
  def get_file_privilege(self, source):
    '''Get privileges of a local file'''
    try:
      return str(oct(os.stat(source).st_mode)[-3:])
    except Exception, e:
      raise Failure('Could not get stat for %s, error_message = %s', source, e)

  @log_calls
  @auto_reset
  def upload(self, mpi, source, target, pos = 0, chunk = 0, part = 0):

    '''Thread worker for upload operation.'''
    s3url = S3URL(target)
    bucket = self.s3.lookup(s3url.bucket)

    # Initialization: Set up multithreaded uploads.
    if not mpi:
      fsize = os.path.getsize(source)
      key = bucket.get_key(s3url.path)

      # optional checks
      if self.opt.dry_run:
        message(INDENT + '%s => %s', source, target)
        return
      elif self.opt.sync_check and self.sync_check(source, key):
        message(INDENT + '%s => %s (synced)', source, target)
        return
      elif not self.opt.force and key:
        raise Failure('File already exists: %s' % target)

      # Small file optimization.
      if fsize < SINGLEPART_UPLOAD_MAX:
        key = boto.s3.key.Key(bucket)
        key.key = s3url.path
        key.set_metadata('privilege',  self.get_file_privilege(source))
        key.set_contents_from_filename(source)
        message(INDENT + '%s => %s', source, target)
        return

      # Here we need to have our own md5 value because multipart upload calculates
      # different md5 values.
      mpu = bucket.initiate_multipart_upload(s3url.path, metadata = {'md5': self.file_hash(source), 'privilege': self.get_file_privilege(source)})

      for args in self.get_file_splits(mpu.id, source, target, fsize, DEFAULT_SPLIT):
        self.pool.upload(*args)
      return

    # Handle each part in parallel, post initialization.
    for mp in bucket.list_multipart_uploads():
      if mp.id == mpi.id:
        mpu = mp
        break
    if mpu is None:
      raise Failure('Could not find MultiPartUpload %s' % mpu_id)

    data = None
    with open(source, 'rb') as f:
      f.seek(pos)
      data = f.read(chunk)
    if not data:
      raise Failure('Unable to read data from source: %s' % source)

    mpu.upload_part_from_file(cStringIO.StringIO(data), part)

    # Finalize
    if mpi.complete():
      try:
        mpu.complete_upload()
        message(INDENT + '%s => %s', source, target)
      except Exception, e:
        mpu.cancel_upload()
        raise RetryFailure('Upload failed: Unable to complete upload %s.' % source)

  @log_calls
  def _verify_file_size(self, key, downloaded_file):
    '''Verify the file size of the downloaded file.'''
    file_size = os.path.getsize(downloaded_file)
    if key.size != file_size:
      raise RetryFailure('Downloaded file size inconsistent: %s' % (repr(key)))

  @log_calls
  def _kick_off_downloads(self, s3url, bucket, source, target):
    '''Kick off download tasks, or directly download the file if the file is small.'''
    key = bucket.get_key(s3url.path)

    # optional checks
    if self.opt.dry_run:
      message(INDENT + '%s => %s', source, target)
      return
    elif self.opt.sync_check and self.sync_check(target, key):
      message(INDENT + '%s => %s (synced)', source, target)
      return
    elif not self.opt.force and os.path.exists(target):
      raise Failure('File already exists: %s' % target)

    if key is None:
      raise Failure('The key "%s" does not exists.' % (s3url.path,))

    resp = self.s3.make_request('HEAD', bucket = bucket, key = key)
    fsize = int(resp.getheader('content-length'))

    # Small file optimization.
    if fsize < SINGLEPART_DOWNLOAD_MAX:
      tempfile = tempfile_get(target)
      try: # Atomically download file with
        if self.opt.recursive:
          self.mkdirs(tempfile)
        key.get_contents_to_filename(tempfile)
        self.update_privilege(source, tempfile)
        self._verify_file_size(key, tempfile)
        tempfile_set(tempfile, target)
        message(INDENT + '%s => %s', source, target)
      except Exception, e:
        tempfile_set(tempfile, None)
        raise RetryFailure('Download Failure: %s, Source: %s.' % (e.message, source))

      return

    # Here we use temp filename as the id of mpi.
    for args in self.get_file_splits(tempfile_get(target), source, target, fsize, DEFAULT_SPLIT):
      self.pool.download(*args)
    return

  @log_calls
  @auto_reset
  def download(self, mpi, source, target, pos = 0, chunk = 0, part = 0):
    '''Thread worker for download operation.'''
    s3url = S3URL(source)
    bucket = self.s3.lookup(s3url.bucket)

    # Initialization
    if not mpi:
      self._kick_off_downloads(s3url, bucket, source, target)
      return

    tempfile = mpi.id
    if self.opt.recursive:
      self.mkdirs(tempfile)

    # Download part of the file, range is inclusive.
    resp = self.s3.make_request('GET', bucket = s3url.bucket, key = s3url.path, headers = {'Range': 'bytes=%d-%d' % (pos, pos + chunk - 1)})
    fd = os.open(tempfile, os.O_CREAT | os.O_WRONLY)
    try:
      os.lseek(fd, pos, os.SEEK_SET)

      while True:
        data = resp.read(chunk)
        if not data:
          break
        os.write(fd, data)
    finally:
      os.close(fd)

    # Finalize
    if mpi.complete():
      key = bucket.get_key(s3url.path)
      try:
        self.update_privilege(source, tempfile)
        self._verify_file_size(key, tempfile)
        tempfile_set(tempfile, target)
        message(INDENT + '%s => %s', source, target)
      except Exception, e:
        # Note that we don't retry in this case, because
        # We are going to remove the temp file, and if we
        # retry here with original parameters (wrapped in
        # the task item), it would fail anyway
        tempfile_set(tempfile, None)
        raise Failure('Download Failure: %s, Source: %s.' % (e.message, source))

  @log_calls
  @auto_reset
  def copy(self, source, target, delete_source = False):
    '''Copy a single file from source to target using boto S3 library.'''
    source_url = S3URL(source)
    target_url = S3URL(target)

    if not self.opt.dry_run:
      bucket = self.s3.lookup(source_url.bucket)
      key = bucket.get_key(source_url.path)
      key.copy(target_url.bucket, target_url.path)
      if delete_source:
        key.delete()
    message(INDENT + '%s => %s' % (source, target))

  @log_calls
  @auto_reset
  def delete(self, source):
    '''Thread worker for download operation.'''
    s3url = S3URL(source)
    bucket = self.s3.lookup(s3url.bucket)
    key = bucket.get_key(s3url.path)

    if not self.opt.dry_run:
      key.delete()
    message(INDENT + 'Delete %s', source)

class CommandHandler(object):

  '''Main class to handle commands.
     This class is responsible for parameter validation and call the corresponding
     operations.
  '''

  # def __init__(self, opt):
  def __init__(self):
    '''Constructor'''
    self.pathprefix = "/"
    self.buffer = {
      k: [] for k in BUFFER_KEYS
      }

  def run(self, opt, args):
    self.opt = opt

    '''Main entry to handle commands.
    Dispatch to individual command handler.'''

    if len(args) == 0:
      raise InvalidArgument(INDENT + 'No command provided')

    cmd = args[0]
    if CommandHandler.__dict__.has_key(cmd + '_handler'):

      try:
        CommandHandler.__dict__[cmd + '_handler'](self, args)
        return self

      except InvalidArgument, e: fail(INDENT + '[Invalid Argument] ', exc_info = e)
      except Failure, e: fail(INDENT + '[Runtime Failure] ', exc_info = e)
      except boto.exception.S3ResponseError, e: fail(INDENT + '[S3ResponseError] %s: %s' % (e.error_code, e.error_message))
      except Exception, e: fail(INDENT + '[Runtime Exception] ', exc_info = e, stacktrace = True)

    else:
      raise InvalidArgument(INDENT + 'Unknown command %s' % cmd)

  def s3handler(self):
    '''Create a S3Handler instances for multithread operations.'''
    return S3Handler(self.opt)

  @log_calls
  def validate(self, format, args):
    '''Validate input parameters with given format.
       This function also checks for wildcards for recursive mode.
    '''
    fmtMap = {
      'cmd': 'Command',
      's3': 's3 path',
      'local': 'local path'
    }
    fmts = format.split('|')
    if len(fmts) != len(args):
      raise InvalidArgument('Invalid number of parameters')

    for i, fmt in enumerate(fmts):
      valid = False
      for f in fmt.split(','):
        if f == 'cmd' and CommandHandler.__dict__.has_key(args[i] + '_handler'):
          valid = True
        if f == 's3' and S3URL.is_valid(args[i]):
          valid = True
        if f == 'local' and not S3URL.is_valid(args[i]):
          valid = True
      if not valid:
        raise InvalidArgument('Invalid parameter: %s, %s expected' % (args[i], fmtMap[fmt.split(',')[0]]))

  @log_calls
  def pretty_print(self, keylist):

    '''
      Pretty print the result of s3walk. Here we calculate the maximum width
      of each column and align them.
    '''

    def normalize_time(timestamp):
      '''Normalize the timestamp format for pretty print.'''
      if not timestamp:
        return ' ' * 16
      m = TIMESTAMP_REGEX.match(timestamp)
      result = m.groups()[0:5] if m else ['', '', '', '', '']
      return TIMESTAMP_FORMAT % result

    cwidth = [0, 0, 0]
    format = '%%%ds %%%ds %%-%ds'

    # Calculate maximum width for each column.
    result = []
    for key in keylist:
      last_modified = normalize_time(key['last_modified'])
      size = str(key['size']) if not key['is_dir'] else 'DIR'
      name = key['name']
      item = (last_modified, size, name)
      for i, value in enumerate(item):
        if cwidth[i] < len(value):
          cwidth[i] = len(value)
      result.append(item)

    # Format output.
    for item in result:
      text = (format % tuple(cwidth)) % item
      message(INDENT + '%s', text.rstrip())

  @log_calls
  def help_handler(self, args):
    return

  @log_calls
  def ls_handler(self, args):
    '''Handler for ls command'''

    if len(args) == 1:
      args.append(self.pathprefix)
      if self.pathprefix == "/" :
        s3handler = self.s3handler()
        bucket_list = s3handler.list_buckets()
        self.pretty_print(bucket_list["buckets"])
        self.buffer = s3handler.buffer
        return

    self.validate('cmd|s3', args)
    s3handler = self.s3handler()
    s3walk = s3handler.s3walk(args[1])
    self.buffer = s3handler.buffer
    self.pretty_print(s3walk)

  @log_calls
  def put_handler(self, args):
    '''Handler for put command'''

    # Special check for shell expansion
    if len(args) < 3:
      raise InvalidArgument('Invalid number of parameters')

    args[-1] = self.pathprefix + args[1:-1][0] if args[-1] == "." else args[-1]
    self.validate('|'.join(['cmd'] + ['local'] * (len(args) - 2) + ['s3']), args)

    source = args[1:-1]; target = args[-1]
    self.s3handler().put_files(source, target)

  @log_calls
  def get_handler(self, args):
    '''Handler for get command'''

    # Special case when we don't have target directory.
    if len(args) == 2:
      args += ['.']

    self.validate('cmd|s3|local', args)
    source = args[1]
    target = args[2]
    self.s3handler().get_files(source, target)

  @log_calls
  def sync_handler(self, args):
    '''Handler for sync command.
       XXX Here we emulate sync command with get/put -r -f --sync-check. So
           it doesn't provide delete operation.
    '''
    self.opt.recursive = True
    self.opt.sync_check = True
    self.opt.force = True

    self.validate('cmd|s3,local|s3,local', args)
    source = args[1]; target = args[2]
    self.s3handler().sync_files(source, target)

  @log_calls
  def cd_handler(self, args):
    '''Handler for cd command'''
    assert args[-1].startswith("s3://") and args[-1].endswith("/") or args[-1] == "/"
    self.pathprefix = args[-1]
    return

  @log_calls
  def pwd_handler(self, args):
    '''Handler for pwd command'''
    print INDENT + self.pathprefix
    return

  @log_calls
  def cp_handler(self, args):
    '''Handler for cp command'''

    self.validate('cmd|s3|s3', args)
    source = args[1]
    target = args[2]
    self.s3handler().cp_files(source, target)

  @log_calls
  def mv_handler(self, args):
    '''Handler for mv command'''

    self.validate('cmd|s3|s3', args)
    source = args[1]
    target = args[2]
    self.s3handler().cp_files(source, target, True)

  @log_calls
  def del_handler(self, args):
    '''Handler for del command'''
    self.validate('cmd|s3', args)
    source = args[1]
    self.s3handler().del_files(source)

  @log_calls
  def du_handler(self, args):
    '''Handler for size command'''
    for src, size in self.s3handler().size(args[1:]):
      message('%s\t%s' % (size, src))

  @log_calls
  def _totalsize_handler(self, args):
    '''Handler of total_size command'''
    total_size = 0
    for src, size in self.s3handler().size(args[1:]):
      total_size += size
    message(str(total_size))


text:
import { path } from 'less( ./assets/styles.less )';
console.log( 'styles path', path );


text:
CC = mpic++

svenn: chess.c boardutil.c moves.c strutil.c
	$(CC) -o chess chess.c boardutil.c moves.c strutil.c

clean:
	rm chess


text:
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Threading.Tasks;

namespace Nager.PublicSuffix
{
    public class WebTldRuleProvider : ITldRuleProvider
    {
        private readonly string _fileUrl;
        private readonly ICacheProvider _cacheProvider;

        public ICacheProvider CacheProvider { get { return this._cacheProvider; } }

        public WebTldRuleProvider(string url = "https://publicsuffix.org/list/public_suffix_list.dat", ICacheProvider cacheProvider = null)
        {
            this._fileUrl = url;

            if (cacheProvider == null)
            {
                this._cacheProvider = new FileCacheProvider();
            }
            else
            {
                this._cacheProvider = cacheProvider;
            }
        }

        public async Task<IEnumerable<TldRule>> BuildAsync()
        {
            var ruleParser = new TldRuleParser();

            var ruleData = await this._cacheProvider.GetValueAsync().ConfigureAwait(false);
            if (string.IsNullOrEmpty(ruleData))
            {
                ruleData = await this.LoadFromUrl(this._fileUrl).ConfigureAwait(false);
                await this._cacheProvider.SetValueAsync(ruleData).ConfigureAwait(false);
            }

            var rules = ruleParser.ParseRules(ruleData);
            return rules;
        }

        public async Task<string> LoadFromUrl(string url)
        {
            try
            {
                using (var httpClient = new HttpClient())
                {
                    using (var response = await httpClient.GetAsync(url).ConfigureAwait(false))
                    {
                        return await response.Content.ReadAsStringAsync().ConfigureAwait(false);
                    }
                }
            }
            catch (Exception exception)
            {
                return await Task.FromResult("error");
            }
         }
    }
}


text:
'use strict';

var Levenshtein = require('levenshtein');
var normalize = require('libamd/modules/normalize');

var findProjectFiles = require('./findProjectFiles');

var suggest = function(rjsconfig, name) {
	var iexact;
	var candidates = [];
	findProjectFiles(rjsconfig).every(function(file) {
		var normalized = normalize(rjsconfig, file);

		if (file.toLowerCase() === normalized.toLowerCase()) {
			iexact = normalized;
			return false;
		}

		var l = new Levenshtein(name.toLowerCase(), normalized.toLowerCase()).distance;
		candidates.push({
			mod: normalized,
			distance: l
		});
		return true;
	});

	if (iexact) {
		return [iexact];
	}

	return candidates
		.filter(function(file) {
			return file.distance <= 5;
		})
		.sort(function(a, b) {
			return a.distance - b.distance;
		})
		.map(function(file) {
			return file.mod;
		});

};

module.exports = suggest;


text:
'use strict';

// @ngInject

var router = function ($routeProvider, $locationProvider) {

  $locationProvider.html5Mode(true).hashPrefix('!');

  $routeProvider.when('/:id', {
    templateUrl: 'show/show.html',
    controller: 'InventoryShowController'
  }).when('/:id/edit', {
    template: '<npdc:formula></npdc:formula>',
    controller: 'InventoryEditController'
  }).when('/', {
    template: '<npdc:search-input feed="feed"></npdc:search-input><npdc:search feed="feed"></npdc:search>',
    controller: 'InventorySearchController',
    reloadOnSearch: false
  });
};

module.exports = router;


text:
/*!
 * @module bigtable
 */

'use strict';

var googleProtoFiles = require('google-proto-files');
var is = require('is');
var nodeutil = require('util');
var format = require('string-format-obj');
var extend = require('extend');

/**
 * @type {module:bigtable/table}
 * @private
 */
var Table = require('./table.js');

/**
 * @type {module:common/grpcService}
 * @private
 */
var GrpcService = require('../common/grpc-service.js');

/**
 * @type {module:common/util}
 * @private
 */
var util = require('../common/util.js');

/**
 * Interact with
 * [Google Cloud Bigtable](https://cloud.google.com/bigtable/).
 *
 * @constructor
 * @alias module:bigtable
 *
 * @classdesc
 * The `gcloud.bigtable` object allows you interact with Google Cloud Bigtable.
 *
 * To learn more about Bigtable, read the
 * [Google Cloud Bigtable Concepts Overview](https://cloud.google.com/bigtable/docs/concepts)
 *
 * @resource [Creating a Cloud Bigtable Cluster]{@link https://cloud.google.com/bigtable/docs/creating-compute-instance}
 *
 * @param {object=} options - [Configuration object](#/docs).
 * @param {string} options.cluster - The cluster name that hosts your tables.
 * @param {string|module:compute/zone} options.zone - The zone in which your
 *     cluster resides.
 *
 * @example
 * var gcloud = require('gcloud')({
 *   keyFilename: '/path/to/keyfile.json',
 *   projectId: 'my-project'
 * });
 *
 * var bigtable = gcloud.bigtable({
 *   zone: 'us-central1-b',
 *   cluster: 'gcloud-node'
 * });
 *
 * //-
 * //
Creating a Cluster

 * //
 * // Before you create your table, you first need to create a Bigtable Cluster
 * // for the table to be served from. This can be done from either the
 * // Google Cloud Platform Console or the `gcloud` cli tool. Please refer to
 * // the
 * // official Bigtable documentation for more information.
 * //-
 *
 * //-
 * //
Creating Tables

 * //
 * // After creating your cluster and enabling the Bigtable APIs, you are now
 * // ready to create your table with {module:bigtable#createTable}.
 * //-
 * bigtable.createTable('prezzy', function(err, table) {
 *   // `table` is your newly created Table object.
 * });
 *
 * //-
 * //
Creating Column Families

 * //
 * // Column families are used to group together various pieces of data within
 * // your table. You can think of column families as a mechanism to categorize
 * // all of your data.
 * //
 * // We can create a column family with {module:bigtable/table#createFamily}.
 * //-
 * var table = bigtable.table('prezzy');
 *
 * table.createFamily('follows', function(err, family) {
 *   // `family` is your newly created Family object.
 * });
 *
 * //-
 * //
Creating Rows

 * //
 * // New rows can be created within your table using
 * // {module:bigtable/table#insert}. You must provide a unique key for each row
 * // to be inserted, this key can then be used to retrieve your row at a later
 * // time.
 * //
 * // With Bigtable, all columns have a unique id composed of a column family
 * // and a column qualifier. In the example below `follows` is the column
 * // family and `tjefferson` is the column qualifier. Together they could be
 * // referred to as `follows:tjefferson`.
 * //-
 * var rows = [
 *   {
 *     key: 'wmckinley',
 *     data: {
 *       follows: {
 *         tjefferson: 1
 *       }
 *     }
 *   }
 * ];
 *
 * table.insert(rows, function(err) {
 *   if (!err) {
 *     // Your rows were successfully inserted.
 *   }
 * });
 *
 * //-
 * //
Retrieving Rows

 * //
 * // If you're anticipating a large number of rows to be returned, we suggest
 * // using the {module:bigtable/table#getRows} streaming API.
 * //-
 * table.getRows()
 *   .on('error', console.error)
 *   .on('data', function(row) {
 *     // `row` is a Row object.
 *   });
 *
 * //-
 * // If you're not anticpating a large number of results, a callback mode
 * // is also available.
 * //-
 * var callback = function(err, rows) {
 *   // `rows` is an array of Row objects.
 * };
 *
 * table.getRows(callback);
 *
 * //-
 * // A range of rows can be retrieved by providing `start` and `end` row keys.
 * //-
 * var options = {
 *   start: 'gwashington',
 *   end: 'wmckinley'
 * };
 *
 * table.getRows(options, callback);
 *
 * //-
 * // Retrieve an individual row with {module:bigtable/row#get}.
 * //-
 * var row = table.row('alincoln');
 *
 * row.get(function(err) {
 *   // `row.data` is now populated.
 * });
 *
 * //-
 * //
Accessing Row Data

 * //
 * // When retrieving rows, upon success the `row.data` property will be
 * // populated by an object. That object will contain additional objects
 * // for each family in your table that the row has data for.
 * //
 * // By default, when retrieving rows, each column qualifier will provide you
 * // with all previous versions of the data. So your `row.data` object could
 * // resemble the following.
 * //-
 * console.log(row.data);
 * // {
 * //   follows: {
 * //     wmckinley: [
 * //       {
 * //         value: 1,
 * //         timestamp: 1466017315951
 * //       }, {
 * //         value: 2,
 * //         timestamp: 1458619200000
 * //       }
 * //     ]
 * //   }
 * // }
 *
 * //-
 * // The `timestamp` field can be used to order cells from newest to oldest.
 * // If you only wish to retrieve the most recent version of the data, you
 * // can specify the number of cells with a {module:bigtable/filter} object.
 * //-
 * var filter = [
 *   {
 *     column: {
 *       cellLimit: 1
 *     }
 *   }
 * ];
 *
 * table.getRows({
 *   filter: filter
 * }, callback);
 *
 * //-
 * //
Deleting Row Data

 * //
 * // We can delete all of an individual row's cells using
 * // {module:bigtable/row#delete}.
 * //-
 * var callback = function(err) {
 *   if (!err) {
 *     // All cells for this row were deleted successfully.
 *   }
 * };
 *
 * row.delete(callback);
 *
 * //-
 * // To delete a specific set of cells, we can provide an array of
 * // column families and qualifiers.
 * //-
 * var cells = [
 *   'follows:gwashington',
 *   'traits'
 * ];
 *
 * row.delete(cells, callback);
 *
 * //-
 * //
Deleting Rows

 * //
 * // If you wish to delete multiple rows entirely, we can do so with
 * // {module:bigtable/table#deleteRows}. You can provide this method with a
 * // row key prefix.
 * //-
 * var options = {
 *   prefix: 'gwash'
 * };
 *
 * table.deleteRows(options, function(err) {
 *   if (!err) {
 *     // Rows were deleted successfully.
 *   }
 * });
 *
 * //-
 * // If you omit the prefix, you can delete all rows in your table.
 * //-
 * table.deleteRows(function(err) {
 *   if (!err) {
 *     // All rows were deleted successfully.
 *   }
 * });
 */
function Bigtable(options) {
  if (!(this instanceof Bigtable)) {
    options = util.normalizeArguments(this, options);
    return new Bigtable(options);
  }

  options = extend({}, options, {
    zone: options.zone.name || options.zone
  });

  this.clusterName = format(
    'projects/{projectId}/zones/{zone}/clusters/{cluster}',
    options
  );

  var config = {
    baseUrl: 'bigtable.googleapis.com',
    service: 'bigtable',
    apiVersion: 'v1',
    protoServices: {
      BigtableService: googleProtoFiles.bigtable.v1,
      BigtableTableService: {
        path: googleProtoFiles.bigtable.admin,
        service: 'bigtable.admin.table'
      }
    },
    scopes: [
      'https://www.googleapis.com/auth/bigtable.admin',
      'https://www.googleapis.com/auth/bigtable.data'
    ]
  };

  GrpcService.call(this, config, options);
}

nodeutil.inherits(Bigtable, GrpcService);

/**
 * Formats the full table name into a user friendly version.
 *
 * @private
 *
 * @param {string} name - The formatted Table name.
 * @return {string}
 *
 * @example
 * Bigtable.formatTableName_('projects/p/zones/z/clusters/c/tables/my-table');
 * // => 'my-table'
 */
Bigtable.formatTableName_ = function(name) {
  if (name.indexOf('/') === -1) {
    return name;
  }

  var parts = name.split('/');
  return parts[parts.length - 1];
};

/**
 * Create a table on your Bigtable cluster.
 *
 * @resource [Designing Your Schema]{@link https://cloud.google.com/bigtable/docs/schema-design}
 * @resource [Splitting Keys]{@link https://cloud.google.com/bigtable/docs/managing-tables#splits}
 *
 * @param {string} name - The name of the table.
 * @param {object=} options - Table creation options.
 * @param {string} options.operation - Operation used for table that has already
 *    been queued to be created.
 * @param {string[]} options.splits - Initial
 *    [split keys](https://cloud.google.com/bigtable/docs/managing-tables#splits).
 * @param {function} callback - The callback function.
 * @param {?error} callback.err - An error returned while making this request.
 * @param {module:bigtable/table} callback.table - The newly created table.
 * @param {object} callback.apiResponse - The full API response.
 *
 * @example
 * var callback = function(err, table, apiResponse) {
 *   // `table` is a Table object.
 * };
 *
 * bigtable.createTable('prezzy', callback);
 *
 * //-
 * // Pre-split the table based on the row key to spread the load across
 * // multiple Cloud Bigtable nodes.
 * //-
 * var options = {
 *   splits: ['10', '20']
 * };
 *
 * bigtable.createTable('prezzy', options, callback);
 */
Bigtable.prototype.createTable = function(name, options, callback) {
  var self = this;

  options = options || {};

  if (is.function(options)) {
    callback = options;
    options = {};
  }

  var protoOpts = {
    service: 'BigtableTableService',
    method: 'createTable'
  };

  var reqOpts = {
    name: this.clusterName,
    tableId: name,
    table: {
      // The granularity at which timestamps are stored in the table.
      // Currently only milliseconds is supported, so it's not configurable.
      granularity: 0
    }
  };

  if (options.operation) {
    reqOpts.table.currentOperation = options.operation;
  }

  if (options.splits) {
    reqOpts.initialSplitKeys = options.splits;
  }

  this.request(protoOpts, reqOpts, function(err, resp) {
    if (err) {
      callback(err, null, resp);
      return;
    }

    var table = self.table(resp.name);
    table.metadata = resp;

    callback(null, table, resp);
  });
};

/**
 * Get Table objects for all the tables in your Bigtable cluster.
 *
 * @param {function} callback - The callback function.
 * @param {?error} callback.err - An error returned while making this request.
 * @param {module:bigtable/table[]} callback.tables - List of all Tables.
 * @param {object} callback.apiResponse - The full API response.
 *
 * @example
 * bigtable.getTables(function(err, tables) {
 *   if (!err) {
 *     // `tables` is an array of Table objects.
 *   }
 * });
 */
Bigtable.prototype.getTables = function(callback) {
  var self = this;

  var protoOpts = {
    service: 'BigtableTableService',
    method: 'listTables'
  };

  var reqOpts = {
    name: this.clusterName
  };

  this.request(protoOpts, reqOpts, function(err, resp) {
    if (err) {
      callback(err, null, resp);
      return;
    }

    var tables = resp.tables.map(function(metadata) {
      var name = Bigtable.formatTableName_(metadata.name);
      var table = self.table(name);

      table.metadata = metadata;
      return table;
    });

    callback(null, tables, resp);
  });
};

/**
 * Get a reference to a Bigtable table.
 *
 * @param {string} name - The name of the table.
 * @return {module:bigtable/table}
 *
 * @example
 * var table = bigtable.table('presidents');
 */
Bigtable.prototype.table = function(name) {
  return new Table(this, name);
};

module.exports = Bigtable;


text:
import { Component, OnInit, EventEmitter, Output, Input, HostListener, OnDestroy } from '@angular/core';
import { FormBuilder, Validators, FormGroup } from '@angular/forms';
import { Subscription } from 'rxjs';

import { upperCase } from 'lodash';

import { ITroop, IClub, IUser } from 'app/model';
import { UserService } from 'app/shared/services/api';

import { ClubEditorComponent } from 'app/views/configure/club/club-editor/club-editor.component';
import { ActivatedRoute, Router } from '@angular/router';
import { GraphService } from 'app/shared/services/graph.service';
import { CommonService } from 'app/shared/services/common.service';

@Component({
  selector: 'app-troop-editor',
  templateUrl: './troop-editor.component.html',
  styleUrls: ['./troop-editor.component.scss']
})
export class TroopEditorComponent implements OnInit, OnDestroy {
  @Input() troop: ITroop = <ITroop>{};
  @Output() troopChanged: EventEmitter<any> = new EventEmitter<any>();

  troopForm: FormGroup;

  _currentUser: IUser;
  get currentUser() { return this._currentUser; }
  set currentUser(value) {
    this._currentUser = value;
  }
  subscriptions: Subscription[] = [];

  memberListHidden = true;

  club: IClub;
  troopsCount = 0;

  get clubName() {
    return upperCase(this.troop.club ? this.troop.club.name : this.clubComponent.clubName);
  }

  get troopSuggestion() {
    setTimeout(() => this.troopForm.markAsDirty());
    let teamCounter = this.troopsCount;
    return this.clubName.split(' ')[0].toLowerCase() + '-' + ++teamCounter;
  }

  constructor(
    private fb: FormBuilder,
    private router: Router,
    private route: ActivatedRoute,
    private graph: GraphService,
    private userService: UserService,
    private common: CommonService,
    private clubComponent: ClubEditorComponent) { }

  ngOnInit() {
    this.subscriptions.push(this.userService.getMe().subscribe(user => this.currentUser = user));

    this.troopForm = this.fb.group({
      id: [null],
      name: [null, [Validators.required]],
      club: [null],
      gymnasts: [[]]
    });

    this.clubComponent.clubSubject.subscribe(club => {
      this.club = club;
      this.route.params.subscribe(params => {
        if (params.id) {
          this.graph.getData(`{troop(id:${+params.id}){id,name,gymnasts{id,name,gender,birthYear}}}`).subscribe(res => this.troopReceived(res.troop));
        } else {
          this.troopForm.get('club').setValue(this.club);
          this.graph.getData(`{club(id:${this.club.id}){id,troopCount}}`).subscribe(result => {
            this.troopsCount = result.club.troopCount;
            this.troopForm.get('name').setValue(this.troopSuggestion);
          });
        }
      });
    });
  }

  ngOnDestroy() {
    this.subscriptions.forEach(s => s ? s.unsubscribe() : null);
  }

  troopReceived(troop: ITroop) {
    this.troop = troop;
    this.troopForm.setValue({
      id: this.troop.id || null,
      name: this.troop.name || '',
      club: this.club,
      gymnasts: this.troop.gymnasts || []
    });
  }

  async save(keepOpen?: boolean) {
    const troop = <ITroop>this.troopForm.value;

    // Save team
    return new Promise((resolve, reject) => {
      troop.clubId = troop.club.id;
      this.graph.saveData('Troop', CommonService.omit(troop, ['club']), `{id,name,gymnasts{id,name,gender,birthYear}}`).subscribe(result => {
        this.troopReceived(result.saveTroop);
        this.close(result.saveTroop);
        resolve(result.saveTroop);
      });
    });
  }

  delete() {
    this.common.confirm().subscribe(shouldRemove => {
      if (shouldRemove) {
        this.graph.deleteData('Troop', this.troopForm.value.id).subscribe(result => {
          this.close(result);
        });
      }
    });
  }

  close(result?) {
    this.troopChanged.emit(result || this.troop);
    this.router.navigate(['../'], { relativeTo: this.route });
  }

  @HostListener('keyup', ['$event'])
  onKeyup(evt: KeyboardEvent) {
    if (evt.key === 'Escape') {
      this.close();
    }
  }
}


text:
/**
 * Se encarga de mostrar en un grafico de barras, la suma de los
 * pagos de cada uno de los meses.
 *
 */
var PagosPorMesController = Class.$extend({

    /**
     * Constructor.
     *
     * @param {jQuery} element: el elemento del DOM en donde es que se tiene que
     *                          graficar toda la informacion.
     *
     * @param {APIManager} apiManager: el manager que se va a usar para obtener
     *                                 toda la informacion a graficar.
     */
    __init__: function(element, apiManager, searchController) {
        this.$element = element;
        this.apiManager = apiManager;
        this.searchController = searchController;
    },

    render: function() {
        this.$element.mask('Loading');

        var filtersData = utils.translateFiltersToAPI(this.searchController.filters);

        var ajaxData = {
                orderBy: 'mes_correspondiente',
                groupBy: 'mes_correspondiente',
                orderDirection: 'ASC',
                columns: ['mes_correspondiente', 'SUM(importe)', 'COUNT(id)'],
                filters: filtersData
        };

        this.apiManager.ajaxCall('pago/',
                                 ajaxData,
                                 'GET',
                                 $.proxy(this.renderInformation, this));
    },

    /**
     * Se encarga de renderar toda la informacion del grafico teniendo en cuenta
     * la respuesta del servidor.
     *
     * En este caso, la informacion del mismo viene en el siguiente formato:
     *
     * {
     *      values: [
     *          ['01/01/2013', 123, 1],
     *          ['01/02/2013', 466, 2],
     *          ...
     *      ]
     * }
     *
     * donde:
     *
     *  - el primer valor corresponde al mes_correspondiente
     *  - el segundo valor corresponde a la suma de los importes que se hizo
     *    ese mes
     *  - el tercer valor corresponde a la cantidad de pagos hechos eses
     *    mes
     *
     */
    renderInformation: function(data) {
        this.$element.unmask();

        // por mas de que tengo el importe dia a dia, tengo que calcular el
        // acumulado correspondiente.
        var pagosMes = [];
        for (var i = 0; i < data.values.length; i++) {
            pointData = data.values[i];
            var date = moment(pointData[0], 'DD/MM/YYYY');
            var utcDate = Date.UTC(date.year(), date.month(), date.date());
            pagosMes.push([utcDate, pointData[1]]);
        }

        this.$element.find('.chart-container').highcharts({
            xAxis: {
                type: 'datetime'
            },
            series: [{
                name: 'Pagos mes a mes',
                data: pagosMes
            }]
        });

    }

});


text:
'use strict';

module.exports = {
  setup: setupAsync,
  compile: compileAsync
};

var util = require('./compile/util');

var ASYNC = {
  '*': checkGenerators,
  'co*': checkGenerators,
  'es7': checkAsyncFunction
};

var TRANSPILE = {
  'nodent': getNodent,
  'regenerator': getRegenerator
};

var MODES = [
  { async: 'co*' },
  { async: 'es7', transpile: 'nodent' },
  { async: 'co*', transpile: 'regenerator' }
];

var regenerator, nodent;

function setupAsync(opts, required) {
  if (required !== false) required = true;
  var async = opts.async
    , transpile = opts.transpile
    , check;

  switch (typeof transpile) {
    case 'string':
      var get = TRANSPILE[transpile];
      if (!get) throw new Error('bad transpiler: ' + transpile);
      return (opts._transpileFunc = get(opts, required));
    case 'undefined':
    case 'boolean':
      if (typeof async == 'string') {
        check = ASYNC[async];
        if (!check) throw new Error('bad async mode: ' + async);
        return (opts.transpile = check(opts, required));
      }

      for (var i=0; i<MODES.length; i++) {
        var _opts = MODES[i];
        if (setupAsync(_opts, false)) {
          util.copy(_opts, opts);
          return opts.transpile;
        }
      }
      /* istanbul ignore next */
      throw new Error('generators, nodent and regenerator are not available');
    case 'function':
      return (opts._transpileFunc = opts.transpile);
    default:
      throw new Error('bad transpiler: ' + transpile);
  }
}

function checkGenerators(opts, required) {
  /* jshint evil: true */
  try {
    eval('(function*(){})()');
    return true;
  } catch(e) {
    /* istanbul ignore next */
    if (required) throw new Error('generators not supported');
  }
}

function checkAsyncFunction(opts, required) {
  /* jshint evil: true */
  try {
    eval('(async function(){})()');
    /* istanbul ignore next */
    return true;
  } catch(e) {
    if (required) throw new Error('es7 async functions not supported');
  }
}

function getRegenerator(opts, required) {
  try {
    if (!regenerator) {
      regenerator = require('' + 'regenerator');
      regenerator.runtime();
    }
    if (!opts.async || opts.async === true)
      opts.async = 'es7';
    return regeneratorTranspile;
  } catch(e) {
    /* istanbul ignore next */
    if (required) throw new Error('regenerator not available');
  }
}

function regeneratorTranspile(code) {
  return regenerator.compile(code).code;
}

function getNodent(opts, required) {
  /* jshint evil: true */
  try {
    if (!nodent) nodent = require('' + 'nodent')({ log: false, dontInstallRequireHook: true });
    if (opts.async != 'es7') {
      if (opts.async && opts.async !== true) console.warn('nodent transpiles only es7 async functions');
      opts.async = 'es7';
    }
    return nodentTranspile;
  } catch(e) {
    /* istanbul ignore next */
    if (required) throw new Error('nodent not available');
  }
}

function nodentTranspile(code) {
  return nodent.compile(code, '', { promises: true, sourcemap: false }).code;
}

/**
 * Creates validating function for passed schema with asynchronous loading of missing schemas.
 * `loadSchema` option should be a function that accepts schema uri and node-style callback.
 * @this  Ajv
 * @param {Object}   schema schema object
 * @param {Function} callback node-style callback, it is always called with 2 parameters: error (or null) and validating function.
 */
function compileAsync(schema, callback) {
  /* eslint no-shadow: 0 */
  /* jshint validthis: true */
  var schemaObj;
  var self = this;
  try {
    schemaObj = this._addSchema(schema);
  } catch(e) {
    setTimeout(function() { callback(e); });
    return;
  }
  if (schemaObj.validate) {
    setTimeout(function() { callback(null, schemaObj.validate); });
  } else {
    if (typeof this._opts.loadSchema != 'function')
      throw new Error('options.loadSchema should be a function');
    _compileAsync(schema, callback, true);
  }

  function _compileAsync(schema, callback, firstCall) {
    var validate;
    try { validate = self.compile(schema); }
    catch(e) {
      if (e.missingSchema) loadMissingSchema(e);
      else deferCallback(e);
      return;
    }
    deferCallback(null, validate);

    function loadMissingSchema(e) {
      var ref = e.missingSchema;
      if (self._refs[ref] || self._schemas[ref])
        return callback(new Error('Schema ' + ref + ' is loaded but' + e.missingRef + 'cannot be resolved'));
      var _callbacks = self._loadingSchemas[ref];
      if (_callbacks) {
        if (typeof _callbacks == 'function')
          self._loadingSchemas[ref] = [_callbacks, schemaLoaded];
        else
          _callbacks[_callbacks.length] = schemaLoaded;
      } else {
        self._loadingSchemas[ref] = schemaLoaded;
        self._opts.loadSchema(ref, function (err, sch) {
          var _callbacks = self._loadingSchemas[ref];
          delete self._loadingSchemas[ref];
          if (typeof _callbacks == 'function') {
            _callbacks(err, sch);
          } else {
            for (var i=0; i<_callbacks.length; i++)
              _callbacks[i](err, sch);
          }
        });
      }

      function schemaLoaded(err, sch) {
        if (err) return callback(err);
        if (!(self._refs[ref] || self._schemas[ref])) {
          try {
            self.addSchema(sch, ref);
          } catch(e) {
            callback(e);
            return;
          }
        }
        _compileAsync(schema, callback);
      }
    }

    function deferCallback(err, validate) {
      if (firstCall) setTimeout(function() { callback(err, validate); });
      else return callback(err, validate);
    }
  }
}


text:
ACCEPTED

#### According to
Index Fungorum

#### Published in
Can. J. Bot. 57(19): 2088 (1979)

#### Original name
Phyllosticta cumminsii Bissett

### Remarks
null


text:
class BoxFilterFilm
  include Film

  attr_accessor :image,
                :width,
                :height,
                :unnormalized,
                :n_samples

  def initialize(width, height)
    @width        = width
    @height       = height
    @image        = []
    @unnormalized = []
    @n_samples    = []

    initialize_containers
  end

  # @param x [Float] projected horizontal hit position
  # @param y [Float] projected vertical hit position
  # @param spectrum [Spectrum] contribution added to hit position.
  def add_sample(x, y, spectrum)
    x_idx = x.to_i
    y_idx = y.to_i

    return unless coordinates_inside_box?(x_idx, y_idx)

    @unnormalized[x_idx][y_idx].add(spectrum)
    @n_samples[x_idx][y_idx] += 1.0
    @image[x_idx][y_idx].r = @unnormalized[x_idx][y_idx].r / @n_samples[x_idx][y_idx].to_f
    @image[x_idx][y_idx].g = @unnormalized[x_idx][y_idx].g / @n_samples[x_idx][y_idx].to_f
    @image[x_idx][y_idx].b = @unnormalized[x_idx][y_idx].b / @n_samples[x_idx][y_idx].to_f
  end

  # Checks whether a given index pair is inside the film's image.
  #
  # @param x_idx [Integer] width index
  # @param y_idx [Integer] height index
  def coordinates_inside_box?(x_idx, y_idx)
    (x_idx >= 0 && x_idx < width) && (y_idx >= 0 && y_idx < height)
  end

  private

  def initialize_containers
    width.times do
      image_row = []
      unnorm_row = []
      sample_row = []

      height.times do
        image_row << Spectrum.new(0.0)
        unnorm_row << Spectrum.new(0.0)
        sample_row << 0.0
      end

      @image << image_row
      @unnormalized << unnorm_row
      @n_samples << sample_row
    end
  end
end