text:
package query

import (
	"bytes"
	"fmt"
	"net/url"
	"reflect"
	"strconv"
	"strings"
	"time"
)

var timeType = reflect.TypeOf(time.Time{})

var encoderType = reflect.TypeOf(new(Encoder)).Elem()

// Encoder is an interface implemented by any type that wishes to encode
// itself into URL values in a non-standard way.
type Encoder interface {
	EncodeValues(key string, v *url.Values) error
}

// Values returns the url.Values encoding of v.
//
// Values expects to be passed a struct, and traverses it recursively using the
// following encoding rules.
//
// Each exported struct field is encoded as a URL parameter unless
//
//	- the field's tag is "-", or
//	- the field is empty and its tag specifies the "omitempty" option
//
// The empty values are false, 0, any nil pointer or interface value, any array
// slice, map, or string of length zero, and any time.Time that returns true
// for IsZero().
//
// The URL parameter name defaults to the struct field name but can be
// specified in the struct field's tag value.  The "url" key in the struct
// field's tag value is the key name, followed by an optional comma and
// options.  For example:
//
// 	// Field is ignored by this package.
// 	Field int `url:"-"`
//
// 	// Field appears as URL parameter "myName".
// 	Field int `url:"myName"`
//
// 	// Field appears as URL parameter "myName" and the field is omitted if
// 	// its value is empty
// 	Field int `url:"myName,omitempty"`
//
// 	// Field appears as URL parameter "Field" (the default), but the field
// 	// is skipped if empty.  Note the leading comma.
// 	Field int `url:",omitempty"`
//
// For encoding individual field values, the following type-dependent rules
// apply:
//
// Boolean values default to encoding as the strings "true" or "false".
// Including the "int" option signals that the field should be encoded as the
// strings "1" or "0".
//
// time.Time values default to encoding as RFC3339 timestamps.  Including the
// "unix" option signals that the field should be encoded as a Unix time (see
// time.Unix())
//
// Slice and Array values default to encoding as multiple URL values of the
// same name.  Including the "comma" option signals that the field should be
// encoded as a single comma-delimited value.  Including the "space" option
// similarly encodes the value as a single space-delimited string. Including
// the "semicolon" option will encode the value as a semicolon-delimited string.
// Including the "brackets" option signals that the multiple URL values should
// have "[]" appended to the value name. "numbered" will append a number to
// the end of each incidence of the value name, example:
// name0=value0&name1=value1, etc.
//
// Anonymous struct fields are usually encoded as if their inner exported
// fields were fields in the outer struct, subject to the standard Go
// visibility rules.  An anonymous struct field with a name given in its URL
// tag is treated as having that name, rather than being anonymous.
//
// Non-nil pointer values are encoded as the value pointed to.
//
// Nested structs are encoded including parent fields in value names for
// scoping. e.g:
//
// 	"user[name]=acme&user[addr][postcode]=1234&user[addr][city]=SFO"
//
// All other values are encoded using their default string representation.
//
// Multiple fields that encode to the same URL parameter name will be included
// as multiple URL values of the same name.
func Values(v interface{}) (url.Values, error) {
	values := make(url.Values)
	val := reflect.ValueOf(v)
	for val.Kind() == reflect.Ptr {
		if val.IsNil() {
			return values, nil
		}
		val = val.Elem()
	}

	if v == nil {
		return values, nil
	}

	if val.Kind() != reflect.Struct {
		return nil, fmt.Errorf("query: Values() expects struct input. Got %v", val.Kind())
	}

	err := reflectValue(values, val, "")
	return values, err
}

// reflectValue populates the values parameter from the struct fields in val.
// Embedded structs are followed recursively (using the rules defined in the
// Values function documentation) breadth-first.
func reflectValue(values url.Values, val reflect.Value, scope string) error {
	var embedded []reflect.Value

	typ := val.Type()
	for i := 0; i < typ.NumField(); i++ {
		sf := typ.Field(i)
		if sf.PkgPath != "" && !sf.Anonymous { // unexported
			continue
		}

		sv := val.Field(i)
		tag := sf.Tag.Get("url")
		if tag == "-" {
			continue
		}
		name, opts := parseTag(tag)
		if name == "" {
			if sf.Anonymous && sv.Kind() == reflect.Struct {
				// save embedded struct for later processing
				embedded = append(embedded, sv)
				continue
			}

			name = sf.Name
		}

		if scope != "" {
			name = scope + "[" + name + "]"
		}

		if opts.Contains("omitempty") && isEmptyValue(sv) {
			continue
		}

		if sv.Type().Implements(encoderType) {
			if !reflect.Indirect(sv).IsValid() {
				sv = reflect.New(sv.Type().Elem())
			}

			m := sv.Interface().(Encoder)
			if err := m.EncodeValues(name, &values); err != nil {
				return err
			}
			continue
		}

		if sv.Kind() == reflect.Slice || sv.Kind() == reflect.Array {
			var del byte
			if opts.Contains("comma") {
				del = ','
			} else if opts.Contains("space") {
				del = ' '
			} else if opts.Contains("semicolon") {
				del = ';'
			} else if opts.Contains("brackets") {
				name = name + "[]"
			}

			if del != 0 {
				s := new(bytes.Buffer)
				first := true
				for i := 0; i < sv.Len(); i++ {
					if first {
						first = false
					} else {
						s.WriteByte(del)
					}
					s.WriteString(valueString(sv.Index(i), opts))
				}
				values.Add(name, s.String())
			} else {
				for i := 0; i < sv.Len(); i++ {
					k := name
					if opts.Contains("numbered") {
						k = fmt.Sprintf("%s%d", name, i)
					}
					values.Add(k, valueString(sv.Index(i), opts))
				}
			}
			continue
		}

		if sv.Type() == timeType {
			values.Add(name, valueString(sv, opts))
			continue
		}

		for sv.Kind() == reflect.Ptr {
			if sv.IsNil() {
				break
			}
			sv = sv.Elem()
		}

		if sv.Kind() == reflect.Struct {
			reflectValue(values, sv, name)
			continue
		}

		values.Add(name, valueString(sv, opts))
	}

	for _, f := range embedded {
		if err := reflectValue(values, f, scope); err != nil {
			return err
		}
	}

	return nil
}

// valueString returns the string representation of a value.
func valueString(v reflect.Value, opts tagOptions) string {
	for v.Kind() == reflect.Ptr {
		if v.IsNil() {
			return ""
		}
		v = v.Elem()
	}

	if v.Kind() == reflect.Bool && opts.Contains("int") {
		if v.Bool() {
			return "1"
		}
		return "0"
	}

	if v.Type() == timeType {
		t := v.Interface().(time.Time)
		if opts.Contains("unix") {
			return strconv.FormatInt(t.Unix(), 10)
		}
		return t.Format(time.RFC3339)
	}

	return fmt.Sprint(v.Interface())
}

// isEmptyValue checks if a value should be considered empty for the purposes
// of omitting fields with the "omitempty" option.
func isEmptyValue(v reflect.Value) bool {
	switch v.Kind() {
	case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
		return v.Len() == 0
	case reflect.Bool:
		return !v.Bool()
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return v.Int() == 0
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		return v.Uint() == 0
	case reflect.Float32, reflect.Float64:
		return v.Float() == 0
	case reflect.Interface, reflect.Ptr:
		return v.IsNil()
	}

	if v.Type() == timeType {
		return v.Interface().(time.Time).IsZero()
	}

	return false
}

// tagOptions is the string following a comma in a struct field's "url" tag, or
// the empty string. It does not include the leading comma.
type tagOptions []string

// parseTag splits a struct field's url tag into its name and comma-separated
// options.
func parseTag(tag string) (string, tagOptions) {
	s := strings.Split(tag, ",")
	return s[0], s[1:]
}

// Contains checks whether the tagOptions contains the specified option.
func (o tagOptions) Contains(option string) bool {
	for _, s := range o {
		if s == option {
			return true
		}
	}
	return false
}


text:
module Anonymize
  extend ActiveSupport::Concern

  module ClassMethods
    def not_anonymized
      where(arel_table[:anonymized_at].eq(nil))
    end
  end

  def anonymized?
    anonymized_at?
  end

  def anonymize!(timestamp)
    return if anonymized?

    with_lock do
      self.name = "Signature #{id}"
      self.email = "signature-#{id}@example.com"
      self.ip_address = "192.168.1.1"
      self.anonymized_at = timestamp

      if constituency_id && constituency
        self.postcode = constituency.example_postcode
      else
        self.postcode = nil
      end

      if postcode.blank? && united_kingdom?
        # Validations require a postcode for the UK so use the NHS
        # 'address not known' pseudo-postcode when we didn't find
        # an example postcode for the constituency:
        # https://adoxoblog.wordpress.com/2012/01/21/the-hitchhikers-guide-to-the-nhs/
        self.postcode = "ZZ993WZ"
      end

      save!
    end
  end
end

redpajama_set_name: RedPajamaGithub


text:
SYNONYM

#### According to
The Catalogue of Life, 3rd January 2011

#### Published in
in Gay, Hist. fis. y Politic. Chile, Bot. 7: 392 (1850)

#### Original name
Exidia catillus Mont., 1850

### Remarks
null


text:
ENV["RAILS_ENV"] ||= "test"
require File.expand_path('../../config/environment', __FILE__)
require 'rails/test_help'
require 'capybara/rails'

class ActiveSupport::TestCase
  ActiveRecord::Migration.check_pending!

  # Setup all fixtures in test/fixtures/*.yml for all tests in alphabetical order.
  #
  # Note: You'll currently still have to declare fixtures explicitly in integration tests
  # -- they do not yet inherit this setting
  fixtures :all

  # Add more helper methods to be used by all tests here...

  def parse_show(name, options = {})
    raw_setlist = Rails.root.join("test/fixtures/raw_setlists/#{name}.txt").read
    Parser.parse({raw_setlist: raw_setlist, venue: venues(:bar), performed_at: Date.today}.merge(options))
  end
end

class ActionDispatch::IntegrationTest
  include Capybara::DSL

  def sign_in_as_admin
    page.driver.browser.basic_authorize('admin', Skyway.admin_password)
    visit "/admin"
  end

  def teardown
    Capybara.reset_sessions!
    Capybara.use_default_driver
  end
end


text:
"""
Implements OAuth2 authorization with Slack.

### Recommended workflow with Flask

Add the `/login` and `/logout` endpoints to your Flask app:

```python
@app.route("/login")
def login():
    return slack_auth.login(SLACK_TEAM_ID, SLACK_CLIENT_ID, SLACK_AUTHORIZATION_REDIRECT_URL)
@app.route("/logout", methods=["POST"])
def logout():
    return slack_auth.logout("/")
```

Add callback endpoint for the Slack API will return authorization results to (the endpoint must also be specified in the Slack App settings):

```python
@app.route("/authorize")
def authorize():
    after_login_url = slack_auth.get_after_login_url() or "/"
    return slack_auth.authorize(SLACK_CLIENT_ID, SLACK_CLIENT_SECRET, after_login_url, flask.redirect(after_login_url))
```

Add a decorator to conveniently protect endpoints using Slack login.

```python
slack_membership_required = slack_auth.team_membership_required(
    SLACK_TEAM_ID,
    lambda page_url: flask.url_for("login", next=page_url)
)
```

Use the decorator in endpoints:

```python
@app.route("/")
@slack_membership_required
def hello():
    return "Hello World!"
```

Now, whenever you try to access the top-level page of the app, you'll be asked to login with Slack, and then redirected to the original page afterward.
"""

import json
import uuid
from functools import wraps
from urllib.parse import urlencode
import time

import flask
import requests

# maximum number of seconds before another users.identity Slack API call is required to re-validate a previously validated token
# we don't call users.identity on every request because that would really slow things down, so we simply make sure we've called it recently enough that we feel safe trusting that it's still validated
AUTHORIZATION_TIMEOUT = 60 * 10

# whether to disable authorization when the app is in debug mode
DISABLE_AUTHORIZATION_IN_DEBUG_MODE = True

def login(slack_team_id, slack_client_id, slack_oauth_redirect_url):
    """
    Redirects to the OAuth2 authorization page for the Slack team specified by `slack_team_id`, using the client ID `slack_client_id` and redirecting to `slack_oauth_redirect_url` afterward.

    `slack_team_id` can be found by going to the [team.info API tester](https://api.slack.com/methods/team.info/test) and taking the value of the "id" field under "team" in the result.

    `slack_client_id` and `slack_oauth_redirect_url` can be found in your [Slack app configuration](https://api.slack.com/apps).
    """
    params = {
        "client_id": slack_client_id,
        "scope": "identity.basic",
        "redirect_url": slack_oauth_redirect_url,
        "team": slack_team_id,
    }
    if "next" in flask.request.args:
        params["state"] = flask.request.args["next"]
    return flask.redirect("https://slack.com/oauth/authorize?" + urlencode(params))

def logout(next_page_url):
    if "slack_auth_api_token" in flask.session:
        requests.get("https://slack.com/api/auth.revoke", params={"token": flask.session["slack_auth_api_token"]})
        del flask.session["slack_auth_api_token"]
    if "slack_auth_user_id" in flask.session:
        del flask.session["slack_auth_user_id"]
    if "slack_auth_team_id" in flask.session:
        del flask.session["slack_auth_team_id"]
    return flask.redirect(next_page_url)

def get_after_login_url():
    return flask.request.args.get("state")

def authorize(slack_client_id, slack_client_secret, next_page_url, authorization_failure_page):
    # handle user declining the authorization request
    if flask.request.args.get("error"):
        return authorization_failure_page

    # exchange the OAuth authorization code for a Slack API token
    oauth_authorization_code = flask.request.args.get("code")
    if oauth_authorization_code is None:
        return authorization_failure_page
    slack_access_result = requests.get("https://slack.com/api/oauth.access", params={
        "client_id": slack_client_id,
        "client_secret": slack_client_secret,
        "code": oauth_authorization_code,
    }).json()
    if not slack_access_result["ok"]:
        return authorization_failure_page
    slack_api_token = slack_access_result.get("access_token")

    # confirm the user identity to get the username
    slack_authorization_result = requests.get("https://slack.com/api/users.identity", params={"token": slack_api_token}).json()
    if not slack_authorization_result.get("ok"):
        return authorization_failure_page
    authorized_user_id, authorized_team_id = slack_authorization_result["user"]["id"], slack_authorization_result["team"]["id"]

    # mark the user as logged in
    flask.session["slack_auth_time"] = time.time()
    flask.session["slack_auth_api_token"] = slack_api_token
    flask.session["slack_auth_user_id"] = authorized_user_id
    flask.session["slack_auth_team_id"] = authorized_team_id

    # redirect to the URL we wanted to visit before
    return flask.redirect(next_page_url)

def team_membership_required(slack_team_id, get_authorization_failure_page):
    """
    Decorator factory for Flask endpoints for ensuring that the user is a member of the Slack team specified by `slack_team_id`.
    """
    def team_membership_required_decorator(route_function):
        @wraps(route_function)
        def team_membership_required_decorator_wrapper(*args, **kwargs):
            # don't require logging in if DISABLE_AUTHORIZATION_IN_DEBUG_MODE is set and we're in debug mode
            if DISABLE_AUTHORIZATION_IN_DEBUG_MODE and flask.current_app.debug:
                return route_function(*args, **kwargs)

            # check if user has been authorized for the required team
            if flask.session.get("slack_auth_team_id") != slack_team_id:
                return get_authorization_failure_page(flask.request.url)

            # check if the user's authorization has been validated recently enough
            current_time = time.time()
            if not (current_time - AUTHORIZATION_TIMEOUT <= flask.session.get("slack_auth_time", float("-inf")) <= current_time):
                # retrieve token authorization metadata
                slack_api_token = flask.session.get("slack_auth_api_token")
                slack_authorization_result = requests.get("https://slack.com/api/users.identity", params={"token": slack_api_token}).json()
                if not slack_authorization_result["ok"]:
                    return get_authorization_failure_page(flask.request.url)
                authorized_user_id, authorized_team_id = slack_authorization_result["user"]["id"], slack_authorization_result["team"]["id"]

                # check if user has been authorized for the required team
                if authorized_team_id != slack_team_id:
                    return get_authorization_failure_page(flask.request.url)

                # refresh the authorization to make sure it's fresh for next time
                flask.session["slack_auth_time"] = time.time()
                flask.session["slack_auth_api_token"] = slack_api_token
                flask.session["slack_auth_user_id"] = authorized_user_id
                flask.session["slack_auth_team_id"] = authorized_team_id

            # run the original route
            return route_function(*args, **kwargs)
        return team_membership_required_decorator_wrapper

    return team_membership_required_decorator


text:
module VCLog

  require 'vclog/core_ext'
  require 'vclog/changelog'
  require 'vclog/tag'
  require 'vclog/release'
  require 'erb'

  # A Release History is very similar to a ChangeLog.
  # It differs in that it is divided into releases with
  # version, release date and release note.
  #
  # The release version, date and release note can be
  # descerened from the underlying SCM by identifying
  # the hard tag commits.
  #
  # TODO: Extract output formating from delta parser. Later---Huh?
  #
  class History

    # Location of this file in the file system.
    DIR = File.dirname(__FILE__)

    #
    attr :repo

    ## Alternate title.
    #attr_accessor :title

    ## Current working version.
    #attr_accessor :version

    ## Exclude commit details.
    #attr_accessor :summary

    #
    def initialize(repo)
      @repo = repo

      #opts = OpenStruct.new(opts) if Hash === opts

      #@title   = opts.title || "RELEASE HISTORY"
      #@extra   = opts.extra
      #@version = opts.version
      #@level =  opts.level || 0
    end

    # Tag list from version control system.
    def tags
      @tags ||= repo.tags
    end

    # Changelog object
    #def changelog
    #  @changlog ||= repo.changelog #ChangeLog.new(changes)
    #end

    # Change list from version control system filter for level setting.
    def changes
      @changes ||= (
        if @repo.point
          repo.change_points
        else
          repo.changes
        end
      )
    end

    #
    def releases
      @releases ||= (
        rel = []

        tags = self.tags

        #ver  = repo.bump(version)
        user = repo.user
        time = ::Time.now + (3600 * 24) # one day ahead

        tags << Tag.new(:name=>'HEAD', :id=>'HEAD', :date=>time, :who=>user, :msg=>"Current Development")

        # TODO: Do we need to add a Time.now tag?
        # add current verion to release list (if given)
        #previous_version = tags[0].name
        #if current_version < previous_version  # TODO: need to use natural comparision
        #  raise ArgumentError, "Release version is less than previous version (#{previous_version})."
        #end
        #rels << [current_version, current_release || Time.now]
        #rels = rels.uniq      # only uniq releases

        # sort by release date
        tags = tags.sort{ |a,b| a.date <=> b.date }

        # organize into deltas
        delta = []
        last  = nil
        tags.each do |tag|
          delta << [tag, [last, tag.commit_date]]
          last = tag.commit_date
        end
        # gather changes for each delta
        delta.each do |tag, (started, ended)|
          if started
            set = changes.select{ |c| c.date >= started && c.date < ended  }
            #gt_vers, gt_date = gt.name, gt.date
            #lt_vers, lt_date = lt.name, lt.date
            #gt_date = Time.parse(gt_date) unless Time===gt_date
            #lt_date = Time.parse(lt_date) unless Time===lt_date
            #log = changelog.after(gt).before(lt)
          else
            #lt_vers, lt_date = lt.name, lt.date
            #lt_date = Time.parse(lt_date) unless Time===lt_date
            #log = changelog.before(lt_date)
            set = changes.select{ |c| c.date < ended }
          end

          rel << Release.new(tag, set)
        end
        rel
      )
    end

    # Group +changes+ by tag type.
    def groups(changes)
      @groups ||= changes.group_by{ |e| e.label }  #type
    end

    #
    def to_h
      releases.map{ |rel| rel.to_h }
    end

  end

end


text:
<?php namespace Modules\Workshop\Manager;

use Illuminate\Config\Repository as Config;
use Illuminate\Filesystem\Filesystem;
use Illuminate\Support\Collection;
use Pingpong\Modules\Module;
use Symfony\Component\Yaml\Parser;

class ModuleManager
{
    /**
     * @var Module
     */
    private $module;
    /**
     * @var Config
     */
    private $config;
    /**
     * @var PackageInformation
     */
    private $packageVersion;
    /**
     * @var Filesystem
     */
    private $finder;

    /**
     * @param Config $config
     * @param PackageInformation $packageVersion
     * @param Filesystem $finder
     */
    public function __construct(Config $config, PackageInformation $packageVersion, Filesystem $finder)
    {
        $this->module = app('modules');
        $this->config = $config;
        $this->packageVersion = $packageVersion;
        $this->finder = $finder;
    }

    /**
     * Return all modules
     * @return \Illuminate\Support\Collection
     */
    public function all()
    {
        $modules = new Collection($this->module->all());

        foreach ($modules as $module) {
            $moduleName = $module->getName();
            $package = $this->packageVersion->getPackageInfo("asgardcms/$moduleName-module");
            $module->version = isset($package->version) ? $package->version: 'N/A';
            $module->versionUrl = '#';
            if (isset($package->source->url)) {
                $packageUrl = str_replace('.git', '', $package->source->url);
                $module->versionUrl = $packageUrl . '/tree/' . $package->dist->reference;
            }
        }

        return $modules;
    }

    /**
     * Return all the enabled modules
     * @return array
     */
    public function enabled()
    {
        return $this->module->enabled();
    }

    /**
     * Get the core modules that shouldn't be disabled
     * @return array|mixed
     */
    public function getCoreModules()
    {
        $coreModules = $this->config->get('asgard.core.config.CoreModules');
        $coreModules = array_flip($coreModules);

        return $coreModules;
    }

    /**
     * Get the enabled modules, with the module name as the key
     * @return array
     */
    public function getFlippedEnabledModules()
    {
        $enabledModules = $this->module->enabled();

        $enabledModules = array_map(function (Module $module) {
            return $module->getName();
        }, $enabledModules);

        return array_flip($enabledModules);
    }

    /**
     * Disable the given modules
     * @param $enabledModules
     */
    public function disableModules($enabledModules)
    {
        $coreModules = $this->getCoreModules();

        foreach ($enabledModules as $moduleToDisable => $value) {
            if (isset($coreModules[$moduleToDisable])) {
                continue;
            }
            $module = $this->module->get($moduleToDisable);
            $module->disable();
        }
    }

    /**
     * Enable the given modules
     * @param $modules
     */
    public function enableModules($modules)
    {
        foreach ($modules as $moduleToEnable => $value) {
            $module = $this->module->get($moduleToEnable);
            $module->enable();
        }
    }

    /**
     * Get the changelog for the given module
     * @param Module $module
     * @return array
     */
    public function changelogFor(Module $module)
    {
        $path = $module->getPath() . '/changelog.yml';
        if (! $this->finder->isFile($path)) {
            return [];
        }

        $yamlParser = new Parser();

        $changelog = $yamlParser->parse(file_get_contents($path));

        $changelog['versions'] = $this->limitLastVersionsAmount(array_get($changelog, 'versions', []));

        return $changelog;
    }

    /**
     * Limit the versions to the last 5
     * @param array $versions
     * @return array
     */
    private function limitLastVersionsAmount(array $versions)
    {
        return array_slice($versions, 0, 5);
    }
}


text:
Waypoints is a jQuery plugin that makes it easy to execute a function whenever you scroll to an element.

```js
$('.thing').waypoint(function() {
  alert('You have scrolled to a thing.');
});
```
If you're new to Waypoints, check out the [Get Started](http://imakewebthings.github.com/jquery-waypoints/#get-started) section.

[Read the full documentation](http://imakewebthings.github.com/jquery-waypoints/#docs) for more details on usage and customization.

## Shortcuts

In addition to the normal Waypoints script, extensions exist to make common UI patterns just a little easier to implement:

- [Infinite Scrolling](http://imakewebthings.github.com/jquery-waypoints/shortcuts/infinite-scroll)
- [Sticky Elements](http://imakewebthings.github.com/jquery-waypoints/shortcuts/sticky-elements)

## Examples

Waypoints can also be used as a base for your own custom UI patterns. Here are a few examples:

- [Scroll Analytics](http://imakewebthings.github.com/jquery-waypoints/examples/scroll-analytics)
- [Dial Controls](http://imakewebthings.github.com/jquery-waypoints/examples/dial-controls)

## AMD Module Loader Support

If you're using an AMD loader like [RequireJS](http://requirejs.org/), Waypoints registers itself as a named module, `'waypoints'`. Shortcut scripts are anonymous modules.

## Development Environment

If you want to contribute to Waypoints, I love pull requests that include changes to the source `coffee` files as well as the compiled JS and minified files. You can set up the same environment by running `make setup` (which just aliases to `npm install`). This will install the version of CoffeeScript and UglifyJS that I'm using. From there, running `make build` will compile and minify all the necessary files. Test coffee files are compiled on the fly, so compile and minify do not apply to those files.

## License

Copyright (c) 2011-2014 Caleb Troughton
Licensed under the [MIT license](https://github.com/imakewebthings/jquery-waypoints/blob/master/licenses.txt).

## Support

Unit tests for Waypoints are written with [Jasmine](http://pivotal.github.com/jasmine/) and [jasmine-jquery](https://github.com/velesin/jasmine-jquery).  You can [run them here](http://imakewebthings.github.com/jquery-waypoints/test/). If any of the tests fail, please open an issue and include the browser used, operating system, and description of the failed test.

## Donations

[![Gittip donate button](http://img.shields.io/gittip/imakewebthings.png)](https://www.gittip.com/imakewebthings/ "Donate weekly to this project using Gittip")


text:
/*
 *  FeatureData.h
 *  mert - Minimum Error Rate Training
 *
 *  Created by Nicola Bertoldi on 13/05/08.
 *
 */

#ifndef MERT_FEATURE_DATA_H_
#define MERT_FEATURE_DATA_H_

#include <vector>
#include <iostream>
#include <stdexcept>
#include <boost/lexical_cast.hpp>
#include "FeatureArray.h"

namespace MosesTuning
{

class FeatureData
{
private:
  std::size_t m_num_features;
  std::string m_features;
  std::map<std::string, std::size_t> m_feature_name_to_index; // map from name to index of features
  std::map<std::size_t, std::string> m_index_to_feature_name; // map from index to name of features
  featdata_t m_array;
  idx2name m_index_to_array_name; // map from index to name of array
  name2idx m_array_name_to_index; // map from name to index of array

public:
  FeatureData();
  ~FeatureData() {}

  void clear() {
    m_array.clear();
  }

  FeatureArray& get(size_t idx) {
    return m_array.at(idx);
  }
  const FeatureArray& get(size_t idx) const {
    return m_array.at(idx);
  }

  inline bool exists(int sent_idx) const {
    return existsInternal(getIndex(sent_idx));
  }

  inline bool existsInternal(int sent_idx) const {
    return (sent_idx > -1 && sent_idx < static_cast<int>(m_array.size())) ? true : false;
  }

  inline FeatureStats& get(std::size_t i, std::size_t j) {
    return m_array.at(i).get(j);
  }

  inline const FeatureStats& get(std::size_t i, std::size_t j) const {
    return m_array.at(i).get(j);
  }

  void add(FeatureArray& e);
  void add(FeatureStats& e, int sent_idx);

  std::size_t size() const {
    return m_array.size();
  }

  std::size_t NumberOfFeatures() const {
    return m_num_features;
  }
  void NumberOfFeatures(std::size_t v) {
    m_num_features = v;
  }

  std::string Features() const {
    return m_features;
  }
  void Features(const std::string& f) {
    m_features = f;
  }

  void save(const std::string &file, bool bin=false);
  void save(std::ostream* os, bool bin=false);
  void save(bool bin=false);

  void load(std::istream* is, const SparseVector& sparseWeights);
  void load(const std::string &file, const SparseVector& sparseWeights);

  bool check_consistency() const;

  void setIndex();

  inline int getIndex(int idx) const {
    name2idx::const_iterator i = m_array_name_to_index.find(idx);
    if (i != m_array_name_to_index.end())
      return i->second;
    else
      return -1;
  }

  inline int getName(std::size_t idx) const {
    idx2name::const_iterator i = m_index_to_array_name.find(idx);
    if (i != m_index_to_array_name.end())
      throw std::runtime_error("there is no entry at index " + boost::lexical_cast<std::string>(idx));
    return i->second;
  }

  bool existsFeatureNames() const {
    return (m_index_to_feature_name.size() > 0) ? true : false;
  }

  std::string getFeatureName(std::size_t idx) const {
    if (idx >= m_index_to_feature_name.size())
      throw std::runtime_error("Error: you required an too big index");
    std::map<std::size_t, std::string>::const_iterator it = m_index_to_feature_name.find(idx);
    if (it == m_index_to_feature_name.end()) {
      throw std::runtime_error("Error: specified id is unknown: " + boost::lexical_cast<std::string>(idx));
    } else {
      return it->second;
    }
  }

  std::size_t getFeatureIndex(const std::string& name) const {
    std::map<std::string, std::size_t>::const_iterator it = m_feature_name_to_index.find(name);
    if (it == m_feature_name_to_index.end()) {
      std::string msg = "Error: feature " + name + " is unknown. Known features: ";
      for (std::map<std::string, std::size_t>::const_iterator cit = m_feature_name_to_index.begin();
           cit != m_feature_name_to_index.end(); cit++) {
        msg += cit->first;
        msg += ", ";
      }

      throw std::runtime_error(msg);
    }
    return it->second;
  }

  void setFeatureMap(const std::string& feat);

  /* For debugging */
  std::string ToString() const;
};

}

#endif  // MERT_FEATURE_DATA_H_


text:
namespace {

void SimulateUserTypingInField(content::RenderViewHost* render_view_host,
                               content::WebContents* web_contents,
                               const std::string& field_id) {
  std::string focus("document.getElementById('" + field_id + "').focus();");
  ASSERT_TRUE(content::ExecuteScript(render_view_host, focus));

  content::SimulateKeyPress(web_contents, ui::VKEY_O, false, false, false,
                            false);
  content::SimulateKeyPress(web_contents, ui::VKEY_R, false, false, false,
                            false);
  content::SimulateKeyPress(web_contents, ui::VKEY_A, false, false, false,
                            false);
  content::SimulateKeyPress(web_contents, ui::VKEY_R, false, false, false,
                            false);
  content::SimulateKeyPress(web_contents, ui::VKEY_Y, false, false, false,
                            false);
}

}  // namespace

namespace password_manager {

IN_PROC_BROWSER_TEST_F(PasswordManagerBrowserTestBase, UsernameChanged) {
  scoped_refptr<password_manager::TestPasswordStore> password_store =
      static_cast<password_manager::TestPasswordStore*>(
          PasswordStoreFactory::GetForProfile(
              browser()->profile(), ServiceAccessType::IMPLICIT_ACCESS).get());

  NavigateToFile("/password/signup_form.html");

  NavigationObserver observer(WebContents());
  std::unique_ptr<BubbleObserver> prompt_observer(
      BubbleObserver::Create(WebContents()));
  std::string fill_and_submit =
      "document.getElementById('username_field').value = 'temp';"
      "document.getElementById('password_field').value = 'random';"
      "document.getElementById('input_submit_button').click()";
  ASSERT_TRUE(content::ExecuteScript(RenderViewHost(), fill_and_submit));
  observer.Wait();
  EXPECT_TRUE(prompt_observer->IsSaveShowingPrompt());
  prompt_observer->AcceptSavePrompt();

  // Spin the message loop to make sure the password store had a chance to save
  // the password.
  base::RunLoop run_loop;
  run_loop.RunUntilIdle();
  EXPECT_FALSE(password_store->IsEmpty());

  // Reload the original page to have the saved credentials autofilled.
  NavigationObserver reload_observer(WebContents());
  NavigateToFile("/password/signup_form.html");
  reload_observer.Wait();

  // Let the user interact with the page, so that DOM gets modification events,
  // needed for autofilling fields.
  content::SimulateMouseClickAt(
      WebContents(), 0, blink::WebMouseEvent::ButtonLeft, gfx::Point(1, 1));

  WaitForElementValue("username_field", "temp");
  WaitForElementValue("password_field", "random");

  // Change username and submit. This should add the characters "ORARY" to the
  // already autofilled username.
  SimulateUserTypingInField(RenderViewHost(), WebContents(), "username_field");
  // TODO(gcasto): Not sure why this click is required.
  content::SimulateMouseClickAt(
      WebContents(), 0, blink::WebMouseEvent::ButtonLeft, gfx::Point(1, 1));
  WaitForElementValue("username_field", "tempORARY");

  NavigationObserver second_observer(WebContents());
  std::unique_ptr<BubbleObserver> second_prompt_observer(
      BubbleObserver::Create(WebContents()));
  std::string submit =
      "document.getElementById('input_submit_button').click();";
  ASSERT_TRUE(content::ExecuteScript(RenderViewHost(), submit));
  second_observer.Wait();
  EXPECT_TRUE(second_prompt_observer->IsSaveShowingPrompt());
  second_prompt_observer->AcceptSavePrompt();

  // Spin the message loop to make sure the password store had a chance to save
  // the password.
  base::RunLoop third_run_loop;
  third_run_loop.RunUntilIdle();
  EXPECT_FALSE(password_store->IsEmpty());

  // Verify that there are two saved password, the old password and the new
  // password.
  password_manager::TestPasswordStore::PasswordMap stored_passwords =
      password_store->stored_passwords();
  EXPECT_EQ(1u, stored_passwords.size());
  EXPECT_EQ(2u, stored_passwords.begin()->second.size());
  EXPECT_EQ(base::UTF8ToUTF16("temp"),
            (stored_passwords.begin()->second)[0].username_value);
  EXPECT_EQ(base::UTF8ToUTF16("tempORARY"),
            (stored_passwords.begin()->second)[1].username_value);
}

}  // namespace password_manager


text:
This program calculates the shear and moment induced in a simply supported beam
with a constant moment of area (moment of inertia) along the span length. Each
axle is placed at each node (critial section) and the location of the other axles
is calculated. The shear on each side of the node is calculated and the moment at each node is
calculated.

By placing the axle at each node it ensures that the maximum forces are
calculated at those nodes.

This program uses a functional programming paradigm approach.

For documentation use pydoc or see the poorly formatted html here.


text:
ACCEPTED

#### According to
Index Fungorum

#### Published in
Bull. trimest. Soc. mycol. Fr. 51(3-4): 366 (1936)

#### Original name
Coriolus subradula Pilát

### Remarks
null


text:
using AlephNote.Common.AlephXMLSerialization;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Xml.Linq;
using System.Collections;
using AlephNote.PluginInterface.Objects;
using AlephNote.PluginInterface.Objects.AXML;
using MSHC.WPF.MVVM;

namespace AlephNote.Common.Settings.Types
{
	public class KeyValueFlatCustomList<TValue> : ObservableObject, IAlephCustomSerializableField, IEnumerable<KeyValuePair<string, TValue>> where TValue : IAlephCustomFlatSerializableField
	{
		public readonly TValue DefaultDictValue;

		private readonly Dictionary<string, TValue> _data;

		public IReadOnlyDictionary<string, TValue> ListData => _data;

		public KeyValueFlatCustomList(IEnumerable<Tuple<string, TValue>> data, TValue ddv)
		{
			_data = data.ToDictionary(d => d.Item1, d => d.Item2);
			DefaultDictValue = ddv;
		}

		public object DeserializeNew(XElement source, AXMLSerializationSettings opt)
		{
			var d = source.Elements("Value").Select(p => Tuple.Create(p.Attribute("Key").Value, (TValue)DefaultDictValue.DeserializeNew(p.Value, opt))).ToArray();
			return new KeyValueFlatCustomList<TValue>(d, DefaultDictValue);
		}

		public object GetTypeStr()
		{
			return $"KeyValueCustomList[{DefaultDictValue.GetTypeStr()}]";
		}

		public void Serialize(XElement target, AXMLSerializationSettings opt)
		{
			foreach (var d in _data)
			{
				var x = new XElement("Value");
				x.Add(new XAttribute("Key", d.Key));
				if ((opt & AXMLSerializationSettings.IncludeTypeInfo) != 0) x.Add(new XAttribute("type", d.Value.GetTypeStr()));
				x.Value = d.Value.Serialize(opt);
				target.Add(x);
			}
		}

		public bool TryGetValue(string key, out TValue value)
		{
			return _data.TryGetValue(key, out value);
		}

		public bool Contains(string key)
		{
			return _data.ContainsKey(key);
		}

		public IEnumerator<KeyValuePair<string, TValue>> GetEnumerator()
		{
			return ListData.GetEnumerator();
		}

		IEnumerator IEnumerable.GetEnumerator()
		{
			return ListData.GetEnumerator();
		}

		public KeyValueFlatCustomList<TValue> Concat(Tuple<string, TValue> v)
		{
			return new KeyValueFlatCustomList<TValue>(
				ListData
					.AsEnumerable()
					.Select(ld => Tuple.Create(ld.Key, ld.Value))
					.Concat(new []{v})
					.GroupBy(p => p.Item1)
					.Select(p => p.First()),
				DefaultDictValue);
		}
	}
}


text:
cg.ThematicLayer = function( parent, data ) {
	this.parent = parent;
	this.map = parent.map;
	this.data = data;
	this.bbox = null;
}

cg.ThematicLayer.prototype.bounds = function() {
	return this.bbox;
}

cg.ThematicLayer.prototype.zoomend = function( oldlevel, newlevel ) {
	return;
}

cg.ThematicLayer.prototype.moveend = function() {
	return;
}

cg.ThematicLayer.prototype.maptypechanged = function() {
	return;
}

cg.ThematicLayer.prototype.render = function() {
	return;
}


text:
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />

    <title>Pandoc — file_helper 0.1.0 documentation</title>

    <link rel="stylesheet" href="_static/classic.css" type="text/css" />
    <link rel="stylesheet" href="_static/pygments.css" type="text/css" />

    <link rel="top" title="file_helper 0.1.0 documentation" href="index.html" />
    <link rel="prev" title="Package Setup" href="package.setup.html" />

Navigation

          previous

file_helper 0.1.0 documentation »

Pandoc¶

What Pandoc Is¶

Pandoc is a Linux application for converting documents from one markup format to another.

Visit the Pandoc website http://pandoc.org/

Why Pandoc is Used¶

To convert Markdown files to reStructuredText.

How Pandoc is Used¶

Create the Markdown file(s)

Install Pandoc

Create a docs folder

Navigate to the docs folder

Put a copy of the modified Sphinx Makefile

Put a copy of the modified Sphinx Config (conf.py)

Create an index file: index.rs_

This is the main page of the documentation and must include a table of contents

The file extension is required for non-code based files to work with the modified Sphinx Makefile

Link to the converted files within the index file

Generate the documenation

make html

Table Of Contents

Pandoc

What Pandoc Is

Why Pandoc is Used

How Pandoc is Used

Previous topic

Package Setup

This Page

Show Source

Quick search

    <form class="search" action="search.html" method="get">
      <input type="text" name="q" />
      <input type="submit" value="Go" />
      <input type="hidden" name="check_keywords" value="yes" />
      <input type="hidden" name="area" value="default" />
    </form>

    Enter search terms or a module, class or function name.

Navigation

          previous

file_helper 0.1.0 documentation »

        © Copyright 2015, Qevo.
      Last updated on 13 Sep 2015.


text:
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet href="CoreNLP-to-HTML.xsl" type="text/xsl"?>
<root>
  <document>
    <sentences>
      <sentence id="1">
        <tokens>
          <token id="1">
            <word>What</word>
            <lemma>what</lemma>
            <CharacterOffsetBegin>0</CharacterOffsetBegin>
            <CharacterOffsetEnd>4</CharacterOffsetEnd>
            <POS>WP</POS>
          </token>
          <token id="2">
            <word>are</word>
            <lemma>be</lemma>
            <CharacterOffsetBegin>5</CharacterOffsetBegin>
            <CharacterOffsetEnd>8</CharacterOffsetEnd>
            <POS>VBP</POS>
          </token>
          <token id="3">
            <word>some</word>
            <lemma>some</lemma>
            <CharacterOffsetBegin>9</CharacterOffsetBegin>
            <CharacterOffsetEnd>13</CharacterOffsetEnd>
            <POS>DT</POS>
          </token>
          <token id="4">
            <word>of</word>
            <lemma>of</lemma>
            <CharacterOffsetBegin>14</CharacterOffsetBegin>
            <CharacterOffsetEnd>16</CharacterOffsetEnd>
            <POS>IN</POS>
          </token>
          <token id="5">
            <word>my</word>
            <lemma>my</lemma>
            <CharacterOffsetBegin>17</CharacterOffsetBegin>
            <CharacterOffsetEnd>19</CharacterOffsetEnd>
            <POS>PRP$</POS>
          </token>
          <token id="6">
            <word>options</word>
            <lemma>option</lemma>
            <CharacterOffsetBegin>20</CharacterOffsetBegin>
            <CharacterOffsetEnd>27</CharacterOffsetEnd>
            <POS>NNS</POS>
          </token>
          <token id="7">
            <word>for</word>
            <lemma>for</lemma>
            <CharacterOffsetBegin>28</CharacterOffsetBegin>
            <CharacterOffsetEnd>31</CharacterOffsetEnd>
            <POS>IN</POS>
          </token>
          <token id="8">
            <word>a</word>
            <lemma>a</lemma>
            <CharacterOffsetBegin>32</CharacterOffsetBegin>
            <CharacterOffsetEnd>33</CharacterOffsetEnd>
            <POS>DT</POS>
          </token>
          <token id="9">
            <word>good</word>
            <lemma>good</lemma>
            <CharacterOffsetBegin>34</CharacterOffsetBegin>
            <CharacterOffsetEnd>38</CharacterOffsetEnd>
            <POS>JJ</POS>
          </token>
          <token id="10">
            <word>VPN</word>
            <lemma>VPN</lemma>
            <CharacterOffsetBegin>39</CharacterOffsetBegin>
            <CharacterOffsetEnd>42</CharacterOffsetEnd>
            <POS>NNP</POS>
          </token>
          <token id="11">
            <word>Program</word>
            <lemma>Program</lemma>
            <CharacterOffsetBegin>43</CharacterOffsetBegin>
            <CharacterOffsetEnd>50</CharacterOffsetEnd>
            <POS>NNP</POS>
          </token>
          <token id="12">
            <word>?</word>
            <lemma>?</lemma>
            <CharacterOffsetBegin>50</CharacterOffsetBegin>
            <CharacterOffsetEnd>51</CharacterOffsetEnd>
            <POS>.</POS>
          </token>
        </tokens>
        <parse>(ROOT (SBARQ (WHNP (WP What)) (VBP are) (NP (NP (DT some)) (PP (IN of) (NP (NP (PRP$ my) (NNS options)) (PP (IN for) (NP (DT a) (JJ good) (NNP VPN) (NNP Program)))))) (. ?))) </parse>
        <dependencies type="basic-dependencies">
          <dep type="root">
            <governor idx="0">ROOT</governor>
            <dependent idx="1">What</dependent>
          </dep>
          <dep type="cop">
            <governor idx="1">What</governor>
            <dependent idx="2">are</dependent>
          </dep>
          <dep type="nsubj">
            <governor idx="1">What</governor>
            <dependent idx="3">some</dependent>
          </dep>
          <dep type="prep">
            <governor idx="3">some</governor>
            <dependent idx="4">of</dependent>
          </dep>
          <dep type="poss">
            <governor idx="6">options</governor>
            <dependent idx="5">my</dependent>
          </dep>
          <dep type="pobj">
            <governor idx="4">of</governor>
            <dependent idx="6">options</dependent>
          </dep>
          <dep type="prep">
            <governor idx="6">options</governor>
            <dependent idx="7">for</dependent>
          </dep>
          <dep type="det">
            <governor idx="11">Program</governor>
            <dependent idx="8">a</dependent>
          </dep>
          <dep type="amod">
            <governor idx="11">Program</governor>
            <dependent idx="9">good</dependent>
          </dep>
          <dep type="nn">
            <governor idx="11">Program</governor>
            <dependent idx="10">VPN</dependent>
          </dep>
          <dep type="pobj">
            <governor idx="7">for</governor>
            <dependent idx="11">Program</dependent>
          </dep>
        </dependencies>
        <dependencies type="collapsed-dependencies">
          <dep type="root">
            <governor idx="0">ROOT</governor>
            <dependent idx="1">What</dependent>
          </dep>
          <dep type="cop">
            <governor idx="1">What</governor>
            <dependent idx="2">are</dependent>
          </dep>
          <dep type="nsubj">
            <governor idx="1">What</governor>
            <dependent idx="3">some</dependent>
          </dep>
          <dep type="poss">
            <governor idx="6">options</governor>
            <dependent idx="5">my</dependent>
          </dep>
          <dep type="prep_of">
            <governor idx="3">some</governor>
            <dependent idx="6">options</dependent>
          </dep>
          <dep type="det">
            <governor idx="11">Program</governor>
            <dependent idx="8">a</dependent>
          </dep>
          <dep type="amod">
            <governor idx="11">Program</governor>
            <dependent idx="9">good</dependent>
          </dep>
          <dep type="nn">
            <governor idx="11">Program</governor>
            <dependent idx="10">VPN</dependent>
          </dep>
          <dep type="prep_for">
            <governor idx="6">options</governor>
            <dependent idx="11">Program</dependent>
          </dep>
        </dependencies>
        <dependencies type="collapsed-ccprocessed-dependencies">
          <dep type="root">
            <governor idx="0">ROOT</governor>
            <dependent idx="1">What</dependent>
          </dep>
          <dep type="cop">
            <governor idx="1">What</governor>
            <dependent idx="2">are</dependent>
          </dep>
          <dep type="nsubj">
            <governor idx="1">What</governor>
            <dependent idx="3">some</dependent>
          </dep>
          <dep type="poss">
            <governor idx="6">options</governor>
            <dependent idx="5">my</dependent>
          </dep>
          <dep type="prep_of">
            <governor idx="3">some</governor>
            <dependent idx="6">options</dependent>
          </dep>
          <dep type="det">
            <governor idx="11">Program</governor>
            <dependent idx="8">a</dependent>
          </dep>
          <dep type="amod">
            <governor idx="11">Program</governor>
            <dependent idx="9">good</dependent>
          </dep>
          <dep type="nn">
            <governor idx="11">Program</governor>
            <dependent idx="10">VPN</dependent>
          </dep>
          <dep type="prep_for">
            <governor idx="6">options</governor>
            <dependent idx="11">Program</dependent>
          </dep>
        </dependencies>
      </sentence>
      <sentence id="2">
        <tokens>
          <token id="1">
            <word>benefits</word>
            <lemma>benefit</lemma>
            <CharacterOffsetBegin>52</CharacterOffsetBegin>
            <CharacterOffsetEnd>60</CharacterOffsetEnd>
            <POS>NNS</POS>
          </token>
          <token id="2">
            <word>under</word>
            <lemma>under</lemma>
            <CharacterOffsetBegin>61</CharacterOffsetBegin>
            <CharacterOffsetEnd>66</CharacterOffsetEnd>
            <POS>IN</POS>
          </token>
          <token id="3">
            <word>a</word>
            <lemma>a</lemma>
            <CharacterOffsetBegin>67</CharacterOffsetBegin>
            <CharacterOffsetEnd>68</CharacterOffsetEnd>
            <POS>DT</POS>
          </token>
          <token id="4">
            <word>health</word>
            <lemma>health</lemma>
            <CharacterOffsetBegin>69</CharacterOffsetBegin>
            <CharacterOffsetEnd>75</CharacterOffsetEnd>
            <POS>NN</POS>
          </token>
          <token id="5">
            <word>insurance</word>
            <lemma>insurance</lemma>
            <CharacterOffsetBegin>76</CharacterOffsetBegin>
            <CharacterOffsetEnd>85</CharacterOffsetEnd>
            <POS>NN</POS>
          </token>
          <token id="6">
            <word>plan</word>
            <lemma>plan</lemma>
            <CharacterOffsetBegin>86</CharacterOffsetBegin>
            <CharacterOffsetEnd>90</CharacterOffsetEnd>
            <POS>NN</POS>
          </token>
        </tokens>
        <parse>(ROOT (NP (NP (NNS benefits)) (PP (IN under) (NP (DT a) (NN health) (NN insurance) (NN plan))))) </parse>
        <dependencies type="basic-dependencies">
          <dep type="root">
            <governor idx="0">ROOT</governor>
            <dependent idx="1">benefits</dependent>
          </dep>
          <dep type="prep">
            <governor idx="1">benefits</governor>
            <dependent idx="2">under</dependent>
          </dep>
          <dep type="det">
            <governor idx="6">plan</governor>
            <dependent idx="3">a</dependent>
          </dep>
          <dep type="nn">
            <governor idx="6">plan</governor>
            <dependent idx="4">health</dependent>
          </dep>
          <dep type="nn">
            <governor idx="6">plan</governor>
            <dependent idx="5">insurance</dependent>
          </dep>
          <dep type="pobj">
            <governor idx="2">under</governor>
            <dependent idx="6">plan</dependent>
          </dep>
        </dependencies>
        <dependencies type="collapsed-dependencies">
          <dep type="root">
            <governor idx="0">ROOT</governor>
            <dependent idx="1">benefits</dependent>
          </dep>
          <dep type="det">
            <governor idx="6">plan</governor>
            <dependent idx="3">a</dependent>
          </dep>
          <dep type="nn">
            <governor idx="6">plan</governor>
            <dependent idx="4">health</dependent>
          </dep>
          <dep type="nn">
            <governor idx="6">plan</governor>
            <dependent idx="5">insurance</dependent>
          </dep>
          <dep type="prep_under">
            <governor idx="1">benefits</governor>
            <dependent idx="6">plan</dependent>
          </dep>
        </dependencies>
        <dependencies type="collapsed-ccprocessed-dependencies">
          <dep type="root">
            <governor idx="0">ROOT</governor>
            <dependent idx="1">benefits</dependent>
          </dep>
          <dep type="det">
            <governor idx="6">plan</governor>
            <dependent idx="3">a</dependent>
          </dep>
          <dep type="nn">
            <governor idx="6">plan</governor>
            <dependent idx="4">health</dependent>
          </dep>
          <dep type="nn">
            <governor idx="6">plan</governor>
            <dependent idx="5">insurance</dependent>
          </dep>
          <dep type="prep_under">
            <governor idx="1">benefits</governor>
            <dependent idx="6">plan</dependent>
          </dep>
        </dependencies>
      </sentence>
    </sentences>
  </document>
</root>


text:
package org.peergreen.vaadin.diagram.client.event;

import com.google.gwt.event.dom.client.MouseUpEvent;

public interface IObjectMouseUp {

    void mouseUp(MouseUpEvent event);
}


text:
module Lyndon
  class Delegate
    def webView(sender, didFinishLoadForFrame:frame)
      NSApplication.sharedApplication.stop(self)
    end
  end
end


text:
<?php
$uri="http://slapp002i0:28880/adxwsvc/services/CAdxWebServiceXmlCC?wsdl";
$trace = 'off';       // values 'on' or 'off'
$performance = False;    // values True or False
$CallContext["codeLang"] = "FRA";
$CallContext["codeUser"] = "RDUVAL002";
$CallContext["password"] = "Cm2k4";
$CallContext["poolAlias"] = "PPAVEO";
$CallContext["requestConfig"] = "adxwss.trace.on=$trace&adxwss.trace.size=16384&adonix.trace.on=$trace&adonix.trace.level=3&adonix.trace.size=8";
?>


text:
/**
 */
package requirements.impl;

import org.eclipse.emf.ecore.EClass;

import requirements.Constraint;
import requirements.RequirementsPackage;

/**
 *
 * An implementation of the model object 'Constraint'.
 *
 *
 * @generated
 */
public class ConstraintImpl extends SystemRequirementImpl implements Constraint {
	/**
	 *
	 *
	 * @generated
	 */
	protected ConstraintImpl() {
		super();
	}

	/**
	 *
	 *
	 * @generated
	 */
	@Override
	protected EClass eStaticClass() {
		return RequirementsPackage.Literals.CONSTRAINT;
	}

} //ConstraintImpl


text:
"""
Copyright (c) 2014-2016 Miroslav Stampar (@stamparm)
See the file 'LICENSE' for copying permission
"""

from __future__ import print_function  # Requires: Python >= 2.6

import sys

sys.dont_write_bytecode = True

import core.versioncheck

import math
import mmap
import optparse
import os
import platform
import re
import socket
import subprocess
import struct
import threading
import time
import traceback
import urllib
import urlparse

from core.attribdict import AttribDict
from core.common import check_sudo
from core.common import load_trails
from core.enums import BLOCK_MARKER
from core.enums import PROTO
from core.enums import TRAIL
from core.log import create_log_directory
from core.log import get_error_log_handle
from core.log import log_error
from core.log import log_event
from core.parallel import worker
from core.parallel import write_block
from core.settings import check_memory
from core.settings import config
from core.settings import CAPTURE_TIMEOUT
from core.settings import CONFIG_FILE
from core.settings import DLT_OFFSETS
from core.settings import IGNORE_DNS_QUERY_SUFFIXES
from core.settings import IPPROTO_LUT
from core.settings import LOCALHOST_IP
from core.settings import MMAP_ZFILL_CHUNK_LENGTH
from core.settings import MAX_RESULT_CACHE_ENTRIES
from core.settings import NAME
from core.settings import NO_SUCH_NAME_COUNTERS
from core.settings import NO_SUCH_NAME_PER_HOUR_THRESHOLD
from core.settings import PORT_SCANNING_THRESHOLD
from core.settings import read_config
from core.settings import REGULAR_SENSOR_SLEEP_TIME
from core.settings import SNAP_LEN
from core.settings import SUSPICIOUS_DIRECT_DOWNLOAD_EXTENSIONS
from core.settings import SUSPICIOUS_DOMAIN_CONSONANT_THRESHOLD
from core.settings import SUSPICIOUS_DOMAIN_ENTROPY_THRESHOLD
from core.settings import SUSPICIOUS_DOMAIN_LENGTH_THRESHOLD
from core.settings import SUSPICIOUS_FILENAMES
from core.settings import SUSPICIOUS_HTTP_REQUEST_REGEX
from core.settings import SUSPICIOUS_HTTP_REQUEST_FORCE_ENCODE_CHARS
from core.settings import SUSPICIOUS_UA_LENGTH_THRESHOLD
from core.settings import SUSPICIOUS_UA_REGEX
from core.settings import trails
from core.settings import VERSION
from core.settings import WHITELIST
from core.settings import WHITELIST_DIRECT_DOWNLOAD_KEYWORDS
from core.settings import WHITELIST_LONG_DOMAIN_NAME_KEYWORDS
from core.settings import WHITELIST_HTTP_REQUEST_KEYWORDS
from core.settings import WHITELIST_UA_KEYWORDS
from core.update import update_ipcat
from core.update import update_trails

_buffer = None
_caps = []
_connect_sec = 0
_connect_src_dst = {}
_connect_src_details = {}
_count = 0
_locks = AttribDict()
_multiprocessing = None
_n = None
_result_cache = {}

try:
    import pcapy
except ImportError:
    if subprocess.mswindows:
        exit("[!] please install 'WinPcap' (e.g. 'http://www.winpcap.org/install/') and Pcapy (e.g. 'https://breakingcode.wordpress.com/?s=pcapy')")
    else:
        msg, _ = "[!] please install 'Pcapy'", platform.linux_distribution()[0].lower()
        for distro, install in {("fedora", "centos"): "sudo yum install pcapy", ("debian", "ubuntu"): "sudo apt-get install python-pcapy"}.items():
            if _ in distro:
                msg += " (e.g. '%s')" % install
                break
        exit(msg)

def _check_domain(query, sec, usec, src_ip, src_port, dst_ip, dst_port, proto):
    parts = query.lower().split('.')

    for i in xrange(0, len(parts)):
        domain = '.'.join(parts[i:])
        if domain in trails:
            if domain == query:
                trail = domain
            else:
                _ = ".%s" % domain
                trail = "(%s)%s" % (query[:-len(_)], _)

            log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, proto, TRAIL.DNS, trail, trails[domain][0], trails[domain][1]))
            return

    if config.USE_HEURISTICS:
        if len(parts[0]) > SUSPICIOUS_DOMAIN_LENGTH_THRESHOLD and '-' not in parts[0]:
            trail = None

            if len(parts) > 2:
                if '.'.join(parts[-2:]) not in WHITELIST:
                    trail = "(%s).%s" % ('.'.join(parts[:-2]), '.'.join(parts[-2:]))
            elif len(parts) == 2:
                if '.'.join(parts) not in WHITELIST:
                    trail = "(%s).%s" % (parts[0], parts[1])
            else:
                trail = query

            if trail and not any(_ in trail for _ in WHITELIST_LONG_DOMAIN_NAME_KEYWORDS):
                log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, proto, TRAIL.DNS, trail, "long domain name (suspicious)", "(heuristic)"))

        elif "sinkhole" in query:
            log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, proto, TRAIL.DNS, query, "potential sinkhole domain (suspicious)", "(heuristic)"))

def _process_ip(ip_data, sec, usec):
    """
    Processes single (raw) IP layer data
    """

    global _connect_sec

    try:
        if len(_result_cache) > MAX_RESULT_CACHE_ENTRIES:
            _result_cache.clear()

        if config.USE_HEURISTICS:
            if _locks.connect_sec:
                _locks.connect_sec.acquire()

            connect_sec = _connect_sec
            _connect_sec = sec

            if _locks.connect_sec:
                _locks.connect_sec.release()

            if sec > connect_sec:
                for key in _connect_src_dst:
                    if len(_connect_src_dst[key]) > PORT_SCANNING_THRESHOLD:
                        _src_ip, _dst_ip = key.split(':')
                        if _src_ip not in WHITELIST:
                            _src_ports = set(str(_[2]) for _ in _connect_src_details[key])
                            _dst_ports = set(str(_[3]) for _ in _connect_src_details[key])
                            log_event((sec, usec, _src_ip, ','.join(_src_ports), _dst_ip, ','.join(_dst_ports), PROTO.TCP, TRAIL.IP, _src_ip, "potential port scanning", "(heuristic)"))

                _connect_src_dst.clear()
                _connect_src_details.clear()

        ip_header = struct.unpack("!BBHHHBBH4s4s", ip_data[:20])

        ip_length = ip_header[2]
        ip_data = ip_data[:ip_length]  # truncate
        iph_length = (ip_header[0] & 0xf) << 2
        protocol = ip_header[6]
        src_ip = socket.inet_ntoa(ip_header[8])
        dst_ip = socket.inet_ntoa(ip_header[9])

        if protocol == socket.IPPROTO_TCP:  # TCP
            src_port, dst_port, _, _, doff_reserved, flags = struct.unpack("!HHLLBB", ip_data[iph_length:iph_length+14])

            if flags == 2:  # SYN set (only)
                if dst_ip in trails:
                    log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, PROTO.TCP, TRAIL.IP, dst_ip, trails[dst_ip][0], trails[dst_ip][1]))
                elif src_ip in trails and dst_ip != LOCALHOST_IP:
                    log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, PROTO.TCP, TRAIL.IP, src_ip, trails[src_ip][0], trails[src_ip][1]))

                if config.USE_HEURISTICS:
                    if dst_ip != LOCALHOST_IP:
                        key = "%s:%s" % (src_ip, dst_ip)
                        if key not in _connect_src_dst:
                            _connect_src_dst[key] = set()
                            _connect_src_details[key] = set()
                        _connect_src_dst[key].add(dst_port)
                        _connect_src_details[key].add((sec, usec, src_port, dst_port))

            if flags & 8 != 0:  # PSH set
                tcph_length = doff_reserved >> 4
                h_size = iph_length + (tcph_length << 2)
                tcp_data = ip_data[h_size:]

                if src_port == 80 and tcp_data.startswith("HTTP/") and any(_ in tcp_data[:tcp_data.find("\r\n\r\n")] for _ in ("X-Sinkhole:", "Server: Apache 1.0/SinkSoft")):
                    log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, PROTO.TCP, TRAIL.IP, src_ip, "sinkhole response (malware)", "(heuristic)"))

                method, path = None, None
                index = tcp_data.find("\n")
                if index >= 0:
                    line = tcp_data[:index]
                    if line.count(' ') == 2 and " HTTP/" in line:
                        method = line.split(' ')[0].upper()
                        path = line.split(' ')[1].lower()

                if method and path:
                    post_data = None
                    host = dst_ip
                    index = tcp_data.find("\r\nHost:")

                    if index >= 0:
                        index = index + len("\r\nHost:")
                        host = tcp_data[index:tcp_data.find("\r\n", index)]
                        host = host.strip()
                        host = re.sub(r":80\Z", "", host)
                        if dst_ip in trails and not (host[-1].isdigit() and ':' not in host):
                            log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, PROTO.TCP, TRAIL.IP, dst_ip, trails[dst_ip][0], trails[dst_ip][1]))
                    elif config.USE_HEURISTICS and config.CHECK_MISSING_HOST:
                        log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, PROTO.TCP, TRAIL.HTTP, "%s%s" % (host, path), "suspicious http request (missing host header)", "(heuristic)"))

                    index = tcp_data.find("\r\n\r\n")
                    if index >= 0:
                        post_data = tcp_data[index:]

                    if "://" in path:
                        url = path.split("://", 1)[1]

                        if '/' not in url:
                            url = "%s/" % url

                        host, path = url.split('/', 1)
                        host = re.sub(r":80\Z", "", host)
                        path = "/%s" % path
                        proxy_domain = host.split(':')[0]
                        _check_domain(proxy_domain, sec, usec, src_ip, src_port, dst_ip, dst_port, PROTO.TCP)
                    elif method == "CONNECT":
                        if '/' in path:
                            host, path = path.split('/', 1)
                            path = "/%s" % path
                        else:
                            host, path = path, '/'
                        host = re.sub(r":80\Z", "", host)
                        url = "%s%s" % (host, path)
                        proxy_domain = host.split(':')[0]
                        _check_domain(proxy_domain, sec, usec, src_ip, src_port, dst_ip, dst_port, PROTO.TCP)
                    else:
                        url = "%s%s" % (host, path)

                    if config.USE_HEURISTICS:
                        user_agent, result = None, None
                        match = re.search("(?i)\r\nUser-Agent:([^\r\n]+)", tcp_data)
                        if match:
                            user_agent = urllib.unquote(match.group(1)).strip()

                        if user_agent:
                            result = _result_cache.get(user_agent)
                            if result is None:
                                if not any(_ in user_agent for _ in WHITELIST_UA_KEYWORDS):
                                    match = re.search(SUSPICIOUS_UA_REGEX, user_agent)
                                    if match:
                                        result = _result_cache[user_agent] = match.group(0).join(("(%s)" if _ else "%s") % _.replace('(', "\\(").replace(')', "\\)") for _ in user_agent.split(match.group(0), 1))
                                if not result:
                                    _result_cache[user_agent] = False
                            if result:
                                log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, PROTO.TCP, TRAIL.UA, result, "suspicious user agent", "(heuristic)"))

                        if not result and config.CHECK_SHORT_OR_MISSING_USER_AGENT:
                            if user_agent is None:
                                log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, PROTO.TCP, TRAIL.HTTP, url, "suspicious http request (missing user agent header)", "(heuristic)"))
                            elif len(user_agent) < SUSPICIOUS_UA_LENGTH_THRESHOLD:
                                log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, PROTO.TCP, TRAIL.UA, user_agent, "suspicious user agent (too short)", "(heuristic)"))

                    checks = [path.rstrip('/')]
                    if '?' in path:
                        checks.append(path.split('?')[0].rstrip('/'))

                    _ = os.path.splitext(checks[-1])
                    if _[1]:
                        checks.append(_[0])

                    if checks[-1].count('/') > 1:
                        checks.append(checks[-1][:checks[-1].rfind('/')])

                    for check in filter(None, checks):
                        for _ in ("", host):
                            check = "%s%s" % (_, check)
                            if check in trails:
                                parts = url.split(check)
                                other = ("(%s)" % _ if _ else _ for _ in parts)
                                trail = check.join(other)
                                log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, PROTO.TCP, TRAIL.URL, trail, trails[check][0], trails[check][1]))
                                return

                    if config.USE_HEURISTICS:
                        if any(char in path for char in SUSPICIOUS_HTTP_REQUEST_FORCE_ENCODE_CHARS):
                            for char in SUSPICIOUS_HTTP_REQUEST_FORCE_ENCODE_CHARS:
                                path = path.replace(char, urllib.quote(char))

                        if host not in WHITELIST:
                            if not any(_ in path for _ in WHITELIST_HTTP_REQUEST_KEYWORDS):
                                result = _result_cache.get(path)
                                if result is None:
                                    result = _result_cache[path] = re.search(SUSPICIOUS_HTTP_REQUEST_REGEX, urllib.unquote(path)) is not None
                                if result:
                                    trail = "%s(%s)" % (host, path)
                                    log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, PROTO.TCP, TRAIL.URL, trail, "suspicious http request", "(heuristic)"))
                                    return

                            if post_data and not any(_ in post_data for _ in WHITELIST_HTTP_REQUEST_KEYWORDS):
                                result = _result_cache.get(post_data)
                                if result is None:
                                    result = _result_cache[post_data] = re.search(SUSPICIOUS_HTTP_REQUEST_REGEX, urllib.unquote(post_data)) is not None
                                if result:
                                    trail = "%s(%s \(%s %s\))" % (host, path, method, post_data.strip())
                                    log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, PROTO.TCP, TRAIL.URL, trail, "suspicious http request", "(heuristic)"))
                                    return

                        if '.' in path:
                            _ = urlparse.urlparse("http://%s" % url)  # dummy scheme
                            filename = _.path.split('/')[-1]
                            name, extension = os.path.splitext(filename)
                            if extension and extension in SUSPICIOUS_DIRECT_DOWNLOAD_EXTENSIONS and not any(_ in path for _ in WHITELIST_DIRECT_DOWNLOAD_KEYWORDS) and '.'.join(host.split('.')[-2:]) not in WHITELIST and not _.query and len(name) < 10:
                                trail = "%s(%s)" % (host, path)
                                log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, PROTO.TCP, TRAIL.URL, trail, "direct %s download (suspicious)" % extension, "(heuristic)"))
                            elif filename in SUSPICIOUS_FILENAMES:
                                trail = "%s(%s)" % (host, path)
                                log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, PROTO.TCP, TRAIL.URL, trail, "suspicious page", "(heuristic)"))

        elif protocol == socket.IPPROTO_UDP:  # UDP
            _ = ip_data[iph_length:iph_length + 4]
            if len(_) < 4:
                return

            src_port, dst_port = struct.unpack("!HH", _)

            if src_port != 53:
                if dst_ip in trails:
                    log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, PROTO.UDP, TRAIL.IP, dst_ip, trails[dst_ip][0], trails[dst_ip][1]))
                elif src_ip in trails:
                    log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, PROTO.UDP, TRAIL.IP, src_ip, trails[src_ip][0], trails[src_ip][1]))

            if dst_port == 53 or src_port == 53:
                dns_data = ip_data[iph_length + 8:]

                # Reference: http://www.ccs.neu.edu/home/amislove/teaching/cs4700/fall09/handouts/project1-primer.pdf
                if len(dns_data) > 6:
                    qdcount = struct.unpack("!H", dns_data[4:6])[0]
                    if qdcount > 0:
                        offset = 12
                        query = ""

                        while len(dns_data) > offset:
                            length = ord(dns_data[offset])
                            if not length:
                                query = query[:-1]
                                break
                            query += dns_data[offset + 1:offset + length + 1] + '.'
                            offset += length + 1

                        if ' ' in query or '.' not in query or any(query.endswith(_) for _ in IGNORE_DNS_QUERY_SUFFIXES):
                            return

                        if ord(dns_data[2]) == 0x01:  # standard query
                            type_, class_ = struct.unpack("!HH", dns_data[offset + 1:offset + 5])

                            # Reference: http://en.wikipedia.org/wiki/List_of_DNS_record_types
                            if type_ not in (12, 28) and class_ == 1:  # Type not in (PTR, AAAA), Class IN
                                _check_domain(query, sec, usec, src_ip, src_port, dst_ip, dst_port, PROTO.UDP)

                        elif config.USE_HEURISTICS:
                            if (ord(dns_data[2]) & 0x80) and (ord(dns_data[3]) == 0x83):  # standard response, recursion available, no such name
                                parts = query.split('.')
                                if not (len(parts) > 4 and all(_.isdigit() and int(_) < 256 for _ in parts[:4])):  # generic check for DNSBL IP lookups
                                    for _ in filter(None, (query, "*.%s" % '.'.join(parts[-2:]) if query.count('.') > 1 else None)):
                                        if _ not in NO_SUCH_NAME_COUNTERS or NO_SUCH_NAME_COUNTERS[_][0] != sec / 3600:
                                            NO_SUCH_NAME_COUNTERS[_] = [sec / 3600, 1, set()]
                                        else:
                                            NO_SUCH_NAME_COUNTERS[_][1] += 1
                                            NO_SUCH_NAME_COUNTERS[_][2].add(query)

                                            if NO_SUCH_NAME_COUNTERS[_][1] > NO_SUCH_NAME_PER_HOUR_THRESHOLD and _ not in WHITELIST and '.'.join(_.split('.')[-2:]) not in WHITELIST:
                                                if _.startswith("*."):
                                                    log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, PROTO.UDP, TRAIL.DNS, "%s%s" % ("(%s)" % ','.join(item.replace(_[1:], "") for item in NO_SUCH_NAME_COUNTERS[_][2]), _[1:]), "excessive no such domain name (suspicious)", "(heuristic)"))
                                                    for item in NO_SUCH_NAME_COUNTERS[_][2]:
                                                        try:
                                                            del NO_SUCH_NAME_COUNTERS[item]
                                                        except KeyError:
                                                            pass
                                                else:
                                                    log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, PROTO.UDP, TRAIL.DNS, _, "excessive no such domain name (suspicious)", "(heuristic)"))

                                                try:
                                                    del NO_SUCH_NAME_COUNTERS[_]
                                                except KeyError:
                                                    pass

                                                break

                                    # Reference: https://github.com/exp0se/dga_detector
                                    for part in parts:
                                        if part:
                                            consonants = re.findall("(?i)[bcdfghjklmnpqrstvwxyz]", part)
                                            if len(consonants) > SUSPICIOUS_DOMAIN_CONSONANT_THRESHOLD:
                                                log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, PROTO.UDP, TRAIL.DNS, query, "high consonant no such domain name (suspicious)", "(heuristic)"))
                                                break

                                            probabilities = (float(part.count(c)) / len(part) for c in set(_ for _ in part))
                                            entropy = -sum(p * math.log(p) / math.log(2.0) for p in probabilities)
                                            if entropy > SUSPICIOUS_DOMAIN_ENTROPY_THRESHOLD:
                                                log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, PROTO.UDP, TRAIL.DNS, query, "high entropy no such domain name (suspicious)", "(heuristic)"))
                                                break

        elif protocol in IPPROTO_LUT:  # non-TCP/UDP (e.g. ICMP)
            if protocol == socket.IPPROTO_ICMP:
                if ord(ip_data[iph_length]) != 8:  # Non-echo request
                    return

            if dst_ip in trails:
                log_event((sec, usec, src_ip, '-', dst_ip, '-', IPPROTO_LUT[protocol], TRAIL.IP, dst_ip, trails[dst_ip][0], trails[dst_ip][1]))
            elif src_ip in trails:
                log_event((sec, usec, src_ip, '-', dst_ip, '-', IPPROTO_LUT[protocol], TRAIL.IP, src_ip, trails[src_ip][0], trails[src_ip][1]))

    except struct.error:
        pass

    except Exception:
        if config.SHOW_DEBUG:
            traceback.print_exc()

def init():
    """
    Performs sensor initialization
    """

    global _multiprocessing

    if config.USE_MULTIPROCESSING:
        try:
            import multiprocessing

            if multiprocessing.cpu_count() > 1:
                _multiprocessing = multiprocessing
        except (ImportError, OSError, NotImplementedError):
            pass

    def update_timer():
        _ = update_trails(server=config.UPDATE_SERVER)

        update_ipcat()

        if _:
            trails.clear()
            trails.update(_)
        elif not trails:
            trails.update(load_trails())

        thread = threading.Timer(config.UPDATE_PERIOD, update_timer)
        thread.daemon = True
        thread.start()

    create_log_directory()
    get_error_log_handle()

    check_memory()
    update_timer()

    if check_sudo() is False:
        exit("[!] please run '%s' with sudo/Administrator privileges" % __file__)

    interfaces = set(_.strip() for _ in config.MONITOR_INTERFACE.split(','))

    if (config.MONITOR_INTERFACE or "").lower() == "any":
        if subprocess.mswindows or "any" not in pcapy.findalldevs():
            print("[x] virtual interface 'any' missing. Replacing it with all interface names")
            interfaces = pcapy.findalldevs()
        else:
            print("[?] in case of any problems with packet capture on virtual interface 'any', please put all monitoring interfaces to promiscuous mode manually (e.g. 'sudo ifconfig eth0 promisc')")

    for interface in interfaces:
        if interface.lower() != "any" and interface not in pcapy.findalldevs():
            hint = "[?] available interfaces: '%s'" % ",".join(pcapy.findalldevs())
            exit("[!] interface '%s' not found\n%s" % (interface, hint))

        print("[i] opening interface '%s'" % interface)
        try:
            _caps.append(pcapy.open_live(interface, SNAP_LEN, True, CAPTURE_TIMEOUT))
        except (socket.error, pcapy.PcapError):
            if "permitted" in str(sys.exc_info()[1]):
                exit("[!] please run '%s' with sudo/Administrator privileges" % __file__)
            elif "No such device" in str(sys.exc_info()[1]):
                exit("[!] no such device '%s'" % interface)
            else:
                raise

    if config.LOG_SERVER and not len(config.LOG_SERVER.split(':')) == 2:
        exit("[!] invalid configuration value for 'LOG_SERVER' ('%s')" % config.LOG_SERVER)

    if config.CAPTURE_FILTER:
        print("[i] setting filter '%s'" % config.CAPTURE_FILTER)
        for _cap in _caps:
            _cap.setfilter(config.CAPTURE_FILTER)

    if _multiprocessing:
        _init_multiprocessing()

    if not subprocess.mswindows:
        try:
            try:
                mod = int(subprocess.check_output("grep -c ^processor /proc/cpuinfo", stderr=subprocess.STDOUT, shell=True).strip())
                used = subprocess.check_output("for pid in $(ps aux | grep python | grep sensor.py | grep -E -o 'root[ ]*[0-9]*' | tr -d '[:alpha:] '); do schedtool $pid; done | grep -E -o 'AFFINITY .*' | cut -d ' ' -f 2 | grep -v 0xf", stderr=subprocess.STDOUT, shell=True).strip().split('\n')
                max_used = max(int(_, 16) for _ in used)
                affinity = max(1, (max_used << 1) % 2 ** mod)
            except:
                affinity = 1
            p = subprocess.Popen("schedtool -n -2 -M 2 -p 10 -a 0x%02x %d" % (affinity, os.getpid()), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            _, stderr = p.communicate()
            if "not found" in stderr:
                msg, _ = "[?] please install 'schedtool' for better CPU scheduling", platform.linux_distribution()[0].lower()
                for distro, install in {("fedora", "centos"): "sudo yum install schedtool", ("debian", "ubuntu"): "sudo apt-get install schedtool"}.items():
                    if _ in distro:
                        msg += " (e.g. '%s')" % install
                        break
                print(msg)
        except:
            pass

def _init_multiprocessing():
    """
    Inits worker processes used in multiprocessing mode
    """

    global _buffer
    global _n

    if _multiprocessing:
        try:
            _buffer = mmap.mmap(-1, config.CAPTURE_BUFFER)  # http://www.alexonlinux.com/direct-io-in-python

            _ = "\x00" * MMAP_ZFILL_CHUNK_LENGTH
            for i in xrange(config.CAPTURE_BUFFER / MMAP_ZFILL_CHUNK_LENGTH):
                _buffer.write(_)
            _buffer.seek(0)
        except:
            exit("[!] unable to allocate network capture buffer. Please adjust value of 'CAPTURE_BUFFER'")

        print("[i] creating %d more processes (%d CPU cores detected)" % (_multiprocessing.cpu_count() - 1, _multiprocessing.cpu_count()))
        _n = _multiprocessing.Value('L', lock=False)

        for i in xrange(_multiprocessing.cpu_count() - 1):
            process = _multiprocessing.Process(target=worker, name=str(i), args=(_buffer, _n, i, _multiprocessing.cpu_count() - 1, _process_ip))
            process.daemon = True
            process.start()

def monitor():
    """
    Sniffs/monitors given capturing interface
    """

    print("[o] running...")

    def packet_handler(datalink, header, packet):
        global _count

        ip_offset = None
        dlt_offset = DLT_OFFSETS[datalink]

        try:
            if datalink == pcapy.DLT_RAW:
                ip_offset = dlt_offset
            elif datalink == pcapy.DLT_PPP:
                if ord(packet[2]) == 0 and ord(packet[3]) == 0x21:  # IPv4
                    ip_offset = dlt_offset
            else:
                if ord(packet[dlt_offset - 2]) == 8 and ord(packet[dlt_offset - 1]) == 0:  # IPv4
                    ip_offset = dlt_offset

                elif ord(packet[dlt_offset - 2]) == 0x81 and ord(packet[dlt_offset - 1]) == 0:  # VLAN
                    if ord(packet[dlt_offset + 2]) == 8 and ord(packet[dlt_offset + 3]) == 0: # IPv4
                        ip_offset = dlt_offset + 4

        except IndexError:
            pass

        if ip_offset is None:
            return

        data = packet[ip_offset:]

        try:
            sec, usec = header.getts()
            if _multiprocessing:
                if _locks.count:
                    _locks.count.acquire()

                write_block(_buffer, _count, struct.pack("=II", sec, usec) + data)
                _count += 1
                _n.value = _count

                if _locks.count:
                    _locks.count.release()
            else:
                _process_ip(data, sec, usec)
        except socket.timeout:
            pass

    try:
        def _(_cap):
            datalink = _cap.datalink()
            while True:
                try:
                    (header, packet) = _cap.next()
                    packet_handler(datalink, header, packet)
                except (pcapy.PcapError, socket.timeout):
                    pass

        if len(_caps) > 1:
            if _multiprocessing:
                _locks.count = threading.Lock()
            _locks.connect_sec = threading.Lock()

        for _cap in _caps:
            threading.Thread(target=_, args=(_cap,)).start()

        while threading.activeCount() > 1:
            time.sleep(1)
    except SystemError, ex:
        if "error return without" in str(ex):
            print("\r[x] Ctrl-C pressed")
        else:
            raise
    except KeyboardInterrupt:
        print("\r[x] Ctrl-C pressed")
    finally:
        if _multiprocessing:
            for _ in xrange(_multiprocessing.cpu_count() - 1):
                write_block(_buffer, _n.value, "", BLOCK_MARKER.END)
                _n.value = _n.value + 1
            while _multiprocessing.active_children():
                time.sleep(REGULAR_SENSOR_SLEEP_TIME)

def main():
    print("%s (sensor) #v%s\n" % (NAME, VERSION))

    parser = optparse.OptionParser(version=VERSION)
    parser.add_option("-c", dest="config_file", default=CONFIG_FILE, help="Configuration file (default: '%s')" % os.path.split(CONFIG_FILE)[-1])
    options, _ = parser.parse_args()

    if not check_sudo():
        exit("[!] please run '%s' with sudo/Administrator privileges" % __file__)

    read_config(options.config_file)

    try:
        init()
        monitor()
    except KeyboardInterrupt:
        print("\r[x] stopping (Ctrl-C pressed)")

if __name__ == "__main__":
    try:
        main()
    except Exception:
        msg = "\r[!] unhandled exception occurred ('%s')" % sys.exc_info()[1]
        msg += "\n[x] please report the following details at 'https://github.com/stamparm/maltrail/issues':\n---\n'%s'\n---" % traceback.format_exc()
        log_error("\n\n%s" % msg.replace("\r", ""))

        print(msg)

    os._exit(0)


text:
package dependencies

// Dependencies is used as abstraction for the commands used from the operating system
type Dependencies interface {
	// RunOrFail runs a command and panics, if it fails
	RunOrFail(cmd string, args ...string)
	// Run runs a command
	Run(cmd string, args ...string) error
	// RunQuietlyAndIgnore runs a command quietly and ignores errors
	RunQuietlyAndIgnore(cmd string, args ...string)
}


text:
module Fog
  module Storage
    class Rackspace
      class Real
        # Copy object
        #
        # ==== Parameters
        # * source_container_name<~String> - Name of source bucket
        # * source_object_name<~String> - Name of source object
        # * target_container_name<~String> - Name of bucket to create copy in
        # * target_object_name<~String> - Name for new copy of object
        # * options<~Hash> - Additional headers
        # @raise [Fog::Storage::Rackspace::NotFound] - HTTP 404
        # @raise [Fog::Storage::Rackspace::BadRequest] - HTTP 400
        # @raise [Fog::Storage::Rackspace::InternalServerError] - HTTP 500
        # @raise [Fog::Storage::Rackspace::ServiceError]
        def copy_object(source_container_name, source_object_name, target_container_name, target_object_name, options={})
          headers = { 'X-Copy-From' => "/#{source_container_name}/#{source_object_name}" }.merge(options)
          request({
            :expects  => 201,
            :headers  => headers,
            :method   => 'PUT',
            :path     => "#{Fog::Rackspace.escape(target_container_name)}/#{Fog::Rackspace.escape(target_object_name)}"
          })
        end
      end
    end
  end
end


text:
<?php

namespace Backend\Modules\Search\Actions;

use Backend\Core\Engine\Base\Action;
use Backend\Core\Engine\Authentication as BackendAuthentication;
use Backend\Core\Engine\DataGridDatabase as BackendDataGridDatabase;
use Backend\Core\Language\Language as BL;
use Backend\Core\Engine\Model as BackendModel;
use Backend\Modules\Search\Engine\Model as BackendSearchModel;

/**
 * This is the synonyms-action, it will display the overview of search synonyms
 */
class Synonyms extends Action
{
    public function execute(): void
    {
        parent::execute();
        $this->showDataGrid();
        $this->display();
    }

    private function showDataGrid(): void
    {
        $dataGrid = new BackendDataGridDatabase(BackendSearchModel::QUERY_DATAGRID_BROWSE_SYNONYMS, [BL::getWorkingLanguage()]);
        $dataGrid->setSortingColumns(['term'], 'term');
        $dataGrid->setColumnFunction('str_replace', [',', ', ', '[synonym]'], 'synonym', true);

        if (BackendAuthentication::isAllowedAction('EditSynonym')) {
            $editUrl = BackendModel::createUrlForAction('EditSynonym') . '&id=[id]';
            $dataGrid->setColumnURL('term', $editUrl);
            $dataGrid->addColumn('edit', null, BL::lbl('Edit'), $editUrl, BL::lbl('Edit'));
        }

        $this->template->assign('dataGrid', $dataGrid->getContent());
    }
}


text:
'''
This file contains helper functions used at the time that various apps are loaded
'''

import sys

def isPreliminaryRun():
    '''
    Check the arguments passed at runtime to ensure that this
    program is not being run to perform migrations or load data.
    '''
    prelim_params = ['loaddata', 'makemigrations', 'migrate']
    for param in prelim_params:
        if param in sys.argv:
            return True
    return False


text:
package com.google.errorprone.bugpatterns;

/**
 * @author pepstein@google.com (Peter Epstein)
 */
@SuppressWarnings("fallthrough")
public class FallthroughSuppressionPositiveCases {

  @SuppressWarnings("fallthrough")
  public void suppressedMethod1a() {}

  @SuppressWarnings(value = "fallthrough")
  public void suppressedMethod1b() {}

  @SuppressWarnings({"fallthrough"})
  public void suppressedMethod1c() {}

  @java.lang.SuppressWarnings("fallthrough")
  public void suppressedMethod1d() {}

  @SuppressWarnings({"fallthrough", "unchecked"})
  public void suppressedMethod2() {}

  @SuppressWarnings({"varargs", "fallthrough", "unchecked"})
  public void suppressedMethod3() {}
}


text:
namespace HiLaarIsch.Domain
{
    public interface IEntity
    {
        int Id { get; set; }
    }
}


text:
let validatorLib = require('validator');
let phoneLib = require('phone');

let db = require('./db.js');
let config = require('../configs.js');

module.exports.validateFields = function (fields, validateAll = false) {
    let name = fields.name;
    let username = fields.username;
    let phoneNumber = fields.phoneNumber;

    let promises = [];

    // Fields should usually be skipped if they are `undefined`, but still
    // validated if they are other falsy values, like `''`.
    if (name !== undefined || validateAll) {
        promises.push(validateNameFormat(name));
    }
    if (username !== undefined || validateAll) {
        promises.push(validateUsernameFormat(username));
        promises.push(validateUsernameUntaken(username));
    }
    if (phoneNumber !== undefined || validateAll) {
        promises.push(validatePhoneNumberFormat(phoneNumber));
        promises.push(validatePhoneNumberUntaken(phoneNumber));
    }

    return Promise.all(promises);
}

class FieldIssue {
    constructor(field, issue) {
        this.field = field;
        this.issue = issue;
    }
}

function validateNameFormat(name) {
    if (!name) return Promise.reject(
        new FieldIssue('name', 'Names can\'t be empty.')
    );
    return Promise.resolve(name);
}

let usernameTakenFieldIssue = new FieldIssue(
    'username',
    'That username is already taken.'
);

function validateUsernameFormat(username) {
    if (!username || username.length < config.signUp.usernameMinimumLength)
        return Promise.reject(new FieldIssue(
            'username',
            'Usernames must be at least '
                + config.signUp.usernameMinimumLength
                + ' characters.'
        ));
    if (!config.signUp.usernameMatchRegExp.test(username))
        return Promise.reject(new FieldIssue(
            'username',
            'Usernames must only contain letters and numbers.'
        ));
    return Promise.resolve(username);
}

function validateUsernameUntaken(username) {
    if (!username) return validateUsernameFormat(username);

    return db.checkChildKeyUntaken('users', 'username', username).then(
        (untaken) => {
            if (!untaken) return Promise.reject(usernameTakenFieldIssue);
        }
    );
}

module.exports.validateUsernameTaken = function (username) {
    return validateUsernameUntaken(username).then(() => {
        throw new FieldIssue(
            'username',
            'That username is not registered.'
        )
    }).catch((fieldIssue) => {
        if (fieldIssue === usernameTakenFieldIssue) return;
        else throw fieldIssue;
    });
}

let phoneNumberTakenFieldIssue = new FieldIssue(
    'phoneNumber',
    'That phone number is already taken.'
);

function validatePhoneNumberFormat(phoneNumber) {
    if (!phoneNumber || !validatorLib.isMobilePhone(phoneNumber, 'any'))
        return Promise.reject(new FieldIssue(
            'phoneNumber',
            'Phone numbers must be in a correct format.'
        ));
    return Promise.resolve(phoneNumber);
}

module.exports.validatePhoneNumberTaken = function (phoneNumber) {
    return validatePhoneNumberUntaken(phoneLib(phoneNumber)[0]).then(() => {
        throw new FieldIssue(
            'phoneNumber',
            'That phone number is not registered.'
        )
    }).catch((fieldIssue) => {
        if (fieldIssue === phoneNumberTakenFieldIssue) return;
        else throw fieldIssue;
    });
}

function validatePhoneNumberUntaken(phoneNumber) {
    if (!phoneNumber) return validatePhoneNumberFormat(phoneNumber);

    return db.checkChildKeyUntaken('users', 'phoneNumber', phoneNumber).then(
        (untaken) => {
            if (!untaken) return Promise.reject(phoneNumberTakenFieldIssue);
        }
    );
}

module.exports.normalizeFields = function (fields) {
    return Promise.resolve(Object.assign({}, fields, {
        phoneNumber:
            fields.phoneNumber ? phoneLib(fields.phoneNumber)[0] : undefined,
        username:
            fields.username ? fields.username.toLowerCase() : undefined,
    }));
}

module.exports.startSignUp = function (fields) {
    return db.signInWithPhoneNumber(fields.phoneNumber);
};

module.exports.endSignUp = function (fields) {
    return db.writeWithCurrentUID('users', fields);
}

module.exports.doLogIn = function (phoneNumber) {
    return db.signInWithPhoneNumber(phoneNumber);
};

module.exports.logOut = function () {
    return db.logOut();
}


text:
package org.apache.activemq.artemis.rest.test;

import org.apache.activemq.artemis.rest.util.Constants;
import org.jboss.resteasy.client.ClientRequest;
import org.jboss.resteasy.client.ClientResponse;
import org.jboss.resteasy.spi.Link;
import org.junit.Assert;

public final class Util {

   private Util() {
      // Utility class
   }

   static ClientResponse head(ClientRequest request) throws Exception {
      ClientResponse<?> response = request.head();
      response.releaseConnection();
      Assert.assertEquals(200, response.getStatus());
      return response;
   }

   static String getUrlPath(String queueName) {
      return Constants.PATH_FOR_QUEUES + "/" + queueName;
   }

   public static ClientResponse setAutoAck(Link link, boolean ack) throws Exception {
      ClientResponse response;
      response = link.request().formParameter("autoAck", Boolean.toString(ack)).post();
      response.releaseConnection();
      Assert.assertEquals(201, response.getStatus());
      return response;
   }
}