text:
// Copyright 2018 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.skylarkbuildapi.android;

import com.google.devtools.build.lib.skylarkinterface.SkylarkModule;

/**
 * Wraps common tools and settings used for working with Android assets, resources, and manifests.
 */
@SkylarkModule(
    name = "AndroidDataContext",
    doc =
        "Wraps common tools and settings used for working with Android assets, resources, and"
            + " manifests")
public interface AndroidDataContextApi {}


text:
'use strict';

module.exports = require.requireActual('containsRelayQueryRootCall');

redpajama_set_name: RedPajamaGithub


text:
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<title>websocket::stream::set_option (2 of 2 overloads)</title>
<link rel="stylesheet" href="../../../../../../../../doc/src/boostbook.css" type="text/css">
<meta name="generator" content="DocBook XSL Stylesheets V1.79.1">
<link rel="home" href="../../../../index.html" title="Chapter 1. Boost.Beast">
<link rel="up" href="../set_option.html" title="websocket::stream::set_option">
<link rel="prev" href="overload1.html" title="websocket::stream::set_option (1 of 2 overloads)">
<link rel="next" href="../auto_fragment.html" title="websocket::stream::auto_fragment">

Home
Libraries
People
FAQ
More

websocket::stream::set_option
          (2 of 2 overloads)

            Set the permessage-deflate extension options.

<h7>
            Synopsis
          </h7>
void
set_option(
    permessage_deflate const& o);

<h7>
            Description
          </h7><h7>
            Exceptions
          </h7>

<colgroup>
<col>
<col>
</colgroup>

                      Type

                      Thrown On

                      invalid_argument

                      if deflateSupported
                      == false,
                      and either client_enable
                      or server_enable
                      is true.

Copyright © 2016-2019 Vinnie
      Falco

        Distributed under the Boost Software License, Version 1.0. (See accompanying
        file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)


text:
import datetime
import re

from saml2.mdstore import MetadataStore
from saml2.mdstore import destinations
from saml2.mdstore import name

from saml2 import md
from saml2 import sigver
from saml2 import BINDING_SOAP
from saml2 import BINDING_HTTP_REDIRECT
from saml2 import BINDING_HTTP_POST
from saml2 import BINDING_HTTP_ARTIFACT
from saml2 import saml
from saml2 import config
from saml2.attribute_converter import ac_factory
from saml2.attribute_converter import d_to_local_name

from saml2.extension import mdui
from saml2.extension import idpdisc
from saml2.extension import dri
from saml2.extension import mdattr
from saml2.extension import ui
from saml2.s_utils import UnknownPrincipal
import xmldsig
import xmlenc

from pathutils import full_path

sec_config = config.Config()
#sec_config.xmlsec_binary = sigver.get_xmlsec_binary(["/opt/local/bin"])

ONTS = {
    saml.NAMESPACE: saml,
    mdui.NAMESPACE: mdui,
    mdattr.NAMESPACE: mdattr,
    dri.NAMESPACE: dri,
    ui.NAMESPACE: ui,
    idpdisc.NAMESPACE: idpdisc,
    md.NAMESPACE: md,
    xmldsig.NAMESPACE: xmldsig,
    xmlenc.NAMESPACE: xmlenc
}

ATTRCONV = ac_factory(full_path("attributemaps"))

METADATACONF = {
    "1": {
        "local": [full_path("swamid-1.0.xml")]
    },
    "2": {
        "local": [full_path("InCommon-metadata.xml")]
    },
    "3": {
        "local": [full_path("extended.xml")]
    },
    "7": {
        "local": [full_path("metadata_sp_1.xml"),
                  full_path("InCommon-metadata.xml")],
        "remote": [
            {"url": "https://kalmar2.org/simplesaml/module.php/aggregator/?id=kalmarcentral2&set=saml2",
             "cert": full_path("kalmar2.pem")}]
    },
    "4": {
        "local": [full_path("metadata_example.xml")]
    },
    "5": {
        "local": [full_path("metadata.aaitest.xml")]
    },
    "8": {
        "mdfile": [full_path("swamid.md")]
    }
}

def _eq(l1, l2):
    return set(l1) == set(l2)

def _fix_valid_until(xmlstring):
    new_date = datetime.datetime.now() + datetime.timedelta(days=1)
    new_date = new_date.strftime("%Y-%m-%dT%H:%M:%SZ")
    return re.sub(r' validUntil=".*?"', ' validUntil="%s"' % new_date,
                  xmlstring)

def test_swami_1():
    UMU_IDP = 'https://idp.umu.se/saml2/idp/metadata.php'
    mds = MetadataStore(list(ONTS.values()), ATTRCONV, sec_config,
                        disable_ssl_certificate_validation=True)

    mds.imp(METADATACONF["1"])
    assert len(mds) == 1  # One source
    idps = mds.with_descriptor("idpsso")
    assert list(idps.keys())
    idpsso = mds.single_sign_on_service(UMU_IDP)
    assert len(idpsso) == 1
    assert destinations(idpsso) == [
        'https://idp.umu.se/saml2/idp/SSOService.php']

    _name = name(mds[UMU_IDP])
    assert _name == 'Umeå University (SAML2)'
    certs = mds.certs(UMU_IDP, "idpsso", "signing")
    assert len(certs) == 1

    sps = mds.with_descriptor("spsso")
    assert len(sps) == 108

    wants = mds.attribute_requirement('https://connect8.sunet.se/shibboleth')
    lnamn = [d_to_local_name(mds.attrc, attr) for attr in wants["optional"]]
    assert _eq(lnamn, ['eduPersonPrincipalName', 'mail', 'givenName', 'sn',
                       'eduPersonScopedAffiliation'])

    wants = mds.attribute_requirement('https://beta.lobber.se/shibboleth')
    assert wants["required"] == []
    lnamn = [d_to_local_name(mds.attrc, attr) for attr in wants["optional"]]
    assert _eq(lnamn, ['eduPersonPrincipalName', 'mail', 'givenName', 'sn',
                       'eduPersonScopedAffiliation', 'eduPersonEntitlement'])

def test_incommon_1():
    mds = MetadataStore(list(ONTS.values()), ATTRCONV, sec_config,
                        disable_ssl_certificate_validation=True)

    mds.imp(METADATACONF["2"])

    print((mds.entities()))
    assert mds.entities() > 1700
    idps = mds.with_descriptor("idpsso")
    print((list(idps.keys())))
    assert len(idps) > 300  # ~ 18%
    try:
        _ = mds.single_sign_on_service('urn:mace:incommon:uiuc.edu')
    except UnknownPrincipal:
        pass

    idpsso = mds.single_sign_on_service('urn:mace:incommon:alaska.edu')
    assert len(idpsso) == 1
    print(idpsso)
    assert destinations(idpsso) == [
        'https://idp.alaska.edu/idp/profile/SAML2/Redirect/SSO']

    sps = mds.with_descriptor("spsso")

    acs_sp = []
    for nam, desc in list(sps.items()):
        if "attribute_consuming_service" in desc:
            acs_sp.append(nam)

    assert len(acs_sp) == 0

    # Look for attribute authorities
    aas = mds.with_descriptor("attribute_authority")

    print((list(aas.keys())))
    assert len(aas) == 180

def test_ext_2():
    mds = MetadataStore(list(ONTS.values()), ATTRCONV, sec_config,
                        disable_ssl_certificate_validation=True)

    mds.imp(METADATACONF["3"])
    # No specific binding defined

    ents = mds.with_descriptor("spsso")
    for binding in [BINDING_SOAP, BINDING_HTTP_POST, BINDING_HTTP_ARTIFACT,
                    BINDING_HTTP_REDIRECT]:
        assert mds.single_logout_service(list(ents.keys())[0], binding, "spsso")

def test_example():
    mds = MetadataStore(list(ONTS.values()), ATTRCONV, sec_config,
                        disable_ssl_certificate_validation=True)

    mds.imp(METADATACONF["4"])
    assert len(list(mds.keys())) == 1
    idps = mds.with_descriptor("idpsso")

    assert list(idps.keys()) == [
        'http://xenosmilus.umdc.umu.se/simplesaml/saml2/idp/metadata.php']
    certs = mds.certs(
        'http://xenosmilus.umdc.umu.se/simplesaml/saml2/idp/metadata.php',
        "idpsso", "signing")
    assert len(certs) == 1

def test_switch_1():
    mds = MetadataStore(list(ONTS.values()), ATTRCONV, sec_config,
                        disable_ssl_certificate_validation=True)

    mds.imp(METADATACONF["5"])
    assert len(list(mds.keys())) > 160
    idps = mds.with_descriptor("idpsso")
    print((list(idps.keys())))
    idpsso = mds.single_sign_on_service(
        'https://aai-demo-idp.switch.ch/idp/shibboleth')
    assert len(idpsso) == 1
    print(idpsso)
    assert destinations(idpsso) == [
        'https://aai-demo-idp.switch.ch/idp/profile/SAML2/Redirect/SSO']
    assert len(idps) > 30
    aas = mds.with_descriptor("attribute_authority")
    print((list(aas.keys())))
    aad = aas['https://aai-demo-idp.switch.ch/idp/shibboleth']
    print((list(aad.keys())))
    assert len(aad["attribute_authority_descriptor"]) == 1
    assert len(aad["idpsso_descriptor"]) == 1

    sps = mds.with_descriptor("spsso")
    dual = [eid for eid, ent in list(idps.items()) if eid in sps]
    print((len(dual)))
    assert len(dual) == 0

def test_metadata_file():
    sec_config.xmlsec_binary = sigver.get_xmlsec_binary(["/opt/local/bin"])
    mds = MetadataStore(list(ONTS.values()), ATTRCONV, sec_config,
                        disable_ssl_certificate_validation=True)

    mds.imp(METADATACONF["8"])
    print((len(list(mds.keys()))))
    assert len(list(mds.keys())) == 560

if __name__ == "__main__":
    test_metadata_file()


text:
require "spec_helper"
require "hamster/hash"

describe Hamster::Hash do
  describe "#invert" do
    let(:hash) { Hamster.hash(a: 3, b: 2, c: 1) }

    it "uses the existing keys as values and values as keys" do
      hash.invert.should eql(Hamster.hash(3 => :a, 2 => :b, 1 => :c))
    end

    it "will select one key/value pair among multiple which have same value" do
      [Hamster.hash(1 => :a),
       Hamster.hash(1 => :b),
       Hamster.hash(1 => :c)].include?(Hamster.hash(a: 1, b: 1, c: 1).invert).should == true
    end

    it "doesn't change the original Hash" do
      hash.invert
      hash.should eql(Hamster.hash(a: 3, b: 2, c: 1))
    end

    context "from a subclass of Hash" do
      it "returns an instance of the subclass" do
        subclass = Class.new(Hamster::Hash)
        instance = subclass.new(a: 1, b: 2)
        instance.invert.class.should be(subclass)
      end
    end
  end
end


text:
require 'rubygems'
require 'bundler/setup'

require 'action_mailer'
require 'actionmailer-maildir'
require 'rspec'
require 'mail'
require 'tape'

require 'fakefs/spec_helpers'

class Notifier < ActionMailer::Base
  default :from        => 'sender@example.com',
          :return_path => 'return@example.com'

  def hello(opts = {})
    defaults = {:subject => 'Hello World', :to => "recipient@example.com"}

    mail(defaults.merge(opts)) do |format|
      format.text { 'Hello World' }
    end
  end
end

class Mail::Message
  def with_inspect_stub(str = "email")
    stub!(:inspect).and_return(str)
    self
  end
end

  RSpec.configure do |config|
  config.mock_with :rspec
end

ENV['TMPDIR'] = File.expand_path '../../tmp', __FILE__


text:
package com.koch.ambeth.util.appendable;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.WritableByteChannel;

import com.koch.ambeth.util.exception.RuntimeExceptionUtil;

public class ByteBufferAppendable implements IAppendable {
	private final byte[] byteArray1 = new byte[1];

	private final byte[] byteArray2 = new byte[2];

	private final byte[] byteArray3 = new byte[3];

	private final byte[] byteArray4 = new byte[4];

	protected final ByteBuffer buffer;

	protected final WritableByteChannel targetChannel;

	public ByteBufferAppendable(WritableByteChannel targetChannel, ByteBuffer buffer) {
		this.buffer = buffer;
		this.targetChannel = targetChannel;
	}

	@Override
	public IAppendable append(CharSequence value) {
		for (int writePos = 0, size = value.length(); writePos < size; writePos++) {
			append(value.charAt(writePos));
		}
		return this;
	}

	@Override
	public IAppendable append(char value) {
		byte[] byteValue = encodeUTF8(value);
		append(byteValue);
		return this;
	}

	protected void append(byte[] value) {
		ByteBuffer byteBuffer = buffer;

		if (byteBuffer.remaining() < value.length) {
			// buffer is to full. Now we write it to the channel
			byteBuffer.flip();
			try {
				while (byteBuffer.hasRemaining()) {
					targetChannel.write(byteBuffer);
				}
			}
			catch (IOException e) {
				throw RuntimeExceptionUtil.mask(e);
			}
			// clear the buffer to activate the "has remaining" state again
			byteBuffer.clear();
		}

		byteBuffer.put(value);
	}

	protected byte[] encodeUTF8(char value) {
		if (value < 128) {
			byte[] output = byteArray1;
			output[0] = (byte) value;
			return output;
		}
		if (value < 0x800) {
			byte[] output = byteArray2;
			output[0] = (byte) (((value & 0x7c0) >> 6) | 0xc0);
			output[1] = (byte) ((value & 0x3f) | 0x80);
			return output;
		}
		if (value < 0xe000) {
			byte[] output = byteArray3;
			output[0] = (byte) (((value & 0xf000) >> 12) | 0xe0);
			output[1] = (byte) (((value & 0xfc) >> 6) | 0x80);
			output[2] = (byte) ((value & 0x3f) | 0x80);
			return output;
		}

		byte[] output = byteArray4;
		output[0] = (byte) (((value & 0x1c0000) >> 18) | 0xf0);
		output[1] = (byte) (((value & 0x3f0) >> 12) | 0x80);
		output[2] = (byte) (((value & 0xfc) >> 6) | 0x80);
		output[3] = (byte) ((value & 0x3f) | 0x80);
		return output;
	}
}


text:
//-----------------------------------------------------------------------------
//
//	Indicator.h
//
//	Implementation of the Z-Wave COMMAND_CLASS_INDICATOR
//
//	Copyright (c) 2010 Mal Lansell <openzwave@lansell.org>
//
//	SOFTWARE NOTICE AND LICENSE
//
//	This file is part of OpenZWave.
//
//	OpenZWave is free software: you can redistribute it and/or modify
//	it under the terms of the GNU Lesser General Public License as published
//	by the Free Software Foundation, either version 3 of the License,
//	or (at your option) any later version.
//
//	OpenZWave is distributed in the hope that it will be useful,
//	but WITHOUT ANY WARRANTY; without even the implied warranty of
//	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//	GNU Lesser General Public License for more details.
//
//	You should have received a copy of the GNU Lesser General Public License
//	along with OpenZWave.  If not, see <http://www.gnu.org/licenses/>.
//
//-----------------------------------------------------------------------------

#ifndef _Indicator_H
#define _Indicator_H

#include "command_classes/CommandClass.h"

namespace OpenZWave
{
	class ValueByte;

	/** \brief Implements COMMAND_CLASS_INDICATOR (0x87), a Z-Wave device command class.
	 */
	class Indicator: public CommandClass
	{
	public:
		static CommandClass* Create( uint32 const _homeId, uint8 const _nodeId ){ return new Indicator( _homeId, _nodeId ); }
		virtual ~Indicator(){}

		static uint8 const StaticGetCommandClassId(){ return 0x87; }
		static string const StaticGetCommandClassName(){ return "COMMAND_CLASS_INDICATOR"; }

		// From CommandClass
		virtual bool RequestState( uint32 const _requestFlags, uint8 const _instance, Driver::MsgQueue const _queue );
		virtual bool RequestValue( uint32 const _requestFlags, uint8 const _index, uint8 const _instance, Driver::MsgQueue const _queue );
		virtual uint8 const GetCommandClassId()const{ return StaticGetCommandClassId(); }
		virtual string const GetCommandClassName()const{ return StaticGetCommandClassName(); }
		virtual bool HandleMsg( uint8 const* _data, uint32 const _length, uint32 const _instance = 1 );
		virtual bool SetValue( Value const& _value );

	protected:
		virtual void CreateVars( uint8 const _instance );

	private:
		Indicator( uint32 const _homeId, uint8 const _nodeId ): CommandClass( _homeId, _nodeId ){}
	};

} // namespace OpenZWave

#endif


text:
module.exports = {
    id: function id(x) { /* use=moduleImport("reexport").getMember("exports").getMember("util2").getMember("id").getParameter(0) */
	return x;
    }
};


text:
using NBi.Core.ResultSet;
using NBi.Core.ResultSet.Filtering;
using NBi.Extensibility.Resolving;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace NBi.Core.Calculation.Predicate
{
    public class PredicateArgs
    {
        public virtual ColumnType ColumnType { get; set; }
        public virtual ComparerType ComparerType { get; set; }
        public virtual bool Not { get; set; }
    }

    public class ReferencePredicateArgs : PredicateArgs
    {
        public virtual IResolver Reference { get; set; }
    }

    public class CaseSensitivePredicateArgs : ReferencePredicateArgs
    {
        public virtual StringComparison StringComparison { get; set; }
    }

    public class SecondOperandPredicateArgs : ReferencePredicateArgs
    {
        public virtual object SecondOperand { get; set; }
    }

    public class CultureSensitivePredicateArgs : PredicateArgs
    {
        public virtual string Culture { get; set; }
    }
}


text:
// Copyright (c) Microsoft Open Technologies, Inc. All rights reserved. See License.txt in the project root for license information.

using System.Collections.Generic;
using System.Globalization;
using System.Web.Http.Controllers;
using System.Web.Http.Routing;

namespace System.Web.Http.ValueProviders.Providers
{
    public class RouteDataValueProvider : NameValuePairsValueProvider
    {
        public RouteDataValueProvider(HttpActionContext actionContext, CultureInfo culture)
            : base(GetRouteValues(actionContext.ControllerContext.RouteData), culture)
        {
        }

        internal static IEnumerable<KeyValuePair<string, string>> GetRouteValues(IHttpRouteData routeData)
        {
            foreach (KeyValuePair<string, object> pair in routeData.Values)
            {
                string value = (pair.Value == null) ? null : pair.Value.ToString();
                yield return new KeyValuePair<string, string>(pair.Key, value);
            }
        }
    }
}


text:
import re

from io import BytesIO

from six import iteritems
from six.moves.urllib.parse import urlencode

UNDERSCORE_RE = re.compile("-")

class TestApplication(object):
    _user_agent = "phi/testing"

    def __init__(self, application):
        self._application = application

    def request(
        self, url, content_type="text/plain", method="GET", body=None,
        headers={}, query_params={}, charset="utf-8"
    ):
        env = self._build_env(
            url=url,
            content_type=content_type,
            method=method,
            body=body,
            headers=headers,
            query_params=query_params,
            charset=charset
        )
        return self._make_request(env)

    def _make_request(self, env):
        response = {}

        def start_response(status, headers):
            response["status"] = status
            response["headers"] = dict(headers)

        body = self._application.handle_wsgi_request(env, start_response)
        body = list(body)
        body = "".join(body)
        response["body"] = body
        return response

    def _build_env(
        self, url, content_type, method, body, headers, query_params, charset
    ):
        stream = BytesIO()
        content_length = 0
        if body is not None:
            body = body.encode(charset)
            content_length = len(body)
            stream.write(body)
            stream.seek(0)

        env = {
            "wsgi.input": stream,
            "PATH_INFO": url,
            "REQUEST_METHOD": method,
            "CONTENT_TYPE": content_type,
            "CONTENT_LENGTH": str(content_length),
            "QUERY_STRING": urlencode(query_params)
        }

        self._update_headers(env, headers)

        return env

    def _update_headers(self, env, headers):
        env["HTTP_USER_AGENT"] = self._user_agent
        for key, value in iteritems(headers):
            header = UNDERSCORE_RE.sub("_", key)
            header = header.upper()
            header = "HTTP_" + header
            env[header] = str(value)


text:
"""WebRTC Demo

This module demonstrates the WebRTC API by implementing a simple video chat app.
"""

import cgi
import json
import logging
import os
import random
import threading

import jinja2
import webapp2
from google.appengine.api import memcache
from google.appengine.api import urlfetch

import analytics
import analytics_page
import compute_page
import constants

jinja_environment = jinja2.Environment(
    loader=jinja2.FileSystemLoader(os.path.dirname(__file__)))

def generate_random(length):
  word = ''
  for _ in range(length):
    word += random.choice('0123456789')
  return word

# HD is on by default for desktop Chrome, but not Android or Firefox (yet)
def get_hd_default(user_agent):
  if 'Android' in user_agent or not 'Chrome' in user_agent:
    return 'false'
  return 'true'

# iceServers will be filled in by the TURN HTTP request.
def make_pc_config(ice_transports):
  config = {
  'iceServers': [],
  'bundlePolicy': 'max-bundle',
  'rtcpMuxPolicy': 'require'
  };
  if ice_transports:
    config['iceTransports'] = ice_transports
  return config

def add_media_track_constraint(track_constraints, constraint_string):
  tokens = constraint_string.split(':')
  mandatory = True
  if len(tokens) == 2:
    # If specified, e.g. mandatory:minHeight=720, set mandatory appropriately.
    mandatory = (tokens[0] == 'mandatory')
  else:
    # Otherwise, default to mandatory, except for goog constraints, which
    # won't work in other browsers.
    mandatory = not tokens[0].startswith('goog')

  tokens = tokens[-1].split('=')
  if len(tokens) == 2:
    if mandatory:
      track_constraints['mandatory'][tokens[0]] = tokens[1]
    else:
      track_constraints['optional'].append({tokens[0]: tokens[1]})
  else:
    logging.error('Ignoring malformed constraint: ' + constraint_string)

def make_media_track_constraints(constraints_string):
  if not constraints_string or constraints_string.lower() == 'true':
    track_constraints = True
  elif constraints_string.lower() == 'false':
    track_constraints = False
  else:
    track_constraints = {'mandatory': {}, 'optional': []}
    for constraint_string in constraints_string.split(','):
      add_media_track_constraint(track_constraints, constraint_string)

  return track_constraints

def make_media_stream_constraints(audio, video, firefox_fake_device):
  stream_constraints = (
      {'audio': make_media_track_constraints(audio),
       'video': make_media_track_constraints(video)})
  if firefox_fake_device:
    stream_constraints['fake'] = True
  logging.info('Applying media constraints: ' + str(stream_constraints))
  return stream_constraints

def maybe_add_constraint(constraints, param, constraint):
  if (param.lower() == 'true'):
    constraints['optional'].append({constraint: True})
  elif (param.lower() == 'false'):
    constraints['optional'].append({constraint: False})

  return constraints

def make_pc_constraints(dtls, dscp, ipv6):
  constraints = {'optional': []};
  maybe_add_constraint(constraints, dtls, 'DtlsSrtpKeyAgreement')
  maybe_add_constraint(constraints, dscp, 'googDscp')
  maybe_add_constraint(constraints, ipv6, 'googIPv6')

  return constraints

def append_url_arguments(request, link):
  arguments = request.arguments()
  if len(arguments) == 0:
    return link
  link += ('?' + cgi.escape(arguments[0], True) + '=' +
           cgi.escape(request.get(arguments[0]), True))
  for argument in arguments[1:]:
    link += ('&' + cgi.escape(argument, True) + '=' +
             cgi.escape(request.get(argument), True))
  return link

def get_wss_parameters(request):
  wss_host_port_pair = request.get('wshpp')
  wss_tls = request.get('wstls')

  if not wss_host_port_pair:
    # Attempt to get a wss server from the status provided by prober,
    # if that fails, use fallback value.
    memcache_client = memcache.Client()
    wss_active_host = memcache_client.get(constants.WSS_HOST_ACTIVE_HOST_KEY)
    if wss_active_host in constants.WSS_HOST_PORT_PAIRS:
      wss_host_port_pair = wss_active_host
    else:
      logging.warning(
          'Invalid or no value returned from memcache, using fallback: '
          + json.dumps(wss_active_host))
      wss_host_port_pair = constants.WSS_HOST_PORT_PAIRS[0]

  if wss_tls and wss_tls == 'false':
    wss_url = 'ws://' + wss_host_port_pair + '/ws'
    wss_post_url = 'http://' + wss_host_port_pair
  else:
    wss_url = 'wss://' + wss_host_port_pair + '/ws'
    wss_post_url = 'https://' + wss_host_port_pair
  return (wss_url, wss_post_url)

def get_version_info():
  try:
    path = os.path.join(os.path.dirname(__file__), 'version_info.json')
    f = open(path)
    if f is not None:
      try:
        return json.load(f)
      except ValueError as e:
        logging.warning('version_info.json cannot be decoded: ' + str(e))
  except IOError as e:
    logging.info('version_info.json cannot be opened: ' + str(e))
  return None

def get_callstats_params():
  try:
    path = os.path.join(os.path.dirname(__file__), 'callstats_params.json')
    f = open(path)
    if f is not None:
      try:
        return json.load(f)
      except ValueError as e:
        logging.warning('callstats_params.json cannot be decoded: ' + str(e))
  except IOError as e:
    logging.info('callstats_params.json cannot be opened: ' + str(e))
  return None

# Returns appropriate room parameters based on query parameters in the request.
# TODO(tkchin): move query parameter parsing to JS code.
def get_room_parameters(request, room_id, client_id, is_initiator):
  error_messages = []
  warning_messages = []
  # Get the base url without arguments.
  base_url = request.path_url
  user_agent = request.headers['User-Agent']

  # HTML or JSON.
  response_type = request.get('t')
  # Which ICE candidates to allow. This is useful for forcing a call to run
  # over TURN, by setting it=relay.
  ice_transports = request.get('it')
  # Which ICE server transport= to allow (i.e., only TURN URLs with
  # transport=<tt> will be used). This is useful for forcing a session to use
  # TURN/TCP, by setting it=relay&tt=tcp.
  ice_server_transports = request.get('tt')
  # A HTTP server that will be used to find the right ICE servers to use, as
  # described in http://tools.ietf.org/html/draft-uberti-rtcweb-turn-rest-00.
  ice_server_base_url = request.get('ts', default_value =
      constants.ICE_SERVER_BASE_URL)

  # Use "audio" and "video" to set the media stream constraints. Defined here:
  # http://goo.gl/V7cZg
  #
  # "true" and "false" are recognized and interpreted as bools, for example:
  #   "?audio=true&video=false" (Start an audio-only call.)
  #   "?audio=false" (Start a video-only call.)
  # If unspecified, the stream constraint defaults to True.
  #
  # To specify media track constraints, pass in a comma-separated list of
  # key/value pairs, separated by a "=". Examples:
  #   "?audio=googEchoCancellation=false,googAutoGainControl=true"
  #   (Disable echo cancellation and enable gain control.)
  #
  #   "?video=minWidth=1280,minHeight=720,googNoiseReduction=true"
  #   (Set the minimum resolution to 1280x720 and enable noise reduction.)
  #
  # Keys starting with "goog" will be added to the "optional" key; all others
  # will be added to the "mandatory" key.
  # To override this default behavior, add a "mandatory" or "optional" prefix
  # to each key, e.g.
  #   "?video=optional:minWidth=1280,optional:minHeight=720,
  #           mandatory:googNoiseReduction=true"
  #   (Try to do 1280x720, but be willing to live with less; enable
  #    noise reduction or die trying.)
  #
  # The audio keys are defined here: talk/app/webrtc/localaudiosource.cc
  # The video keys are defined here: talk/app/webrtc/videosource.cc
  audio = request.get('audio')
  video = request.get('video')

  # Pass firefox_fake_device=1 to pass fake: true in the media constraints,
  # which will make Firefox use its built-in fake device.
  firefox_fake_device = request.get('firefox_fake_device')

  # The hd parameter is a shorthand to determine whether to open the
  # camera at 720p. If no value is provided, use a platform-specific default.
  # When defaulting to HD, use optional constraints, in case the camera
  # doesn't actually support HD modes.
  hd = request.get('hd').lower()
  if hd and video:
    message = 'The "hd" parameter has overridden video=' + video
    logging.warning(message)
    # HTML template is UTF-8, make sure the string is UTF-8 as well.
    warning_messages.append(message.encode('utf-8'))
  if hd == 'true':
    video = 'mandatory:minWidth=1280,mandatory:minHeight=720'
  elif not hd and not video and get_hd_default(user_agent) == 'true':
    video = 'optional:minWidth=1280,optional:minHeight=720'

  if request.get('minre') or request.get('maxre'):
    message = ('The "minre" and "maxre" parameters are no longer ' +
        'supported. Use "video" instead.')
    logging.warning(message)
    # HTML template is UTF-8, make sure the string is UTF-8 as well.
    warning_messages.append(message.encode('utf-8'))

  # Options for controlling various networking features.
  dtls = request.get('dtls')
  dscp = request.get('dscp')
  ipv6 = request.get('ipv6')

  debug = request.get('debug')
  if debug == 'loopback':
    # Set dtls to false as DTLS does not work for loopback.
    dtls = 'false'
    include_loopback_js = '<script src="/js/loopback.js"></script>'
  else:
    include_loopback_js = ''

  # TODO(tkchin): We want to provide a ICE request url on the initial get,
  # but we don't provide client_id until a join. For now just generate
  # a random id, but we should make this better.
  username = client_id if client_id is not None else generate_random(9)
  if len(ice_server_base_url) > 0:
    ice_server_url = constants.ICE_SERVER_URL_TEMPLATE % \
        (ice_server_base_url, constants.ICE_SERVER_API_KEY)
  else:
    ice_server_base_url = ''

  turn_url = constants.TURN_URL_TEMPLATE % \
      (constants.TURN_BASE_URL, username, constants.CEOD_KEY)

  pc_config = make_pc_config(ice_transports)
  pc_constraints = make_pc_constraints(dtls, dscp, ipv6)
  offer_options = {};
  media_constraints = make_media_stream_constraints(audio, video,
                                                    firefox_fake_device)
  wss_url, wss_post_url = get_wss_parameters(request)

  bypass_join_confirmation = 'BYPASS_JOIN_CONFIRMATION' in os.environ and \
      os.environ['BYPASS_JOIN_CONFIRMATION'] == 'True'

  params = {
    'error_messages': error_messages,
    'warning_messages': warning_messages,
    'is_loopback' : json.dumps(debug == 'loopback'),
    'pc_config': json.dumps(pc_config),
    'pc_constraints': json.dumps(pc_constraints),
    'offer_options': json.dumps(offer_options),
    'media_constraints': json.dumps(media_constraints),
    'turn_url': turn_url,
    'ice_server_url': ice_server_url,
    'ice_server_transports': ice_server_transports,
    'include_loopback_js' : include_loopback_js,
    'wss_url': wss_url,
    'wss_post_url': wss_post_url,
    'bypass_join_confirmation': json.dumps(bypass_join_confirmation),
    'version_info': json.dumps(get_version_info()),
    'callstats_params': json.dumps(constants.CALLSTATS_PARAMS)
  }

  if room_id is not None:
    room_link = request.host_url + '/r/' + room_id
    room_link = append_url_arguments(request, room_link)
    params['room_id'] = room_id
    params['room_link'] = room_link
  if client_id is not None:
    params['client_id'] = client_id
  if is_initiator is not None:
    params['is_initiator'] = json.dumps(is_initiator)
  return params

# For now we have (room_id, client_id) pairs are 'unique' but client_ids are
# not. Uniqueness is not enforced however and bad things may happen if RNG
# generates non-unique numbers. We also have a special loopback client id.
# TODO(tkchin): Generate room/client IDs in a unique way while handling
# loopback scenario correctly.
class Client:
  def __init__(self, is_initiator):
    self.is_initiator = is_initiator
    self.messages = []
  def add_message(self, msg):
    self.messages.append(msg)
  def clear_messages(self):
    self.messages = []
  def set_initiator(self, initiator):
    self.is_initiator = initiator
  def __str__(self):
    return '{%r, %d}' % (self.is_initiator, len(self.messages))

class Room:
  def __init__(self):
    self.clients = {}
  def add_client(self, client_id, client):
    self.clients[client_id] = client
  def remove_client(self, client_id):
    del self.clients[client_id]
  def get_occupancy(self):
    return len(self.clients)
  def has_client(self, client_id):
    return client_id in self.clients
  def get_client(self, client_id):
    return self.clients[client_id]
  def get_other_client(self, client_id):
    for key, client in self.clients.items():
      if key is not client_id:
        return client
    return None
  def __str__(self):
    return str(self.clients.keys())

def get_memcache_key_for_room(host, room_id):
  return '%s/%s' % (host, room_id)

def add_client_to_room(request, room_id, client_id, is_loopback):
  key = get_memcache_key_for_room(request.host_url, room_id)
  memcache_client = memcache.Client()
  error = None
  retries = 0
  room = None
  # Compare and set retry loop.
  while True:
    is_initiator = None
    messages = []
    room_state = ''
    room = memcache_client.gets(key)
    if room is None:
      # 'set' and another 'gets' are needed for CAS to work.
      if not memcache_client.set(key, Room()):
        logging.warning('memcache.Client.set failed for key ' + key)
        error = constants.RESPONSE_ERROR
        break
      room = memcache_client.gets(key)

    occupancy = room.get_occupancy()
    if occupancy >= 2:
      error = constants.RESPONSE_ROOM_FULL
      break
    if room.has_client(client_id):
      error = constants.RESPONSE_DUPLICATE_CLIENT
      break

    if occupancy == 0:
      is_initiator = True
      room.add_client(client_id, Client(is_initiator))
      if is_loopback:
        room.add_client(constants.LOOPBACK_CLIENT_ID, Client(False))
    else:
      is_initiator = False
      other_client = room.get_other_client(client_id)
      messages = other_client.messages
      room.add_client(client_id, Client(is_initiator))
      other_client.clear_messages()

    if memcache_client.cas(key, room, constants.ROOM_MEMCACHE_EXPIRATION_SEC):
      logging.info('Added client %s in room %s, retries = %d' \
          %(client_id, room_id, retries))

      if room.get_occupancy() == 2:
        analytics.report_event(analytics.EventType.ROOM_SIZE_2,
                               room_id,
                               host=request.host)
      success = True
      break
    else:
      retries = retries + 1
  return {'error': error, 'is_initiator': is_initiator,
          'messages': messages, 'room_state': str(room)}

def remove_client_from_room(host, room_id, client_id):
  key = get_memcache_key_for_room(host, room_id)
  memcache_client = memcache.Client()
  retries = 0
  # Compare and set retry loop.
  while True:
    room = memcache_client.gets(key)
    if room is None:
      logging.warning('remove_client_from_room: Unknown room ' + room_id)
      return {'error': constants.RESPONSE_UNKNOWN_ROOM, 'room_state': None}
    if not room.has_client(client_id):
      logging.warning('remove_client_from_room: Unknown client ' + client_id + \
          ' for room ' + room_id)
      return {'error': constants.RESPONSE_UNKNOWN_CLIENT, 'room_state': None}

    room.remove_client(client_id)
    if room.has_client(constants.LOOPBACK_CLIENT_ID):
      room.remove_client(constants.LOOPBACK_CLIENT_ID)
    if room.get_occupancy() > 0:
      room.get_other_client(client_id).set_initiator(True)
    else:
      room = None

    if memcache_client.cas(key, room, constants.ROOM_MEMCACHE_EXPIRATION_SEC):
      logging.info('Removed client %s from room %s, retries=%d' \
          %(client_id, room_id, retries))
      return {'error': None, 'room_state': str(room)}
    retries = retries + 1

def save_message_from_client(host, room_id, client_id, message):
  text = None
  try:
      text = message.encode(encoding='utf-8', errors='strict')
  except Exception as e:
    return {'error': constants.RESPONSE_ERROR, 'saved': False}

  key = get_memcache_key_for_room(host, room_id)
  memcache_client = memcache.Client()
  retries = 0
  # Compare and set retry loop.
  while True:
    room = memcache_client.gets(key)
    if room is None:
      logging.warning('Unknown room: ' + room_id)
      return {'error': constants.RESPONSE_UNKNOWN_ROOM, 'saved': False}
    if not room.has_client(client_id):
      logging.warning('Unknown client: ' + client_id)
      return {'error': constants.RESPONSE_UNKNOWN_CLIENT, 'saved': False}
    if room.get_occupancy() > 1:
      return {'error': None, 'saved': False}

    client = room.get_client(client_id)
    client.add_message(text)
    if memcache_client.cas(key, room, constants.ROOM_MEMCACHE_EXPIRATION_SEC):
      logging.info('Saved message for client %s:%s in room %s, retries=%d' \
          %(client_id, str(client), room_id, retries))
      return {'error': None, 'saved': True}
    retries = retries + 1

class LeavePage(webapp2.RequestHandler):
  def post(self, room_id, client_id):
    result = remove_client_from_room(
        self.request.host_url, room_id, client_id)
    if result['error'] is None:
      logging.info('Room ' + room_id + ' has state ' + result['room_state'])

class MessagePage(webapp2.RequestHandler):
  def write_response(self, result):
    content = json.dumps({ 'result' : result })
    self.response.write(content)

  def send_message_to_collider(self, room_id, client_id, message):
    logging.info('Forwarding message to collider for room ' + room_id +
                 ' client ' + client_id)
    wss_url, wss_post_url = get_wss_parameters(self.request)
    url = wss_post_url + '/' + room_id + '/' + client_id
    result = urlfetch.fetch(url=url,
                            payload=message,
                            method=urlfetch.POST)
    if result.status_code != 200:
      logging.error(
          'Failed to send message to collider: %d' % (result.status_code))
      # TODO(tkchin): better error handling.
      self.error(500)
      return
    self.write_response(constants.RESPONSE_SUCCESS)

  def post(self, room_id, client_id):
    message_json = self.request.body
    result = save_message_from_client(
        self.request.host_url, room_id, client_id, message_json)
    if result['error'] is not None:
      self.write_response(result['error'])
      return
    if not result['saved']:
      # Other client joined, forward to collider. Do this outside the lock.
      # Note: this may fail in local dev server due to not having the right
      # certificate file locally for SSL validation.
      # Note: loopback scenario follows this code path.
      # TODO(tkchin): consider async fetch here.
      self.send_message_to_collider(room_id, client_id, message_json)
    else:
      self.write_response(constants.RESPONSE_SUCCESS)

class JoinPage(webapp2.RequestHandler):
  def write_response(self, result, params, messages):
    # TODO(tkchin): Clean up response format. For simplicity put everything in
    # params for now.
    params['messages'] = messages
    self.response.write(json.dumps({
      'result': result,
      'params': params
    }))

  def write_room_parameters(self, room_id, client_id, messages, is_initiator):
    params = get_room_parameters(self.request, room_id, client_id, is_initiator)
    self.write_response('SUCCESS', params, messages)

  def post(self, room_id):
    client_id = generate_random(8)
    is_loopback = self.request.get('debug') == 'loopback'
    result = add_client_to_room(self.request, room_id, client_id, is_loopback)
    if result['error'] is not None:
      logging.info('Error adding client to room: ' + result['error'] + \
          ', room_state=' + result['room_state'])
      self.write_response(result['error'], {}, [])
      return

    self.write_room_parameters(
        room_id, client_id, result['messages'], result['is_initiator'])
    logging.info('User ' + client_id + ' joined room ' + room_id)
    logging.info('Room ' + room_id + ' has state ' + result['room_state'])

class MainPage(webapp2.RequestHandler):
  def write_response(self, target_page, params={}):
    template = jinja_environment.get_template(target_page)
    content = template.render(params)
    self.response.out.write(content)

  def get(self):
    """Renders index.html."""
    if self.request.headers['Host'] == 'apprtc.net':
      webapp2.redirect('https://www.apprtc.net', permanent=True)
    # Parse out parameters from request.
    params = get_room_parameters(self.request, None, None, None)
    # room_id/room_link will not be included in the returned parameters
    # so the client will show the landing page for room selection.
    self.write_response('index_template.html', params)

class RoomPage(webapp2.RequestHandler):
  def write_response(self, target_page, params={}):
    template = jinja_environment.get_template(target_page)
    content = template.render(params)
    self.response.out.write(content)

  def get(self, room_id):
    """Renders index.html or full.html."""
    # Check if room is full.
    room = memcache.get(
        get_memcache_key_for_room(self.request.host_url, room_id))
    if room is not None:
      logging.info('Room ' + room_id + ' has state ' + str(room))
      if room.get_occupancy() >= 2:
        logging.info('Room ' + room_id + ' is full')
        self.write_response('full_template.html')
        return
    # Parse out room parameters from request.
    params = get_room_parameters(self.request, room_id, None, None)
    # room_id/room_link will be included in the returned parameters
    # so the client will launch the requested room.
    self.write_response('index_template.html', params)

class ParamsPage(webapp2.RequestHandler):
  def get(self):
    # Return room independent room parameters.
    params = get_room_parameters(self.request, None, None, None)
    self.response.write(json.dumps(params))

app = webapp2.WSGIApplication([
    ('/', MainPage),
    ('/a/', analytics_page.AnalyticsPage),
    ('/compute/(\w+)/(\S+)/(\S+)', compute_page.ComputePage),
    ('/join/([a-zA-Z0-9-_]+)', JoinPage),
    ('/leave/([a-zA-Z0-9-_]+)/([a-zA-Z0-9-_]+)', LeavePage),
    ('/message/([a-zA-Z0-9-_]+)/([a-zA-Z0-9-_]+)', MessagePage),
    ('/params', ParamsPage),
    ('/r/([a-zA-Z0-9-_]+)', RoomPage),
], debug=True)


text:
package com.datatorrent.lib.io.block;

import org.apache.hadoop.fs.FSDataInputStream;

import com.datatorrent.api.StatsListener;

import com.datatorrent.netlet.util.Slice;

/**
 * An {@link AbstractFSBlockReader} which emits fixed-size byte-arrays wrapped in {@link Slice}.<br/>
 *
 * @category Input
 * @tags fs
 *
 * @since 2.1.0
 */
@StatsListener.DataQueueSize
public class FSSliceReader extends AbstractFSBlockReader<Slice>
{
  public FSSliceReader()
  {
    super();
    this.readerContext = new ReaderContext.FixedBytesReaderContext<FSDataInputStream>();
  }

  @Override
  protected Slice convertToRecord(byte[] bytes)
  {
    return new Slice(bytes);
  }
}


text:
import { ajouterRouteApi } from '../../bacasable/bacasable/routage';
import { WebService } from '../../bacasable/bacasable/webservice';

export class AuthentificationIn
{
    email:string;
    motDePasse:string;
}

export class AuthentificationOut
{
    ok:boolean;
}

export class Authentification extends WebService<AuthentificationIn, AuthentificationOut>
{
    static route = ajouterRouteApi('/api/authentification', Authentification);
}

export class Inscription extends WebService<AuthentificationIn, AuthentificationOut>
{
    static route = ajouterRouteApi('/api/inscription', Inscription);
}


text:
<?php namespace Anomaly\Streams\Platform\Model\Posts;

use Anomaly\Streams\Platform\Entry\EntryModel;

class PostsTypesEntryModel extends EntryModel
{

    use \Illuminate\Database\Eloquent\SoftDeletes;

    protected $searchable = false;

    protected $table = 'posts_types';

    protected $titleName = 'name';

    protected $rules = [
'name' => 'required|unique:posts_types,name',
'slug' => 'required|unique:posts_types,slug',
'layout' => 'required',
'theme_layout' => 'required',
'description' => '',
];

    protected $fields = [
'name',
'slug',
'layout',
'theme_layout',
'description',
];

    protected $dates = ['created_at', 'updated_at', 'deleted_at'];

    protected $relationships = [];

    protected $translatedAttributes = ['name', 'description'];

    protected $translationForeignKey = 'entry_id';

    protected $translationModel = 'Anomaly\Streams\Platform\Model\Posts\PostsTypesEntryTranslationsModel';

    protected $stream = [
'id' => '13',
'namespace' => 'posts',
'slug' => 'types',
'prefix' => 'posts_',
'title_column' => 'name',
'order_by' => 'id',
'locked' => '0',
'hidden' => '0',
'sortable' => '1',
'searchable' => '0',
'trashable' => '1',
'translatable' => '1',
'config' => 'a:0:{}',
'assignments' => [
[
'id' => 85,
'sort_order' => 85,
'stream_id' => 13,
'field_id' => 60,
'config' => 'a:1:{s:3:"max";i:50;}',
'unique' => 1,
'required' => 1,
'translatable' => 1,
'field' => [
'id' => '60',
'namespace' => 'posts',
'slug' => 'name',
'type' => 'anomaly.field_type.text',
'config' => 'a:0:{}',
'locked' => '1',
'translations' => [
[
'id' => 60,
'field_id' => 60,
'locale' => 'en',
'name' => 'anomaly.module.posts::field.name.name',
'placeholder' => 'anomaly.module.posts::field.name.placeholder',
'warning' => 'anomaly.module.posts::field.name.warning',
'instructions' => 'anomaly.module.posts::field.name.instructions',
],
],
],
'translations' => [
[
'id' => 85,
'assignment_id' => 85,
'locale' => 'en',
'label' => 'anomaly.module.posts::field.name.label.types',
'warning' => 'anomaly.module.posts::field.name.warning.types',
'placeholder' => 'anomaly.module.posts::field.name.placeholder.types',
'instructions' => 'anomaly.module.posts::field.name.instructions.types',
],
],
],
[
'id' => 86,
'sort_order' => 86,
'stream_id' => 13,
'field_id' => 62,
'config' => 'a:3:{s:7:"slugify";s:4:"name";s:4:"type";s:1:"_";s:3:"max";i:50;}',
'unique' => 1,
'required' => 1,
'translatable' => 0,
'field' => [
'id' => '62',
'namespace' => 'posts',
'slug' => 'slug',
'type' => 'anomaly.field_type.slug',
'config' => 'a:2:{s:7:"slugify";s:5:"title";s:4:"type";s:1:"-";}',
'locked' => '1',
'translations' => [
[
'id' => 62,
'field_id' => 62,
'locale' => 'en',
'name' => 'anomaly.module.posts::field.slug.name',
'placeholder' => 'anomaly.module.posts::field.slug.placeholder',
'warning' => 'anomaly.module.posts::field.slug.warning',
'instructions' => 'anomaly.module.posts::field.slug.instructions',
],
],
],
'translations' => [
[
'id' => 86,
'assignment_id' => 86,
'locale' => 'en',
'label' => 'anomaly.module.posts::field.slug.label.types',
'warning' => 'anomaly.module.posts::field.slug.warning.types',
'placeholder' => 'anomaly.module.posts::field.slug.placeholder.types',
'instructions' => 'anomaly.module.posts::field.slug.instructions.types',
],
],
],
[
'id' => 87,
'sort_order' => 87,
'stream_id' => 13,
'field_id' => 71,
'config' => 'a:0:{}',
'unique' => 0,
'required' => 1,
'translatable' => 0,
'field' => [
'id' => '71',
'namespace' => 'posts',
'slug' => 'layout',
'type' => 'anomaly.field_type.editor',
'config' => 'a:2:{s:13:"default_value";s:22:"{{ post.content|raw }}";s:4:"mode";s:4:"twig";}',
'locked' => '1',
'translations' => [
[
'id' => 71,
'field_id' => 71,
'locale' => 'en',
'name' => 'anomaly.module.posts::field.layout.name',
'placeholder' => 'anomaly.module.posts::field.layout.placeholder',
'warning' => 'anomaly.module.posts::field.layout.warning',
'instructions' => 'anomaly.module.posts::field.layout.instructions',
],
],
],
'translations' => [
[
'id' => 87,
'assignment_id' => 87,
'locale' => 'en',
'label' => 'anomaly.module.posts::field.layout.label.types',
'warning' => 'anomaly.module.posts::field.layout.warning.types',
'placeholder' => 'anomaly.module.posts::field.layout.placeholder.types',
'instructions' => 'anomaly.module.posts::field.layout.instructions.types',
],
],
],
[
'id' => 88,
'sort_order' => 88,
'stream_id' => 13,
'field_id' => 79,
'config' => 'a:0:{}',
'unique' => 0,
'required' => 1,
'translatable' => 0,
'field' => [
'id' => '79',
'namespace' => 'posts',
'slug' => 'theme_layout',
'type' => 'anomaly.field_type.select',
'config' => 'a:1:{s:7:"handler";s:39:"Anomaly\SelectFieldType\Handler\Layouts";}',
'locked' => '1',
'translations' => [
[
'id' => 79,
'field_id' => 79,
'locale' => 'en',
'name' => 'anomaly.module.posts::field.theme_layout.name',
'placeholder' => 'anomaly.module.posts::field.theme_layout.placeholder',
'warning' => 'anomaly.module.posts::field.theme_layout.warning',
'instructions' => 'anomaly.module.posts::field.theme_layout.instructions',
],
],
],
'translations' => [
[
'id' => 88,
'assignment_id' => 88,
'locale' => 'en',
'label' => 'anomaly.module.posts::field.theme_layout.label.types',
'warning' => 'anomaly.module.posts::field.theme_layout.warning.types',
'placeholder' => 'anomaly.module.posts::field.theme_layout.placeholder.types',
'instructions' => 'anomaly.module.posts::field.theme_layout.instructions.types',
],
],
],
[
'id' => 89,
'sort_order' => 89,
'stream_id' => 13,
'field_id' => 67,
'config' => 'a:0:{}',
'unique' => 0,
'required' => 0,
'translatable' => 1,
'field' => [
'id' => '67',
'namespace' => 'posts',
'slug' => 'description',
'type' => 'anomaly.field_type.textarea',
'config' => 'a:0:{}',
'locked' => '1',
'translations' => [
[
'id' => 67,
'field_id' => 67,
'locale' => 'en',
'name' => 'anomaly.module.posts::field.description.name',
'placeholder' => 'anomaly.module.posts::field.description.placeholder',
'warning' => 'anomaly.module.posts::field.description.warning',
'instructions' => 'anomaly.module.posts::field.description.instructions',
],
],
],
'translations' => [
[
'id' => 89,
'assignment_id' => 89,
'locale' => 'en',
'label' => 'anomaly.module.posts::field.description.label.types',
'warning' => 'anomaly.module.posts::field.description.warning.types',
'placeholder' => 'anomaly.module.posts::field.description.placeholder.types',
'instructions' => 'anomaly.module.posts::field.description.instructions.types',
],
],
],
],
'translations' => [
[
'id' => 13,
'stream_id' => 13,
'locale' => 'en',
'name' => 'anomaly.module.posts::stream.types.name',
'description' => 'anomaly.module.posts::stream.types.description',
],
],
];

}


text:
package com.licryle.veliby.BikeMap;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.Map;

import android.app.Activity;
import android.content.Intent;
import android.os.Bundle;
import android.os.Handler;
import android.os.Parcelable;
import android.os.ResultReceiver;
import android.util.Log;
import android.view.View;

import com.directions.route.Routing;
import com.directions.route.RoutingListener;
import com.google.android.gms.maps.CameraUpdate;
import com.google.android.gms.maps.CameraUpdateFactory;
import com.google.android.gms.maps.GoogleMap;
import com.google.android.gms.maps.GoogleMap.InfoWindowAdapter;
import com.google.android.gms.maps.GoogleMap.OnMapClickListener;
import com.google.android.gms.maps.GoogleMap.OnMarkerClickListener;
import com.google.android.gms.maps.model.BitmapDescriptorFactory;
import com.google.android.gms.maps.model.CameraPosition;
import com.google.android.gms.maps.model.LatLng;
import com.google.android.gms.maps.model.Marker;
import com.google.android.gms.maps.model.MarkerOptions;
import com.google.android.gms.maps.model.Polyline;
import com.google.android.gms.maps.model.PolylineOptions;
import com.licryle.veliby.R;
import com.licryle.veliby.Settings;
import com.licryle.veliby.Util;

public class BikeMap implements OnMarkerClickListener, OnMapClickListener,
    InfoWindowAdapter, RoutingListener {
	private GoogleMap _mMap;

	protected Activity _mContext = null;

  protected boolean	_bModeFindBike = true;
  protected boolean	_bDownloading = false;

  protected ArrayList<BikeMapListener> _aListeners;
  protected Stations _mStations = null;
  protected Settings _mSettings = null;

  protected Polyline _mCurrentDirections = null;

	protected static Hashtable<Integer, Integer> _mBikeResources =
			new Hashtable<Integer, Integer>() {
        private static final long serialVersionUID = -276505145697466182L;
				{
					put(0, R.drawable.presence_invisible);
					put(2, R.drawable.presence_busy);
					put(4, R.drawable.presence_away);
					put(1000, R.drawable.presence_online);
				}
			};

	public BikeMap(Activity mContext, GoogleMap mMap, Settings mSettings) {
    _aListeners = new ArrayList<BikeMapListener>();
    _mSettings = mSettings;

    _mMap = mMap;
    _mMap.setInfoWindowAdapter(this);
    _mContext = mContext;

    setupMap();
	}

  public boolean isDownloading() { return _bDownloading; }
  public boolean isFindBikeMode() { return _bModeFindBike; }
  public Stations stations() { return _mStations; }

  public void changeBikeMode(boolean bFindBike) {
    _bModeFindBike = bFindBike;
    updateMarkers();
  }

  public boolean downloadMarkers() {
    Log.i("BikeMap", "Entered downloadMarkers()");
    if (_bDownloading) return false;
    _bDownloading = true;

    Settings mSettings = Settings.getInstance(_mContext);
    Intent intent = new Intent(_mContext, StationsInfoService.class);

    intent.putExtra("receiver",
        (Parcelable) new DownloadStationsReceiver(new Handler()));
    intent.putExtra("requestor", this.toString());
    intent.putExtra("url_full",
        mSettings.getURLDownloadFull(mSettings.getCurrentContract()));
    intent.putExtra("url_dynamic",
        mSettings.getURLDownloadDynamic(mSettings.getCurrentContract()));
    intent.putExtra("dl_static", mSettings.getStaticDeadLine());
    intent.putExtra("dl_dynamic", mSettings.getDynamicDeadLine());
    intent.putExtra("contract_id", mSettings.getCurrentContract().getId());
    intent.putExtra("stations_file",
        mSettings.getStationsFile().getAbsolutePath());

    Log.i("BikeMap", "Starting Intent in downloadMarkers()");
    _mContext.startService(intent);
    return true;
  }

  public void moveCameraTo(LatLng mPosition, int iZoom) {
    if (mPosition == null) return;

    CameraUpdate cu = CameraUpdateFactory.newCameraPosition(
        new CameraPosition(mPosition, iZoom, 0, 0));

    updateCamera(cu);
  }

  public void updateCamera(CameraUpdate mCameraUpdate) {
    _mMap.animateCamera(mCameraUpdate, 700, null);
  }

  protected void updateMarkers() {
    if (_mMap == null || _mStations == null) return;

    _mMap.clear();
    Iterator<Map.Entry<Integer, Station>> it = _mStations.entrySet().
        iterator();

    boolean bShowfavOnly = _mSettings.isFavStationsOnly();
    ArrayList<Integer> aFavStations = _mSettings.getFavStations();
    while (it.hasNext()) {
      Map.Entry<Integer, Station> entry = it.next();
      Station mStation = entry.getValue();

      if (! bShowfavOnly ||
          (bShowfavOnly && aFavStations.contains(mStation.getId()))) {
        MarkerOptions mOpts = new MarkerOptions();
        mOpts.position(mStation.getPosition());
        mOpts.title(mStation.getName());

        int id = mStation.getId();
        mOpts.title(String.valueOf(id));

        int iIcon;
        if (! mStation.isOpened()) {
          iIcon = R.drawable.presence_offline;
        } else {
          int iBikes = (_bModeFindBike) ?
               mStation.getAvailableBikes() :
               mStation.getAvailableBikeStands();

          iIcon = Util.resolveResourceFromNumber(_mBikeResources, iBikes);
        }

        mOpts.icon(BitmapDescriptorFactory.fromResource(iIcon));
        _mMap.addMarker(mOpts);
      }
    }
  }

  protected boolean setupMap() {
    if (_mMap == null) return false;

    _mMap.setMyLocationEnabled(true);
    _mMap.setOnMarkerClickListener(this);
    _mMap.setOnMapClickListener(this);

    moveCameraTo(Util.getLastPosition(_mContext), 16);

    return true;
  }

  @Override
  public boolean onMarkerClick(Marker mMarker) {
    if (_mStations == null) return false;

    String sStationId = mMarker.getTitle();
    Station mStation = _mStations.get(Integer.valueOf(sStationId));

    dispatchOnStationClick(mStation);
    return false;
  }

  @Override
  public void onMapClick(LatLng mLatLng) {
    dispatchOnMapClick(mLatLng);
  }

  public void registerBikeMapListener(BikeMapListener mListener) {
    _aListeners.add(mListener);
  }

  protected void dispatchOnStationClick(Station mStation) {
    for (BikeMapListener mListener: _aListeners) {
      mListener.onStationClick(this, mStation);
    }
  }

  protected void dispatchOnMapClick(LatLng mLatLng) {
    for (BikeMapListener mListener: _aListeners) {
      mListener.onMapClick(this, mLatLng);
    }
  }

  protected void dispatchOnDownloadFailure() {
    for (BikeMapListener mListener: _aListeners) {
      mListener.onDownloadFailure(this);
    }
  }

  protected void dispatchOnDownloadSuccess() {
    for (BikeMapListener mListener: _aListeners) {
      mListener.onDownloadSuccess(this);
    }
  }

  protected void dispatchOnDirectionsFailed() {
    for (BikeMapListener mListener: _aListeners) {
      mListener.onDirectionsFailed(this);
    }
  }

  protected View dispatchOnGetInfoContents(Marker mMarker, Station mStation) {
    View mResult = null;
    for (BikeMapListener mListener: _aListeners) {
      mResult = mListener.onGetInfoContents(mMarker, mStation);
    }

    return mResult;
  }

  private class DownloadStationsReceiver extends ResultReceiver
      implements Serializable{
    /**
     *
     */
    private static final long serialVersionUID = 5529673172741409950L;

    public DownloadStationsReceiver(Handler handler) {
      super(handler);
    }

    @Override
    protected void onReceiveResult(int resultCode, Bundle resultData) {
      super.onReceiveResult(resultCode, resultData);

      switch (resultCode) {
        case StationsInfoService.SUCCESS:
          Log.i("BikeMap", "onReceiveResult() SUCCESS");
          // date doesn't matter since it was just generated
          _mStations = (Stations) resultData.getSerializable("stations");

          updateMarkers();
          dispatchOnDownloadSuccess();
          _bDownloading = false;
        break;

        case StationsInfoService.FAILURE_CONNECTION:
        case StationsInfoService.FAILURE_GENERIC:
        case StationsInfoService.FAILURE_PARSE:
          Log.i("BikeMap", "onReceiveResult() FAILURE_xxxx");

          _mStations = (Stations) resultData.getSerializable("stations");

          // we need this to display at least static data
          updateMarkers();

          dispatchOnDownloadFailure();
          _bDownloading = false;
        break;

        case StationsInfoService.FINISHED:
          Log.i("BikeMap", "onReceiveResult() FINISHED");
          _bDownloading = false;
        break;
      }
    }
  }

  @Override
  public View getInfoContents(Marker mMarker) {
    if (_mStations == null) return null;

    String sStationId = mMarker.getTitle();
    Station mStation = _mStations.get(Integer.valueOf(sStationId));

    return dispatchOnGetInfoContents(mMarker, mStation);
  }

  @Override
  public View getInfoWindow(Marker mMarker) {
    return null;
  }

  public void ShowAllStations() {
    _mSettings.setFavStationsOnly(false);
    updateMarkers();
  }

  public void ShowFavStations() {
    _mSettings.setFavStationsOnly(true);
    updateMarkers();
  }

  public void displayDirections(LatLng _mPos) {
    Routing mRouting = new Routing(_mContext, Routing.TravelMode.WALKING);
    mRouting.registerListener(this);
    mRouting.execute(Util.getLastPosition(_mContext), _mPos);
  }

  public void clearDirections() {
    if (_mCurrentDirections != null) _mCurrentDirections.remove();
  }

  @Override
  public void onRoutingFailure() {
    clearDirections();
    dispatchOnDirectionsFailed();
  }

  @Override
  public void onRoutingStart() {
  }

  @Override
  public void onRoutingSuccess(PolylineOptions mPolyOptions) {
    PolylineOptions mOptions = _mSettings.getDirectionsStyle();
    mOptions.addAll(mPolyOptions.getPoints());

    clearDirections();
    _mCurrentDirections = _mMap.addPolyline(mOptions);
  }
}


text:
layout: page-fullwidth
title: "San Francisco Crime"
#subheadline: "Exploring and Mapping Crime in San Francisco"
meta_teaser: "Exploring and Mapping Crime in San Francisco"
teaser: "Exploring and Mapping Crime in San Francisco"
breadcrumb: true
comments: true
meta: true
header:
    title: ""
    image_fullwidth: SF-Crime2_Middle.png
    background-color: "#262930"
    caption: Mapping Sanfrancisco Crime
image:
    thumb:  SF-Crime.png
    homepage: SF-Crime2_Middle.png
    caption: Mapping Sanfrancisco Crime
categories:
    - projects
tags:
    - r
    - google maps
    - ggplot2
---

## Intro

I recently stumbled onto the [San Francisco Police Deparatment's Incidents Dataset](https://data.sfgov.org/Public-Safety/SFPD-Incidents-from-1-January-2003/tmnf-yvry), which caught my interest and I decided to investigate further.

[This](https://data.sfgov.org/api/views/tmnf-yvry/rows.csv?accessType=DOWNLOAD) dataset contains records of all the Incidents derived from SFPD Crime Incident Reporting system from 01/01/2003 up until 31/12/2014.

It contains almost 2 million incidents, where each incident has 13 data variables showing the Category, Description, and Resolution of the incident along with Date, Time and Location details.

From these it is possible to explore the data in some detail and also create some geo-data maps of incident locations.

Let's firstly take a look at the main types of information it contains.

## Categories

Category is a broad umbrella term to easily determine the nature of the incident.
There are 39 categories with these being the most numerous:

{% highlight r %}
sf.data %>%
    group_by(Category) %>%
    summarise(Num=n()) %>%
    arrange(desc(Num)) %>%
    head() %>%
    kable()
{% endhighlight %}

|Category       |    Num|
|:--------------|------:|
|Larceny/Theft  | 374680|
|Other Offenses | 263432|
|Non-Criminal   | 195806|
|Assault        | 161609|
|Vehicle Theft  | 112148|
|Drug/Narcotic  | 110181|

{% highlight r %}
sf.data %>%
    select(Category) %>%
    group_by(Category) %>%
    summarise(Num=n()) %>%
    arrange(desc(Num)) %>%
    head(7) %>%

    ggplot(aes(x=Category, y=Num)) +
        geom_bar(stat = "identity", aes(fill=Category)) +
        theme(axis.text.x=element_blank()) +
        labs(x="Category",
             y = "Number",
             title="Top Incident Categories") +
        scale_y_continuous(labels = comma)
{% endhighlight %}

![center](/figs/SFCrimeExplore/top-crime-categories-1.png)

## Crime Type

The dataset also contains a finer grain summary of each incident; these examples are from taken from the Drug/Narcotic category:

{% highlight r %}
sf.data %>%
    filter(Category=="Drug/Narcotic") %>%
    group_by(Descript) %>%
    summarise(Num=n()) %>%
    arrange(desc(Num)) %>%
    head() %>%
    kable()
{% endhighlight %}

|Descript                                 |   Num|
|:----------------------------------------|-----:|
|Possession Of Narcotics Paraphernalia    | 20486|
|Possession Of Base/Rock Cocaine          | 14109|
|Possession Of Marijuana                  | 11228|
|Sale Of Base/Rock Cocaine                |  8805|
|Possession Of Meth-Amphetamine           |  7551|
|Possession Of Base/Rock Cocaine For Sale |  7364|

## Resolution

The outcome of the incident.
Shown below are most common outcomes in the Weapon Laws category.

{% highlight r %}
sf.data %>%
    filter(Category=="Weapon Laws") %>%
    group_by(Resolution) %>%
    summarise(Num=n()) %>%
    arrange(desc(Num)) %>%
    head() %>%
    kable()
{% endhighlight %}

|Resolution          |   Num|
|:-------------------|-----:|
|Arrest, Booked      | 10554|
|None                |  4874|
|Arrest, Cited       |  1156|
|Juvenile Booked     |   636|
|Juvenile Cited      |   262|
|Juvenile Admonished |   191|

## Arrested?

For ease of analysis a new 'Arrested' variable was created which simplifies two arrest resolutions ("Arrest, Booked" and "Arrest, Cited") as 1, and all the other Resolutions as 0.

{% highlight r %}
sf.data <- sf.data %>%
            mutate(Arrested=ifelse(grepl("Arrest", Resolution), 1, 0))
{% endhighlight %}
This allows us to easily show the incidents with the highest arrest rate:

{% highlight r %}
sf.data %>%
    group_by(Category) %>%
    summarise(Num=n(),
              Arrested=sum(Arrested),
              PercentArrested=round(100*sum(Arrested)/n(), 0)) %>%
    arrange(desc(PercentArrested)) %>%
    head(9) %>%
    kable()
{% endhighlight %}

|Category                    |    Num| Arrested| PercentArrested|
|:---------------------------|------:|--------:|---------------:|
|Driving Under The Influence |   4867|     4558|              94|
|Prostitution                |  15429|    14434|              94|
|Warrants                    |  87945|    81579|              93|
|Drug/Narcotic               | 110181|    98538|              89|
|Loitering                   |   2340|     2040|              87|
|Liquor Laws                 |   3821|     3221|              84|
|Stolen Property             |   9825|     8060|              82|
|Drunkenness                 |   8882|     7212|              81|
|Other Offenses              | 263432|   183513|              70|

And also the incidents with the lowest:

{% highlight r %}
sf.data %>%
    group_by(Category) %>%
    summarise(Num=n(),
              Arrested=sum(Arrested),
              PercentArrested=round(100*sum(Arrested)/n(), 0)) %>%
    arrange(desc(PercentArrested)) %>%
    tail(9) %>%
    kable()
{% endhighlight %}

|Category          |    Num| Arrested| PercentArrested|
|:-----------------|------:|--------:|---------------:|
|Larceny/Theft     | 374680|    31419|               8|
|Recovered Vehicle |   6346|      395|               6|
|Bad Checks        |    849|       45|               5|
|Non-Criminal      | 195806|    10082|               5|
|Suspicious Occ    |  66176|     2926|               4|
|Vehicle Theft     | 112148|     4381|               4|
|Suicide           |   1108|       32|               3|
|Missing Person    |  54344|     1224|               2|
|Runaway           |   3957|       17|               0|

## Highest Arrested Percent Crimes

{% highlight r %}
sf.data %>%
    group_by(Category) %>%
    summarise(Num=n(),
              Arrested=sum(Arrested),
              Percent=round(100*sum(Arrested)/n(), 0)) %>%
    arrange(desc(Percent)) %>%
    head(9) %>%

    ggplot(aes(x=reorder(Category, -Percent),
               y=Percent,
               fill=reorder(Category, -Percent))) +
        geom_bar(stat = "identity") +
    theme(axis.text.x=element_blank(),
          legend.title=element_blank()) +
    labs(x="Category",
         y = "Percent Arrested",
         title="Percent Arrested per Category of Incident") +
    ylim(c(0,100))
{% endhighlight %}

![center](/figs/SFCrimeExplore/Highest-Arrested-Percent-Crimes-1.png)

## Robbery/Theft Data

Having briefly looked at arrest rates, let's move onto crime location.

As a visitor to San Fransisco the crimes that would interest me most (in a can I avoid them kind of way) would probably be robbery and theft.

These appear to grouped under the following descriptions:

{% highlight r %}
sf.p.lt <-
    sf.data %>%
    filter(Category=="Larceny/Theft" &
            grepl("Person|Pursesnatch|Pickpocket|Credit Card", Descript) |
           Category=="Robbery" &
            !grepl("Bank|Carjacking|Store|Residence|Vehicle|Station|Estab", Descript))

sf.p.lt %>%
    group_by(Descript) %>%
    summarise(Num=n()) %>%
    arrange(desc(Num)) %>%
    head(10) %>%
    kable()
{% endhighlight %}

|Descript                                          |   Num|
|:-------------------------------------------------|-----:|
|Grand Theft From Person                           | 14799|
|Robbery On The Street, Strongarm                  | 13837|
|Grand Theft Pickpocket                            | 11728|
|Robbery, Bodily Force                             |  9407|
|Robbery On The Street With A Gun                  |  4278|
|Attempted Robbery On The Street With Bodily Force |  2349|
|Robbery, Armed With A Gun                         |  2292|
|Attempted Robbery With Bodily Force               |  1610|
|Robbery On The Street With A Knife                |  1535|
|Grand Theft Pursesnatch                           |  1487|

## Robbery/Theft Incidents Map

{% highlight r %}
sf.big.box = c(-122.52,37.7,-122.35,37.82)
big.map <- get_map(location = sf.big.box)
{% endhighlight %}

{% highlight r %}
ggmap(big.map) +
    geom_point(data = sf.p.lt, aes(x=X, y=Y),
               color = "red", size = 0.6, alpha = 0.3) +
    labs(title = "Locations of Robbery/Theft Incidents in San Francisco")
{% endhighlight %}

![center](/figs/SFCrimeExplore/p.lt-map-1.png)

This map shows that Robbery/Theft occurs all over SF, but gives little indication of which areas are better or worse than others.

## Robbery/Theft Contour Map

Let's look at a contour map and see if there are any hotspots.

{% highlight r %}
# Robbery/Theft Contours
ggmap(big.map, extent='device', legend="bottomleft") +
    stat_density2d(data = sf.p.lt,
        aes(x = X, y = Y, fill = ..level.., alpha=..level..),
        size = 0.1, colour="red", n=100, geom = "polygon") +
    scale_alpha_continuous(range=c(0.6,0.8), guide='none') +
    scale_fill_gradient('Robbery\nTheft\nDensity')+
    ggtitle('Robbery/Theft Density in San Francisco')
{% endhighlight %}

![center](/figs/SFCrimeExplore/p-lt-contour-map-1.png)

## A Closer Look

There are some definite hotspots there so we'll zoom in and take a closer look.

{% highlight r %}
# more focussed map for larceny centre
zoom.centre = c(-122.41, 37.79)
zoom.map <- get_map(location=zoom.centre, zoom = 14)
{% endhighlight %}

{% highlight r %}
# Robbery/Theft Contours
ggmap(zoom.map, extent='device', legend="topleft") +
    stat_density2d(data = sf.p.lt,
                   aes(x = X, y = Y, fill = ..level.., alpha=..level..),
                   size = 0.1, colour="red", n=100,
                   geom = "polygon") +
    scale_alpha_continuous(range=c(0.2,0.4), guide='none') +
    scale_fill_gradient('Robbery\nTheft\nDensity')+
    ggtitle('Robbery/Theft Density in San Francisco')
{% endhighlight %}

![center](/figs/SFCrimeExplore/p-lt-contour-map-zoom-1.png)

# Central District

An even closer look at the central district

{% highlight r %}
# more focussed map for larceny centre
big.zoom.centre = c(-122.409, 37.786)
big.zoom.map <- get_map(location=zoom.centre, zoom = 15)
{% endhighlight %}

{% highlight r %}
# Robbery/Theft Contours
ggmap(big.zoom.map, extent='device', legend="topleft") +
    stat_density2d(data = sf.p.lt,
                   aes(x = X, y = Y, fill = ..level.., alpha=..level..),
                   size = 0.1, colour="red", n=200, geom = "polygon") +
    scale_alpha_continuous(range=c(0.05, 0.1), guide='none') +
    scale_fill_gradient('Robbery\nTheft\nDensity')+
    ggtitle('Robbery/Theft Density in San Francisco')
{% endhighlight %}

![center](/figs/SFCrimeExplore/p-lt-contour-map-big-zoom-1.png)

From the map it's clear that there are areas with elevated levels of robbery/theft, and with the shading mostly removed its possible to see that one of the strongest centres is the area area around Powel St Station.

# Robbery vs Theft Contours

Obviously no-one wants to be a victim of larceny/theft, but robbery (with violence or threat) is likely to be worse - do these two sets of crimes happen in different places?

Larceny contours are in blue, Robbery contours in red.

{% highlight r %}
# Larceny - Blue
# Robbery - Red
ggmap(big.zoom.map, extent='device') +
    stat_density2d(data = sf.p.lt %>% filter(Category=="Larceny/Theft"),
                   aes(x = X, y = Y, fill = ..level.., alpha=..level..),
                   colour="blue", n=200, geom = "polygon", bins=7) +
    stat_density2d(data = sf.p.lt %>% filter(Category=="Robbery"),
                   aes(x = X, y = Y, fill = ..level.., alpha=..level..),
                   colour="red", n=200, geom = "polygon", bins=7) +
    scale_alpha_continuous(range=c(0.0,0.1), guide='none') +
    scale_fill_gradient('Robbery\nTheft\nDensities') +
    ggtitle('Robbery/Theft Contours in San Francisco') +
    theme(legend.position = "none")
{% endhighlight %}

![center](/figs/SFCrimeExplore/rt-lt-contour-map-zoom-1.png)

From the overlapping contours it seems that these two categories of crimes happen in broadly the same areas with only minor differences.

The github repository for this project can be viewed [here](https://github.com/MarkPratley/SF-Crime).


text:
<?php

namespace Lime\Highlighter\Language\Tokenizer;

/**
 * Token Interface
 */
interface TokenInterface {
    /**
     * Create a new token with the given type and value
     *
     * @param int $type Token type
     * @param string $value Token value
     * @param int $offset Token offset in source code
     */
    public function __construct($type, $value, $offset);

    /**
     * Get the type
     * @return int
     */
    public function getType();

    /**
     * Get the value
     * @return string
     */
    public function getValue();

    /**
     * Set the token type
     * @param int $type Token type
     * @return $this
     */
    public function setType($type);

    /**
     * Set the token value
     * @param string $value Token value
     * @return $this
     */
    public function setValue($value);

    /**
     * Set the token offset
     *
     * @param int $offset Token offset
     * @return $this
     */
    public function setOffset($offset);

    /**
     * Get the token offset
     *
     * @return int
     */
    public function getOffset();
}


text:
from __future__ import print_function

import os
import time
import subprocess
import sys
import re
import datetime
import json
import requests

WAIT = 5

def try_mkdir(path):
    try:
        os.mkdir(path)
    except OSError:
        pass

def should_upload(basename):
    assert not '/' in basename, basename
    return not basename.startswith('.') and \
        (basename.endswith('.warc.gz') or basename.endswith('.json') or basename.endswith('.txt'))

def parse_name(basename):
    k = re.split(r'(.*)-\w+-(\d{8})-\d{6}-[^.]*\.warc.gz', basename) # extract domain name and date
    if len(k) != 4:
        return {'dns': 'UNKNOWN', 'date': datetime.datetime.now().strftime("%Y%m%d")}

    return {'dns': k[1], 'date': k[2]}

def ia_upload_allowed(s3_url, accesskey, bucket = ''):
    try:
        resp = requests.get(url=(s3_url + '/?check_limit=1&accesskey={}&bucket={}'.format(accesskey, bucket)))
        data = json.loads(resp.text)
    except Exception as err:
        print('Could not get throttling status - assuming IA is down')
        return False

    if 'over_limit' in data and data['over_limit'] is not 0:
        print('IA S3 API notifies us we are being throttled (over_limit)')
        return False

    if 'detail' in data and 'rationing_engaged' in data['detail'] \
       and data['detail']['rationing_engaged'] is not 0:
        quota_our_remaining = data['detail']['accesskey_ration'] - data['detail']['accesskey_tasks_queued']
        quota_global_remaining = data['detail']['total_global_limit'] - data['detail']['total_tasks_queued']
        quota_bucket_remaining = data['detail']['bucket_ration'] - data['detail']['bucket_tasks_queued']
        if quota_our_remaining < 10 or quota_global_remaining < 10 or quota_bucket_remaining < 5:
            print('IA S3 API notifies us rationing is engaged with little room for new work!')
            print('Our outstanding jobs:   {}'.format(data['detail']['accesskey_tasks_queued']))
            print('Our remaining quota:    {}'.format(quota_our_remaining))
            print('Global remaining quota: {}'.format(quota_global_remaining))
            print('Limit reason given: {}'.format(data['detail']['limit_reason']))
            return False
        else:
            print('IA S3 API notifies us rationing is engaged but we have '
                  'room for another job.')

    return True

def main():
    if len(sys.argv) > 1:
        directory = sys.argv[1]
    elif os.environ.get('FINISHED_WARCS_DIR') != None:
        directory = os.environ['FINISHED_WARCS_DIR']
    else:
        raise RuntimeError('No directory specified (set FINISHED_WARCS_DIR '
                           'or specify directory on command line)')

    mode = None #modes: 'rsync', 's3'

    url = os.environ.get('RSYNC_URL')
    if url != None:
        if '/localhost' in url or '/127.' in url:
            raise RuntimeError('Won\'t let you upload to localhost because I '
                               'remove files after uploading them, and you '
                               'might be uploading to the same directory')
        mode = 'rsync'

    if url is None:
        url = os.environ.get('S3_URL')
        if url is not None:
            mode = 's3'

    if url is None:
        raise RuntimeError('Neither RSYNC_URL nor S3_URL are set - nowhere to '
                           'upload to.  Hint: use'
                           'S3_URL=https://s3.us.archive.org')

    if mode == 's3': #parse IA-S3-specific options
        ia_collection = os.environ.get('IA_COLLECTION')
        if ia_collection is None:
            raise RuntimeError('Must specify IA_COLLECTION if using IA S3 '
                               '(hint: ArchiveBot)')

        ia_item_title = os.environ.get('IA_ITEM_TITLE')
        if ia_item_title is None:
            raise RuntimeError('Must specify IA_ITEM_TITLE if using IA S3 '
                               '(hint: "Archiveteam: Archivebot $pipeline_name '
                               'GO Pack")')

        ia_auth = os.environ.get('IA_AUTH')
        if ia_auth is None:
            raise RuntimeError('Must specify IA_AUTH if using IA S3 '
                               '(hint: access_key:secret_key)')

        ia_item_prefix = os.environ.get('IA_ITEM_PREFIX')
        if ia_auth is None:
            raise RuntimeError('Must specify IA_ITEM_PREFIX if using IA S3 '
                               '(hint: archiveteam_archivebot_go_$pipeline_name'
                               '_}')

        ia_access = os.environ.get('IA_ACCESS')
        if ia_access is None:
            raise RuntimeError('Must specify IA_ACCESS if using IA S3 '
                               '(hint: your access key)')

    print("CHECK THE UPLOAD TARGET: %s as %s endpoint" % (url, mode))
    print()
    print("Upload target must reliably store data")
    print("Each local file will removed after upload")
    print("Hit CTRL-C immediately if upload target is incorrect")
    print()

    uploading_dir = os.path.join(directory, "_uploading")
    try_mkdir(uploading_dir)

    while True:
        print("Waiting %d seconds" % (WAIT,))
        time.sleep(WAIT)

        fnames = sorted(list(f for f in os.listdir(directory) if should_upload(f)))
        if len(fnames):
            basename = fnames[0]
            fname_d = os.path.join(directory, basename)
            fname_u = os.path.join(uploading_dir, basename)
            if os.path.exists(fname_u):
                print("%r already exists - another uploader probably grabbed it" % (fname_u,))
                continue
            try:
                os.rename(fname_d, fname_u)
            except OSError:
                print("Could not rename %r - another uploader probably grabbed it" % (fname_d,))
            else:
                print("Uploading %r" % (fname_u,))

                item = parse_name(basename)

                if mode == 'rsync':
                    exit_code = subprocess.call([
                        "rsync", "-av", "--timeout=300", "--contimeout=300",
                        "--progress", fname_u, url])
                elif mode == 's3':
                    ia_upload_bucket = re.sub(r'[^0-9a-zA-Z-]+', '_', ia_item_prefix + '_' + item['dns'] + '_' + item['date'])
                    if ia_upload_allowed(url, ia_access, ia_upload_bucket): # IA is not throttling
                        # At some point, an ambitious person could try a file belonging in a different bucket if ia_upload_allowed denied this one
                        size_hint = str(os.stat(fname_u).st_size)
                        target = url + '/' + ia_upload_bucket + '/' + \
                                 re.sub(r'[^0-9a-zA-Z-.]+', '_', basename)

                        exit_code = subprocess.call([
                            "curl", "-v", "--location", "--fail",
                            "--speed-limit", "1", "--speed-time", "900",
                            "--header", "x-archive-queue-derive:1",
                            "--header", "x-amz-auto-make-bucket:1",
                            "--header", "x-archive-meta-collection:" + ia_collection,
                            "--header", "x-archive-meta-mediatype:web",
                            "--header", "x-archive-meta-subject:archivebot",
                            "--header", "x-archive-meta-title:" + ia_item_title +
                            ' ' + item['dns'] + ' ' + item['date'],
                            "--header", "x-archive-meta-date:" +
                            item['date'][0:4] + '-' +
                            item['date'][4:6] + '-' +
                            item['date'][6:8],
                            "--header", "x-archive-size-hint:" + size_hint,
                            "--header", "authorization: LOW " + ia_auth,
                            "-o", "/dev/stdout",
                            "--upload-file", fname_u,
                            target])
                    else: # Cannot upload now, try again later
                        exit_code = 1
                else: #no upload mechanism available
                    exit_code = 1

                if exit_code == 0:
                    print("Removing %r" % (fname_u,))
                    os.remove(fname_u)
                else:
                    # Move it out of the _uploading directory so that this
                    # uploader (or another one) can try again.
                    os.rename(fname_u, fname_d)
        else:
            print("Nothing to upload")

if __name__ == '__main__':
    main()


text:
layout: job-posting
type: section
title: Policy Analyst (Co-op)
description: >-
  The Policy Team at CDS is looking for a Policy Analyst to support digital
  service delivery teams and to contribute to the strategy of the organization
  (Fall 2019 student position).
archived: true
leverId: d81198b9-3536-4a4d-a370-576b94ac7f2a
translationKey: policy-analyst-coop
---
We're transforming government. Our products will impact people's everyday lives. It's an exciting thing to be part of. We need you.

The Canadian Digital Service is focused on delivering better public services using the best technology and design, all informed by people's needs. We hire people of all levels of experience.

The Policy team at CDS helps bridge the gap between delivery and public policy. We help the digital service delivery teams at CDS navigate government processes and policies during product development, and we provide strategic advice to support CDS's mandate – to help change government to serve people better. You can learn more about our team [here](
https://digital.canada.ca/2018/09/07/policy/).

We're looking for a Policy Analyst as a Fall 2019 student position. In this role, you will have the opportunity to contribute and learn by:

* Diving into domestic and international practices around privacy, research, communication, administration of government, and technology
* Researching, analyzing, and providing critical feedback on government policies, processes, legislation or regulations
* Working with and learning from multidisciplinary talent like designers, engineers, researchers and others in helping digital delivery teams navigate government processes and problem solve
* Helping measure the performance and impact of digital services and products
* Understanding the impact of technology and design changes on how government works
* Meeting with stakeholders and experts in other parts of government and elsewhere

Don't worry if you only see yourself in some of the items on this list. Each member of the team specializes in different areas, and you'll be able to learn about areas that you're less familiar with.

## You'll fit right in if:

* Your default mode is open, friendly, and collaborative.
* You like to learn new things and help people out.
* You're organized, detail-oriented, and like to get stuff done.
* You have experience communicating through email in a concise and professional manner.
* You know that sometimes just trying something is the best way to find out if it'll work.
* You are comfortable with technology and have experience with web-based collaboration tools like Google Docs, Trello, Slack, etc.
* You're (ideally) fluent in both official languages
* You like working hard and having fun while doing it :)

## At CDS, we don't just accept difference - we celebrate it.

We proudly, passionately, and actively strive to make CDS more reflective and inclusive of the society that we serve. We will only be able to deliver better, more accessible, and more inclusive public services if we harness diverse thoughts, experiences, and skills. We work hard to create an environment where different perspectives and experiences are valued. We are committed to helping diverse talent thrive.

CDS welcomes all applicants regardless of race, ethnicity, religion, sexual orientation, gender identity or expression, national origin, disability, age, Veteran status, marital status, pregnancy or family commitments.


text:
/* global describe it beforeEach*/

import Dashboard from '../../../src/components/ui/Dashboard';
import {expect} from 'chai';
import jsdom from 'jsdom';
import {mount} from 'enzyme';
import React from 'react';
import sinon from 'sinon';

// https://github.com/airbnb/enzyme/issues/341
const doc = jsdom.jsdom('<!doctype html>');
global.document = doc;
global.window = doc.defaultView;

describe('<Dashboard/>', () => {
  let wrapper;
  beforeEach((done) => {
    const protectedTestSpy = sinon.spy();
    wrapper = mount(<Dashboard protectedTest={protectedTestSpy}/> );
    done();
  });

  it('calls protectedTest', (done) => {
    expect(wrapper.props().protectedTest.calledOnce).to.equal(true);
    done();
  });

  it('renders content', (done) => {
    const content = 'this is some content';
    wrapper.setProps({content: content});
    expect(wrapper.find('p').first().text()).to.equal(content);
    done();
  });
});