text:
package consul

import (
	"crypto/tls"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/rpc"
	"os"
	"path/filepath"
	"reflect"
	"strconv"
	"sync"
	"time"

	"github.com/hashicorp/consul/acl"
	"github.com/hashicorp/consul/consul/agent"
	"github.com/hashicorp/consul/consul/state"
	"github.com/hashicorp/consul/consul/structs"
	"github.com/hashicorp/consul/tlsutil"
	"github.com/hashicorp/raft"
	"github.com/hashicorp/raft-boltdb"
	"github.com/hashicorp/serf/coordinate"
	"github.com/hashicorp/serf/serf"
)

// These are the protocol versions that Consul can _understand_. These are
// Consul-level protocol versions, that are used to configure the Serf
// protocol versions.
const (
	ProtocolVersionMin uint8 = 2

	// Version 3 added support for network coordinates but we kept the
	// default protocol version at 2 to ease the transition to this new
	// feature. A Consul agent speaking version 2 of the protocol will
	// attempt to send its coordinates to a server who understands version
	// 3 or greater.
	ProtocolVersion2Compatible = 2

	ProtocolVersionMax = 3
)

const (
	serfLANSnapshot   = "serf/local.snapshot"
	serfWANSnapshot   = "serf/remote.snapshot"
	raftState         = "raft/"
	snapshotsRetained = 2

	// serverRPCCache controls how long we keep an idle connection
	// open to a server
	serverRPCCache = 2 * time.Minute

	// serverMaxStreams controls how many idle streams we keep
	// open to a server
	serverMaxStreams = 64

	// raftLogCacheSize is the maximum number of logs to cache in-memory.
	// This is used to reduce disk I/O for the recently committed entries.
	raftLogCacheSize = 512

	// raftRemoveGracePeriod is how long we wait to allow a RemovePeer
	// to replicate to gracefully leave the cluster.
	raftRemoveGracePeriod = 5 * time.Second
)

// Server is Consul server which manages the service discovery,
// health checking, DC forwarding, Raft, and multiple Serf pools.
type Server struct {
	// aclAuthCache is the authoritative ACL cache.
	aclAuthCache *acl.Cache

	// aclCache is the non-authoritative ACL cache.
	aclCache *aclCache

	// Consul configuration
	config *Config

	// Connection pool to other consul servers
	connPool *ConnPool

	// Endpoints holds our RPC endpoints
	endpoints endpoints

	// eventChLAN is used to receive events from the
	// serf cluster in the datacenter
	eventChLAN chan serf.Event

	// eventChWAN is used to receive events from the
	// serf cluster that spans datacenters
	eventChWAN chan serf.Event

	// fsm is the state machine used with Raft to provide
	// strong consistency.
	fsm *consulFSM

	// localConsuls is used to track the known consuls
	// in the local datacenter. Used to do leader forwarding.
	localConsuls map[raft.ServerAddress]*agent.Server
	localLock    sync.RWMutex

	// Logger uses the provided LogOutput
	logger *log.Logger

	// The raft instance is used among Consul nodes within the DC to protect
	// operations that require strong consistency.
	// the state directly.
	raft          *raft.Raft
	raftLayer     *RaftLayer
	raftStore     *raftboltdb.BoltStore
	raftTransport *raft.NetworkTransport
	raftInmem     *raft.InmemStore

	// reconcileCh is used to pass events from the serf handler
	// into the leader manager, so that the strong state can be
	// updated
	reconcileCh chan serf.Member

	// remoteConsuls is used to track the known consuls in
	// remote datacenters. Used to do DC forwarding.
	remoteConsuls map[string][]*agent.Server
	remoteLock    sync.RWMutex

	// rpcListener is used to listen for incoming connections
	rpcListener net.Listener
	rpcServer   *rpc.Server

	// rpcTLS is the TLS config for incoming TLS requests
	rpcTLS *tls.Config

	// serfLAN is the Serf cluster maintained inside the DC
	// which contains all the DC nodes
	serfLAN *serf.Serf

	// serfWAN is the Serf cluster maintained between DC's
	// which SHOULD only consist of Consul servers
	serfWAN *serf.Serf

	// sessionTimers track the expiration time of each Session that has
	// a TTL. On expiration, a SessionDestroy event will occur, and
	// destroy the session via standard session destroy processing
	sessionTimers     map[string]*time.Timer
	sessionTimersLock sync.Mutex

	// tombstoneGC is used to track the pending GC invocations
	// for the KV tombstones
	tombstoneGC *state.TombstoneGC

	// aclReplicationStatus (and its associated lock) provide information
	// about the health of the ACL replication goroutine.
	aclReplicationStatus     structs.ACLReplicationStatus
	aclReplicationStatusLock sync.RWMutex

	// shutdown and the associated members here are used in orchestrating
	// a clean shutdown. The shutdownCh is never written to, only closed to
	// indicate a shutdown has been initiated.
	shutdown     bool
	shutdownCh   chan struct{}
	shutdownLock sync.Mutex
}

// Holds the RPC endpoints
type endpoints struct {
	Catalog       *Catalog
	Health        *Health
	Status        *Status
	KVS           *KVS
	Session       *Session
	Internal      *Internal
	ACL           *ACL
	Coordinate    *Coordinate
	PreparedQuery *PreparedQuery
	Txn           *Txn
}

// NewServer is used to construct a new Consul server from the
// configuration, potentially returning an error
func NewServer(config *Config) (*Server, error) {
	// Check the protocol version.
	if err := config.CheckVersion(); err != nil {
		return nil, err
	}

	// Check for a data directory.
	if config.DataDir == "" && !config.DevMode {
		return nil, fmt.Errorf("Config must provide a DataDir")
	}

	// Sanity check the ACLs.
	if err := config.CheckACL(); err != nil {
		return nil, err
	}

	// Ensure we have a log output and create a logger.
	if config.LogOutput == nil {
		config.LogOutput = os.Stderr
	}
	logger := log.New(config.LogOutput, "", log.LstdFlags)

	// Create the TLS wrapper for outgoing connections.
	tlsConf := config.tlsConfig()
	tlsWrap, err := tlsConf.OutgoingTLSWrapper()
	if err != nil {
		return nil, err
	}

	// Get the incoming TLS config.
	incomingTLS, err := tlsConf.IncomingTLSConfig()
	if err != nil {
		return nil, err
	}

	// Create the tombstone GC.
	gc, err := state.NewTombstoneGC(config.TombstoneTTL, config.TombstoneTTLGranularity)
	if err != nil {
		return nil, err
	}

	// Create server.
	s := &Server{
		config:        config,
		connPool:      NewPool(config.LogOutput, serverRPCCache, serverMaxStreams, tlsWrap),
		eventChLAN:    make(chan serf.Event, 256),
		eventChWAN:    make(chan serf.Event, 256),
		localConsuls:  make(map[raft.ServerAddress]*agent.Server),
		logger:        logger,
		reconcileCh:   make(chan serf.Member, 32),
		remoteConsuls: make(map[string][]*agent.Server, 4),
		rpcServer:     rpc.NewServer(),
		rpcTLS:        incomingTLS,
		tombstoneGC:   gc,
		shutdownCh:    make(chan struct{}),
	}

	// Initialize the authoritative ACL cache.
	s.aclAuthCache, err = acl.NewCache(aclCacheSize, s.aclLocalFault)
	if err != nil {
		s.Shutdown()
		return nil, fmt.Errorf("Failed to create authoritative ACL cache: %v", err)
	}

	// Set up the non-authoritative ACL cache. A nil local function is given
	// if ACL replication isn't enabled.
	var local acl.FaultFunc
	if s.IsACLReplicationEnabled() {
		local = s.aclLocalFault
	}
	if s.aclCache, err = newAclCache(config, logger, s.RPC, local); err != nil {
		s.Shutdown()
		return nil, fmt.Errorf("Failed to create non-authoritative ACL cache: %v", err)
	}

	// Initialize the RPC layer.
	if err := s.setupRPC(tlsWrap); err != nil {
		s.Shutdown()
		return nil, fmt.Errorf("Failed to start RPC layer: %v", err)
	}

	// Initialize the Raft server.
	if err := s.setupRaft(); err != nil {
		s.Shutdown()
		return nil, fmt.Errorf("Failed to start Raft: %v", err)
	}

	// Initialize the LAN Serf.
	s.serfLAN, err = s.setupSerf(config.SerfLANConfig,
		s.eventChLAN, serfLANSnapshot, false)
	if err != nil {
		s.Shutdown()
		return nil, fmt.Errorf("Failed to start lan serf: %v", err)
	}
	go s.lanEventHandler()

	// Initialize the WAN Serf.
	s.serfWAN, err = s.setupSerf(config.SerfWANConfig,
		s.eventChWAN, serfWANSnapshot, true)
	if err != nil {
		s.Shutdown()
		return nil, fmt.Errorf("Failed to start wan serf: %v", err)
	}
	go s.wanEventHandler()

	// Start ACL replication.
	if s.IsACLReplicationEnabled() {
		go s.runACLReplication()
	}

	// Start listening for RPC requests.
	go s.listen()

	// Start the metrics handlers.
	go s.sessionStats()

	return s, nil
}

// setupSerf is used to setup and initialize a Serf
func (s *Server) setupSerf(conf *serf.Config, ch chan serf.Event, path string, wan bool) (*serf.Serf, error) {
	addr := s.rpcListener.Addr().(*net.TCPAddr)
	conf.Init()
	if wan {
		conf.NodeName = fmt.Sprintf("%s.%s", s.config.NodeName, s.config.Datacenter)
	} else {
		conf.NodeName = s.config.NodeName
	}
	conf.Tags["role"] = "consul"
	conf.Tags["dc"] = s.config.Datacenter
	conf.Tags["vsn"] = fmt.Sprintf("%d", s.config.ProtocolVersion)
	conf.Tags["vsn_min"] = fmt.Sprintf("%d", ProtocolVersionMin)
	conf.Tags["vsn_max"] = fmt.Sprintf("%d", ProtocolVersionMax)
	conf.Tags["build"] = s.config.Build
	conf.Tags["port"] = fmt.Sprintf("%d", addr.Port)
	if s.config.Bootstrap {
		conf.Tags["bootstrap"] = "1"
	}
	if s.config.BootstrapExpect != 0 {
		conf.Tags["expect"] = fmt.Sprintf("%d", s.config.BootstrapExpect)
	}
	conf.MemberlistConfig.LogOutput = s.config.LogOutput
	conf.LogOutput = s.config.LogOutput
	conf.EventCh = ch
	if !s.config.DevMode {
		conf.SnapshotPath = filepath.Join(s.config.DataDir, path)
	}
	conf.ProtocolVersion = protocolVersionMap[s.config.ProtocolVersion]
	conf.RejoinAfterLeave = s.config.RejoinAfterLeave
	if wan {
		conf.Merge = &wanMergeDelegate{}
	} else {
		conf.Merge = &lanMergeDelegate{dc: s.config.Datacenter}
	}

	// Until Consul supports this fully, we disable automatic resolution.
	// When enabled, the Serf gossip may just turn off if we are the minority
	// node which is rather unexpected.
	conf.EnableNameConflictResolution = false
	if err := ensurePath(conf.SnapshotPath, false); err != nil {
		return nil, err
	}

	// Plumb down the enable coordinates flag.
	conf.DisableCoordinates = s.config.DisableCoordinates

	return serf.Create(conf)
}

// setupRaft is used to setup and initialize Raft
func (s *Server) setupRaft() error {
	// If we have an unclean exit then attempt to close the Raft store.
	defer func() {
		if s.raft == nil && s.raftStore != nil {
			if err := s.raftStore.Close(); err != nil {
				s.logger.Printf("[ERR] consul: failed to close Raft store: %v", err)
			}
		}
	}()

	// Create the FSM.
	var err error
	s.fsm, err = NewFSM(s.tombstoneGC, s.config.LogOutput)
	if err != nil {
		return err
	}

	// Create a transport layer.
	trans := raft.NewNetworkTransport(s.raftLayer, 3, 10*time.Second, s.config.LogOutput)
	s.raftTransport = trans

	// Make sure we set the LogOutput.
	s.config.RaftConfig.LogOutput = s.config.LogOutput

	// Our version of Raft protocol requires the LocalID to match the network
	// address of the transport.
	s.config.RaftConfig.LocalID = raft.ServerID(trans.LocalAddr())

	// Build an all in-memory setup for dev mode, otherwise prepare a full
	// disk-based setup.
	var log raft.LogStore
	var stable raft.StableStore
	var snap raft.SnapshotStore
	if s.config.DevMode {
		store := raft.NewInmemStore()
		s.raftInmem = store
		stable = store
		log = store
		snap = raft.NewDiscardSnapshotStore()
	} else {
		// Create the base raft path.
		path := filepath.Join(s.config.DataDir, raftState)
		if err := ensurePath(path, true); err != nil {
			return err
		}

		// Create the backend raft store for logs and stable storage.
		store, err := raftboltdb.NewBoltStore(filepath.Join(path, "raft.db"))
		if err != nil {
			return err
		}
		s.raftStore = store
		stable = store

		// Wrap the store in a LogCache to improve performance.
		cacheStore, err := raft.NewLogCache(raftLogCacheSize, store)
		if err != nil {
			return err
		}
		log = cacheStore

		// Create the snapshot store.
		snapshots, err := raft.NewFileSnapshotStore(path, snapshotsRetained, s.config.LogOutput)
		if err != nil {
			return err
		}
		snap = snapshots

		// For an existing cluster being upgraded to the new version of
		// Raft, we almost never want to run recovery based on the old
		// peers.json file. We create a peers.info file with a helpful
		// note about where peers.json went, and use that as a sentinel
		// to avoid ingesting the old one that first time (if we have to
		// create the peers.info file because it's not there, we also
		// blow away any existing peers.json file).
		peersFile := filepath.Join(path, "peers.json")
		peersInfoFile := filepath.Join(path, "peers.info")
		if _, err := os.Stat(peersInfoFile); os.IsNotExist(err) {
			if err := ioutil.WriteFile(peersInfoFile, []byte(peersInfoContent), 0755); err != nil {
				return fmt.Errorf("failed to write peers.info file: %v", err)
			}

			// Blow away the peers.json file if present, since the
			// peers.info sentinel wasn't there.
			if _, err := os.Stat(peersFile); err == nil {
				if err := os.Remove(peersFile); err != nil {
					return fmt.Errorf("failed to delete peers.json, please delete manually (see peers.info for details): %v", err)
				}
				s.logger.Printf("[INFO] consul: deleted peers.json file (see peers.info for details)")
			}
		} else if _, err := os.Stat(peersFile); err == nil {
			s.logger.Printf("[INFO] consul: found peers.json file, recovering Raft configuration...")
			configuration, err := raft.ReadPeersJSON(peersFile)
			if err != nil {
				return fmt.Errorf("recovery failed to parse peers.json: %v", err)
			}
			tmpFsm, err := NewFSM(s.tombstoneGC, s.config.LogOutput)
			if err != nil {
				return fmt.Errorf("recovery failed to make temp FSM: %v", err)
			}
			if err := raft.RecoverCluster(s.config.RaftConfig, tmpFsm,
				log, stable, snap, trans, configuration); err != nil {
				return fmt.Errorf("recovery failed: %v", err)
			}
			if err := os.Remove(peersFile); err != nil {
				return fmt.Errorf("recovery failed to delete peers.json, please delete manually (see peers.info for details): %v", err)
			}
			s.logger.Printf("[INFO] consul: deleted peers.json file after successful recovery")
		}
	}

	// If we are in bootstrap or dev mode and the state is clean then we can
	// bootstrap now.
	if s.config.Bootstrap || s.config.DevMode {
		hasState, err := raft.HasExistingState(log, stable, snap)
		if err != nil {
			return err
		}
		if !hasState {
			// TODO (slackpad) - This will need to be updated when
			// we add support for node IDs.
			configuration := raft.Configuration{
				Servers: []raft.Server{
					raft.Server{
						ID:      raft.ServerID(trans.LocalAddr()),
						Address: trans.LocalAddr(),
					},
				},
			}
			if err := raft.BootstrapCluster(s.config.RaftConfig,
				log, stable, snap, trans, configuration); err != nil {
				return err
			}
		}
	}

	// Setup the Raft store.
	s.raft, err = raft.NewRaft(s.config.RaftConfig, s.fsm, log, stable, snap, trans)
	if err != nil {
		return err
	}

	// Start monitoring leadership.
	go s.monitorLeadership()
	return nil
}

// setupRPC is used to setup the RPC listener
func (s *Server) setupRPC(tlsWrap tlsutil.DCWrapper) error {
	// Create endpoints
	s.endpoints.Status = &Status{s}
	s.endpoints.Catalog = &Catalog{s}
	s.endpoints.Health = &Health{s}
	s.endpoints.KVS = &KVS{s}
	s.endpoints.Session = &Session{s}
	s.endpoints.Internal = &Internal{s}
	s.endpoints.ACL = &ACL{s}
	s.endpoints.Coordinate = NewCoordinate(s)
	s.endpoints.PreparedQuery = &PreparedQuery{s}
	s.endpoints.Txn = &Txn{s}

	// Register the handlers
	s.rpcServer.Register(s.endpoints.Status)
	s.rpcServer.Register(s.endpoints.Catalog)
	s.rpcServer.Register(s.endpoints.Health)
	s.rpcServer.Register(s.endpoints.KVS)
	s.rpcServer.Register(s.endpoints.Session)
	s.rpcServer.Register(s.endpoints.Internal)
	s.rpcServer.Register(s.endpoints.ACL)
	s.rpcServer.Register(s.endpoints.Coordinate)
	s.rpcServer.Register(s.endpoints.PreparedQuery)
	s.rpcServer.Register(s.endpoints.Txn)

	list, err := net.ListenTCP("tcp", s.config.RPCAddr)
	if err != nil {
		return err
	}
	s.rpcListener = list

	var advertise net.Addr
	if s.config.RPCAdvertise != nil {
		advertise = s.config.RPCAdvertise
	} else {
		advertise = s.rpcListener.Addr()
	}

	// Verify that we have a usable advertise address
	addr, ok := advertise.(*net.TCPAddr)
	if !ok {
		list.Close()
		return fmt.Errorf("RPC advertise address is not a TCP Address: %v", addr)
	}
	if addr.IP.IsUnspecified() {
		list.Close()
		return fmt.Errorf("RPC advertise address is not advertisable: %v", addr)
	}

	// Provide a DC specific wrapper. Raft replication is only
	// ever done in the same datacenter, so we can provide it as a constant.
	wrapper := tlsutil.SpecificDC(s.config.Datacenter, tlsWrap)
	s.raftLayer = NewRaftLayer(advertise, wrapper)
	return nil
}

// Shutdown is used to shutdown the server
func (s *Server) Shutdown() error {
	s.logger.Printf("[INFO] consul: shutting down server")
	s.shutdownLock.Lock()
	defer s.shutdownLock.Unlock()

	if s.shutdown {
		return nil
	}

	s.shutdown = true
	close(s.shutdownCh)

	if s.serfLAN != nil {
		s.serfLAN.Shutdown()
	}

	if s.serfWAN != nil {
		s.serfWAN.Shutdown()
	}

	if s.raft != nil {
		s.raftTransport.Close()
		s.raftLayer.Close()
		future := s.raft.Shutdown()
		if err := future.Error(); err != nil {
			s.logger.Printf("[WARN] consul: error shutting down raft: %s", err)
		}
		if s.raftStore != nil {
			s.raftStore.Close()
		}
	}

	if s.rpcListener != nil {
		s.rpcListener.Close()
	}

	// Close the connection pool
	s.connPool.Shutdown()

	return nil
}

// Leave is used to prepare for a graceful shutdown of the server
func (s *Server) Leave() error {
	s.logger.Printf("[INFO] consul: server starting leave")

	// Check the number of known peers
	numPeers, err := s.numPeers()
	if err != nil {
		s.logger.Printf("[ERR] consul: failed to check raft peers: %v", err)
		return err
	}

	// TODO (slackpad) - This will need to be updated once we support node
	// IDs.
	addr := s.raftTransport.LocalAddr()

	// If we are the current leader, and we have any other peers (cluster has multiple
	// servers), we should do a RemovePeer to safely reduce the quorum size. If we are
	// not the leader, then we should issue our leave intention and wait to be removed
	// for some sane period of time.
	isLeader := s.IsLeader()
	if isLeader && numPeers > 1 {
		future := s.raft.RemovePeer(addr)
		if err := future.Error(); err != nil {
			s.logger.Printf("[ERR] consul: failed to remove ourself as raft peer: %v", err)
		}
	}

	// Leave the WAN pool
	if s.serfWAN != nil {
		if err := s.serfWAN.Leave(); err != nil {
			s.logger.Printf("[ERR] consul: failed to leave WAN Serf cluster: %v", err)
		}
	}

	// Leave the LAN pool
	if s.serfLAN != nil {
		if err := s.serfLAN.Leave(); err != nil {
			s.logger.Printf("[ERR] consul: failed to leave LAN Serf cluster: %v", err)
		}
	}

	// If we were not leader, wait to be safely removed from the cluster. We
	// must wait to allow the raft replication to take place, otherwise an
	// immediate shutdown could cause a loss of quorum.
	if !isLeader {
		left := false
		limit := time.Now().Add(raftRemoveGracePeriod)
		for !left && time.Now().Before(limit) {
			// Sleep a while before we check.
			time.Sleep(50 * time.Millisecond)

			// Get the latest configuration.
			future := s.raft.GetConfiguration()
			if err := future.Error(); err != nil {
				s.logger.Printf("[ERR] consul: failed to get raft configuration: %v", err)
				break
			}

			// See if we are no longer included.
			left = true
			for _, server := range future.Configuration().Servers {
				if server.Address == addr {
					left = false
					break
				}
			}
		}

		// TODO (slackpad) With the old Raft library we used to force the
		// peers set to empty when a graceful leave occurred. This would
		// keep voting spam down if the server was restarted, but it was
		// dangerous because the peers was inconsistent with the logs and
		// snapshots, so it wasn't really safe in all cases for the server
		// to become leader. This is now safe, but the log spam is noisy.
		// The next new version of the library will have a "you are not a
		// peer stop it" behavior that should address this. We will have
		// to evaluate during the RC period if this interim situation is
		// not too confusing for operators.

		// TODO (slackpad) When we take a later new version of the Raft
		// library it won't try to complete replication, so this peer
		// may not realize that it has been removed. Need to revisit this
		// and the warning here.
		if !left {
			s.logger.Printf("[WARN] consul: failed to leave raft configuration gracefully, timeout")
		}
	}

	return nil
}

// numPeers is used to check on the number of known peers, including the local
// node.
func (s *Server) numPeers() (int, error) {
	future := s.raft.GetConfiguration()
	if err := future.Error(); err != nil {
		return 0, err
	}
	configuration := future.Configuration()
	return len(configuration.Servers), nil
}

// JoinLAN is used to have Consul join the inner-DC pool
// The target address should be another node inside the DC
// listening on the Serf LAN address
func (s *Server) JoinLAN(addrs []string) (int, error) {
	return s.serfLAN.Join(addrs, true)
}

// JoinWAN is used to have Consul join the cross-WAN Consul ring
// The target address should be another node listening on the
// Serf WAN address
func (s *Server) JoinWAN(addrs []string) (int, error) {
	return s.serfWAN.Join(addrs, true)
}

// LocalMember is used to return the local node
func (c *Server) LocalMember() serf.Member {
	return c.serfLAN.LocalMember()
}

// LANMembers is used to return the members of the LAN cluster
func (s *Server) LANMembers() []serf.Member {
	return s.serfLAN.Members()
}

// WANMembers is used to return the members of the LAN cluster
func (s *Server) WANMembers() []serf.Member {
	return s.serfWAN.Members()
}

// RemoveFailedNode is used to remove a failed node from the cluster
func (s *Server) RemoveFailedNode(node string) error {
	if err := s.serfLAN.RemoveFailedNode(node); err != nil {
		return err
	}
	if err := s.serfWAN.RemoveFailedNode(node); err != nil {
		return err
	}
	return nil
}

// IsLeader checks if this server is the cluster leader
func (s *Server) IsLeader() bool {
	return s.raft.State() == raft.Leader
}

// KeyManagerLAN returns the LAN Serf keyring manager
func (s *Server) KeyManagerLAN() *serf.KeyManager {
	return s.serfLAN.KeyManager()
}

// KeyManagerWAN returns the WAN Serf keyring manager
func (s *Server) KeyManagerWAN() *serf.KeyManager {
	return s.serfWAN.KeyManager()
}

// Encrypted determines if gossip is encrypted
func (s *Server) Encrypted() bool {
	return s.serfLAN.EncryptionEnabled() && s.serfWAN.EncryptionEnabled()
}

// inmemCodec is used to do an RPC call without going over a network
type inmemCodec struct {
	method string
	args   interface{}
	reply  interface{}
	err    error
}

func (i *inmemCodec) ReadRequestHeader(req *rpc.Request) error {
	req.ServiceMethod = i.method
	return nil
}

func (i *inmemCodec) ReadRequestBody(args interface{}) error {
	sourceValue := reflect.Indirect(reflect.Indirect(reflect.ValueOf(i.args)))
	dst := reflect.Indirect(reflect.Indirect(reflect.ValueOf(args)))
	dst.Set(sourceValue)
	return nil
}

func (i *inmemCodec) WriteResponse(resp *rpc.Response, reply interface{}) error {
	if resp.Error != "" {
		i.err = errors.New(resp.Error)
		return nil
	}
	sourceValue := reflect.Indirect(reflect.Indirect(reflect.ValueOf(reply)))
	dst := reflect.Indirect(reflect.Indirect(reflect.ValueOf(i.reply)))
	dst.Set(sourceValue)
	return nil
}

func (i *inmemCodec) Close() error {
	return nil
}

// RPC is used to make a local RPC call
func (s *Server) RPC(method string, args interface{}, reply interface{}) error {
	codec := &inmemCodec{
		method: method,
		args:   args,
		reply:  reply,
	}
	if err := s.rpcServer.ServeRequest(codec); err != nil {
		return err
	}
	return codec.err
}

// InjectEndpoint is used to substitute an endpoint for testing.
func (s *Server) InjectEndpoint(endpoint interface{}) error {
	s.logger.Printf("[WARN] consul: endpoint injected; this should only be used for testing")
	return s.rpcServer.Register(endpoint)
}

// Stats is used to return statistics for debugging and insight
// for various sub-systems
func (s *Server) Stats() map[string]map[string]string {
	toString := func(v uint64) string {
		return strconv.FormatUint(v, 10)
	}
	s.remoteLock.RLock()
	numKnownDCs := len(s.remoteConsuls)
	s.remoteLock.RUnlock()
	stats := map[string]map[string]string{
		"consul": map[string]string{
			"server":            "true",
			"leader":            fmt.Sprintf("%v", s.IsLeader()),
			"leader_addr":       string(s.raft.Leader()),
			"bootstrap":         fmt.Sprintf("%v", s.config.Bootstrap),
			"known_datacenters": toString(uint64(numKnownDCs)),
		},
		"raft":     s.raft.Stats(),
		"serf_lan": s.serfLAN.Stats(),
		"serf_wan": s.serfWAN.Stats(),
		"runtime":  runtimeStats(),
	}
	return stats
}

// GetLANCoordinate returns the coordinate of the server in the LAN gossip pool.
func (s *Server) GetLANCoordinate() (*coordinate.Coordinate, error) {
	return s.serfLAN.GetCoordinate()
}

// GetWANCoordinate returns the coordinate of the server in the WAN gossip pool.
func (s *Server) GetWANCoordinate() (*coordinate.Coordinate, error) {
	return s.serfWAN.GetCoordinate()
}

// peersInfoContent is used to help operators understand what happened to the
// peers.json file. This is written to a file called peers.info in the same
// location.
const peersInfoContent = `
As of Consul 0.7.0, the peers.json file is only used for recovery
after an outage. It should be formatted as a JSON array containing the address
and port of each Consul server in the cluster, like this:

["10.1.0.1:8500","10.1.0.2:8500","10.1.0.3:8500"]

Under normal operation, the peers.json file will not be present.

When Consul starts for the first time, it will create this peers.info file and
delete any existing peers.json file so that recovery doesn't occur on the first
startup.

Once this peers.info file is present, any peers.json file will be ingested at
startup, and will set the Raft peer configuration manually to recover from an
outage. It's crucial that all servers in the cluster are shut down before
creating the peers.json file, and that all servers receive the same
configuration. Once the peers.json file is successfully ingested and applied, it
will be deleted.

Please see https://www.consul.io/docs/guides/outage.html for more information.
`


text:
<?php

// @TODO: Replace $_GET and $_POST globals

class SiteController extends AdminController
{
    public $layout = '//layouts/index';

    /**
     * Specifies the access control rules.
     * This method is used by the 'accessControl' filter.
     * @return array access control rules
     */
    public function accessRules()
    {
        return array(
            array('allow', // allow all users to perform 'index' and 'view' actions
                'actions'   => array('error'),
                'users'     => array('*'),
            ),
            array('allow', // allow admin user to perform 'admin' and 'delete' actions
                'actions'   => array('index', 'email'),
                'users'     => Model::$admin_emails,
            ),
            array('deny', // deny all users
                'users'     => array('*'),
            ),
        );
    }

    /**
     * This is the action to handle external exceptions.
     */
    public function actionIndex()
    {
        // Get Event data provider
        $events = new Event('search');
        $events->unsetAttributes();

        if (isset($_GET['Event'])) {
            $events->attributes = $_GET['Event'];
        }

        // Get Git commits
        ob_start();
        system('git log -n 30 --pretty=format:"%ar: %s;"');
        $commits = ob_get_contents();
        $commits = explode(';', $commits);
        ob_end_clean();

        $opt = array('target' => '_blank');
        $emails = array(
            CHtml::link('You have an update on your booking', '/site/email?t=' . Event::BOOKING_UPDATE, $opt),
            CHtml::link('Artist has confirmed a new gig', '/site/email?t=' . Event::BOOKING_CREATE, $opt),
            CHtml::link('Your follower has made a new booking', '/site/email?t=' . Event::BOOKING_TRACK, $opt),
            CHtml::link('Your follower is now following artist', '/site/email?t=' . Event::FOLLOW_ARTIST, $opt),
            CHtml::link('Your follower is now following promoter', '/site/email?t=' . Event::FOLLOW_PROMOTER, $opt),
            CHtml::link('Your follower is now not following artist', '/site/email?t=' . Event::UNFOLLOW_ARTIST, $opt),
            CHtml::link('Your follower is now not following promoter', '/site/email?t=' . Event::UNFOLLOW_PROMOTER, $opt),
            CHtml::link('New artist is in the game', '/site/email?t=' . Event::ARTIST_CREATE, $opt),
            CHtml::link('Your follower has made a new booking', '/site/email?t=' . Event::ARTIST_TRACK, $opt),
            CHtml::link('Artist added a new gig', '/site/email?t=' . Event::GIG_CREATE, $opt),
            CHtml::link('New promoter is in the game', '/site/email?t=' . Event::PROMOTER_CREATE, $opt),
        );

        $this->render('index', array(
            'events'  => $events,
            'commits' => $commits,
            'emails'  => $emails
        ));
    }

    /**
     * This is the action to handle external exceptions.
     */
    public function actionError()
    {
        if ($error = Yii::app()->errorHandler->error) {
            if (Yii::app()->request->isAjaxRequest) {
                echo $error['message'];
            } else {
                $this->render('error', $error);
            }
        }
    }

    /**
     * This is the action to handle external exceptions.
     */
    public function actionEmail()
    {
        $this->layout = 'email';
        $request = Yii::app()->request;

        if ($id = $request->getQuery('id')) {
            if ($event = Event::model()->findByPk($id)) {
                $eventEmail = new EventEmail($event);
                echo $eventEmail->getMessage();
            } else {
                $this->render('error', array(
                    'code'      => '0x0011',
                    'message'   => 'Cant find event #' . $id
                ));
                return;
            }
        } else {
            $type = $request->getQuery('t');
            echo EventEmail::getDummyMessage($type);
        }
    }

}

redpajama_set_name: RedPajamaGithub


text:
import json

import mock

from solum.api.controllers.v1 import app
from solum.common import exception
from solum import objects
from solum.tests import base
from solum.tests import fakes

@mock.patch('pecan.request', new_callable=fakes.FakePecanRequest)
@mock.patch('pecan.response', new_callable=fakes.FakePecanResponse)
@mock.patch('solum.api.handlers.app_handler.AppHandler')
class TestAppController(base.BaseTestCase):
    def setUp(self):
        super(TestAppController, self).setUp()
        objects.load()

    def test_app_get(self, AppHandler, resp_mock, request_mock):
        fake_app = fakes.FakeApp()
        hand_get = AppHandler.return_value.get
        hand_get.return_value = fake_app
        cont = app.AppController('test_id')
        resp = cont.get()
        self.assertIsNotNone(resp)
        hand_get.assert_called_with('test_id')
        self.assertEqual(200, resp_mock.status)

    def test_app_get_notfound(self, AppHandler, resp_mock, request_mock):
        hand_get = AppHandler.return_value.get
        hand_get.side_effect = exception.ResourceNotFound(name='app',
                                                          id='test_id')
        app.AppController('test_id').get()
        hand_get.assert_called_with('test_id')
        self.assertEqual(404, resp_mock.status)

    def test_app_patch(self, AppHandler, resp_mock, request_mock):
        json_update = {
            'name': 'newfakeappname',
            'workflow_config': {
                'run_cmd': 'newruncmd',
            },
            'source': {
                'repository': 'newrepo',
            },
        }
        request_mock.body = json.dumps(json_update)
        request_mock.content_type = 'application/json'
        request_mock.security_context = None
        hand_patch = AppHandler.return_value.patch
        fake_app = objects.registry.App(**json_update)
        hand_patch.return_value = fake_app
        app.AppController('test_id').patch()
        hand_patch.assert_called_once_with('test_id', mock.ANY)
        updated_app = hand_patch.call_args[0][1]
        self.assertEqual('newfakeappname', updated_app.name)
        self.assertEqual('newruncmd', updated_app.workflow_config['run_cmd'])
        self.assertEqual('newrepo', updated_app.source['repository'])
        self.assertEqual(200, resp_mock.status)

    def test_app_patch_no_data(self, AppHandler, resp_mock, request_mock):
        request_mock.body = ''
        request_mock.content_type = 'application/json'
        request_mock.security_context = None
        app.AppController('test_id').patch()
        self.assertEqual(400, resp_mock.status)

    def test_app_delete(self, AppHandler, resp_mock, request_mock):
        hand_delete = AppHandler.return_value.delete
        hand_delete.return_value = None
        obj = app.AppController('test_id')
        obj.delete()
        hand_delete.assert_called_with('test_id')
        self.assertEqual(202, resp_mock.status)

    def test_app_delete_notfound(self, AppHandler, resp_mock, request_mock):
        hand_delete = AppHandler.return_value.delete
        hand_delete.side_effect = exception.ResourceNotFound(
            name='missing_app', app_id='test_id')
        obj = app.AppController('test_id')
        obj.delete()
        hand_delete.assert_called_with('test_id')
        self.assertEqual(404, resp_mock.status)

@mock.patch('solum.objects.registry.Image')
@mock.patch('pecan.request', new_callable=fakes.FakePecanRequest)
@mock.patch('pecan.response', new_callable=fakes.FakePecanResponse)
@mock.patch('solum.api.handlers.app_handler.AppHandler')
class TestAppsController(base.BaseTestCase):
    def setUp(self):
        super(TestAppsController, self).setUp()
        objects.load()

    def test_apps_post(self, AppHandler, resp_mock, request_mock, mock_img):
        json_create = {'name': 'fakeapp',
                       'languagepack': 'fakelp'}
        mock_img.get_lp_by_name_or_uuid.return_value = {}
        request_mock.body = json.dumps(json_create)
        request_mock.content_type = 'application/json'
        request_mock.security_context = None
        hand_create = AppHandler.return_value.create
        fake_app = objects.registry.App(**json_create)
        hand_create.return_value = fake_app
        app.AppsController().post()
        self.assertTrue(hand_create.called)
        created_app = hand_create.call_args[0][0]
        self.assertEqual(created_app['name'], json_create['name'])
        self.assertEqual(201, resp_mock.status)

    def test_apps_post_lp_not_found(self, AppHandler, resp_mock, request_mock,
                                    mock_img):
        json_create = {'name': 'fakeapp',
                       'languagepack': 'fakelp'}
        fi = fakes.FakeImage()
        not_found = exception.ResourceNotFound(name='lp', id='fakelp')
        mock_img.get_lp_by_name_or_uuid.side_effect = not_found
        mock_img.return_value = fi
        request_mock.body = json.dumps(json_create)
        request_mock.content_type = 'application/json'
        request_mock.security_context = None
        hand_create = AppHandler.return_value.create
        fake_app = objects.registry.App(**json_create)
        hand_create.return_value = fake_app
        ret = app.AppsController().post()
        expected = 'The Languagepack fakelp could not be found.'
        self.assertEqual(expected, ret['faultstring'])

    def test_apps_post_no_data(self, AppHandler, resp_mock, request_mock,
                               mock_img):
        request_mock.body = ''
        request_mock.content_type = 'application/json'
        request_mock.security_context = None
        app.AppsController().post()
        self.assertEqual(400, resp_mock.status)

    def test_apps_post_no_app_name(self, AppHandler, resp_mock, request_mock,
                                   mock_img):
        json_create = {'name': '',
                       'languagepack': 'fakelp'}
        request_mock.body = json.dumps(json_create)
        request_mock.content_type = 'application/json'
        request_mock.security_context = None
        hand_create = AppHandler.return_value.create
        fake_app = objects.registry.App(**json_create)
        hand_create.return_value = fake_app
        app.AppsController().post()
        self.assertEqual(400, resp_mock.status)

    def test_apps_get_all(self, AppHandler, resp_mock, request_mock,
                          mock_img):
        hand_get = AppHandler.return_value.get_all
        fake_app = fakes.FakeApp()
        hand_get.return_value = [fake_app]
        resp = app.AppsController().get_all()
        self.assertIsNotNone(resp)
        self.assertEqual(200, resp_mock.status)
        hand_get.assert_called_with()


text:
import torch
import torch.nn as nn
import torch.nn.functional as F

@torch.jit.script
def swish(x):
    # type: (Tensor) -> Tensor
    return x * x.sigmoid()

@torch.jit.script
def hard_sigmoid(x):
    # type: (Tensor) -> Tensor
    return F.relu6(x+3).div_(6)

@torch.jit.script
def hard_swish(x):
    # type: (Tensor) -> Tensor
    return x * hard_sigmoid(x)

class Swish(nn.Module):
    def __init__(self):
        super(Swish, self).__init__()

    def forward(self, x):
        return swish(x)

class HardSigmoid(nn.Module):
    def __init__(self):
        super(HardSigmoid, self).__init__()

    def forward(self, x):
        return hard_sigmoid(x)

class HardSwish(nn.Module):
    def __init__(self):
        super(HardSwish, self).__init__()

    def forward(self, x):
        return hard_swish(x)


text:
package reborncore.client.gui;

import net.minecraft.client.MinecraftClient;
import net.minecraft.client.gui.widget.ButtonWidget;
import net.minecraft.client.render.DiffuseLighting;
import net.minecraft.client.render.item.ItemRenderer;
import net.minecraft.item.ItemStack;
import org.lwjgl.opengl.GL11;
import reborncore.common.util.Color;

public class GuiButtonItemTexture extends ButtonWidget {

	public int textureU;
	public int textureV;
	public ItemStack itemstack;
	public String LINKED_PAGE;
	public String NAME;

	public GuiButtonItemTexture(int xPos, int yPos, int u, int v, int width, int height, ItemStack stack,
	                            String linkedPage, String name, ButtonWidget.PressAction pressAction) {
		super(xPos, yPos, width, height, "_", pressAction);
		textureU = u;
		textureV = v;
		itemstack = stack;
		NAME = name;
		this.LINKED_PAGE = linkedPage;
	}

	@Override
	public void render(int mouseX, int mouseY, float ticks) {
		if (this.visible) {
			MinecraftClient mc = MinecraftClient.getInstance();
			boolean flag = mouseX >= this.x && mouseY >= this.y && mouseX < this.x + this.width
				&& mouseY < this.y + this.height;
			mc.getTextureManager().bindTexture(WIDGETS_LOCATION);
			int u = textureU;
			int v = textureV;
			if (flag) {
				u += mc.textRenderer.getStringWidth(this.NAME) + 25;
				v += mc.textRenderer.getStringWidth(this.NAME) + 25;
				GL11.glPushMatrix();
				GL11.glColor4f(0f, 0f, 0f, 1f);
				this.blit(this.x, this.y, u, v, mc.textRenderer.getStringWidth(this.NAME) + 25, height);
				GL11.glPopMatrix();
			}
			GL11.glColor4f(1.0F, 1.0F, 1.0F, 1.0F);
			GL11.glEnable(32826);
			DiffuseLighting.enable();
			ItemRenderer itemRenderer = MinecraftClient.getInstance().getItemRenderer();
			itemRenderer.renderGuiItemIcon(itemstack, this.x, this.y);
			this.drawString(mc.textRenderer, this.NAME, this.x + 20, this.y + 3,
			                Color.WHITE.getColor());
		}
	}

}


text:
import calendar as tcalendar
import logging
import sys
from collections import OrderedDict
from datetime import datetime, date, timedelta
from math import ceil

from dateutil.relativedelta import relativedelta
from django.conf import settings
from django.contrib import messages
from django.contrib.auth.decorators import user_passes_test
from django.core.exceptions import PermissionDenied
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect
from django.shortcuts import render, get_object_or_404
from django.contrib.contenttypes.models import ContentType
from pytz import timezone

from dojo.filters import ProductFilter, ProductFindingFilter
from dojo.forms import ProductForm, EngForm, DeleteProductForm, ProductMetaDataForm, JIRAPKeyForm, JIRAFindingForm, AdHocFindingForm
from dojo.models import Product_Type, Finding, Product, Engagement, ScanSettings, Risk_Acceptance, Test, JIRA_PKey, \
    Cred_User, Cred_Mapping, Test_Type, Endpoint
from dojo.utils import get_page_items, add_breadcrumb, get_punchcard_data, get_system_setting, create_notification
from custom_field.models import CustomFieldValue, CustomField
from  dojo.tasks import add_epic_task
from tagging.models import Tag
from tagging.utils import get_tag_list
from tagging.views import TaggedItem

localtz = timezone(get_system_setting('time_zone'))

logging.basicConfig(
    level=logging.DEBUG,
    format='[%(asctime)s] %(levelname)s [%(name)s:%(lineno)d] %(message)s',
    datefmt='%d/%b/%Y %H:%M:%S',
    filename=settings.DOJO_ROOT + '/../django_app.log',
)
logger = logging.getLogger(__name__)

def product(request):
    if request.user.is_staff:
        initial_queryset = Product.objects.all()
        name_words = [product.name for product in
                      Product.objects.all()]
    else:
        initial_queryset = Product.objects.filter(
            authorized_users__in=[request.user])
        name_words = [word for product in
                      Product.objects.filter(
                          authorized_users__in=[request.user])
                      for word in product.name.split() if len(word) > 2]

    product_type = None

    if 'prod_type' in request.GET:
        p = request.GET.getlist('prod_type', [])
        if len(p) == 1:
            product_type = get_object_or_404(Product_Type, id=p[0])
    """
    if 'tags' in request.GET:
        tags = request.GET.getlist('tags', [])
        initial_queryset = TaggedItem.objects.get_by_model(initial_queryset, Tag.objects.filter(name__in=tags))
    """

    prods = ProductFilter(request.GET, queryset=initial_queryset, user=request.user)
    prod_list = get_page_items(request, prods.qs, 25)
    add_breadcrumb(title="Product List", top_level=not len(request.GET), request=request)
    return render(request,
                  'dojo/product.html',
                  {'prod_list': prod_list,
                   'prods': prods,
                   'name_words': sorted(set(name_words)),
                   'user': request.user})

def iso_to_gregorian(iso_year, iso_week, iso_day):
    jan4 = date(iso_year, 1, 4)
    start = jan4 - timedelta(days=jan4.isoweekday() - 1)
    return start + timedelta(weeks=iso_week - 1, days=iso_day - 1)

def view_product(request, pid):
    prod = get_object_or_404(Product, id=pid)
    engs = Engagement.objects.filter(product=prod, active=True)
    i_engs = Engagement.objects.filter(product=prod, active=False)
    scan_sets = ScanSettings.objects.filter(product=prod)
    auth = request.user.is_staff or request.user in prod.authorized_users.all()
    creds = Cred_Mapping.objects.filter(product=prod).select_related('cred_id').order_by('cred_id')
    endpoints = Endpoint.objects.filter(product=prod)

    if not auth:
        # will render 403
        raise PermissionDenied

    ct = ContentType.objects.get_for_model(prod)
    product_cf = CustomField.objects.filter(content_type=ct)
    product_metadata = {}

    for cf in product_cf:
        cfv = CustomFieldValue.objects.filter(field=cf, object_id=prod.id)
        if len(cfv):
            product_metadata[cf.name] = cfv[0].value

    try:
        start_date = Finding.objects.filter(test__engagement__product=prod).order_by('date')[:1][0].date
    except:
        start_date = localtz.localize(datetime.today())

    end_date = localtz.localize(datetime.today())

    tests = Test.objects.filter(engagement__product=prod)

    risk_acceptances = Risk_Acceptance.objects.filter(engagement__in=Engagement.objects.filter(product=prod))

    accepted_findings = [finding for ra in risk_acceptances
                         for finding in ra.accepted_findings.all()]

    verified_findings = Finding.objects.filter(test__engagement__product=prod,
                                               date__range=[start_date, end_date],
                                               false_p=False,
                                               verified=True,
                                               duplicate=False,
                                               out_of_scope=False).order_by("date")

    week_date = end_date - timedelta(days=7)  # seven days and /newnewer are considered "new"

    new_verified_findings = Finding.objects.filter(test__engagement__product=prod,
                                                   date__range=[week_date, end_date],
                                                   false_p=False,
                                                   verified=True,
                                                   duplicate=False,
                                                   out_of_scope=False).order_by("date")

    open_findings = Finding.objects.filter(test__engagement__product=prod,
                                           date__range=[start_date, end_date],
                                           false_p=False,
                                           verified=True,
                                           duplicate=False,
                                           out_of_scope=False,
                                           active=True,
                                           mitigated__isnull=True)

    closed_findings = Finding.objects.filter(test__engagement__product=prod,
                                             date__range=[start_date, end_date],
                                             false_p=False,
                                             verified=True,
                                             duplicate=False,
                                             out_of_scope=False,
                                             mitigated__isnull=False)

    start_date = localtz.localize(datetime.combine(start_date, datetime.min.time()))

    r = relativedelta(end_date, start_date)
    weeks_between = int(ceil((((r.years * 12) + r.months) * 4.33) + (r.days / 7)))
    if weeks_between <= 0:
        weeks_between += 2

    punchcard, ticks, highest_count = get_punchcard_data(verified_findings, weeks_between, start_date)
    add_breadcrumb(parent=prod, top_level=False, request=request)

    open_close_weekly = OrderedDict()
    new_weekly = OrderedDict()
    severity_weekly = OrderedDict()
    critical_weekly = OrderedDict()
    high_weekly = OrderedDict()
    medium_weekly = OrderedDict()

    for v in verified_findings:
        iso_cal = v.date.isocalendar()
        x = iso_to_gregorian(iso_cal[0], iso_cal[1], 1)
        y = x.strftime("%m/%d
%Y")
        x = (tcalendar.timegm(x.timetuple()) * 1000)
        if x not in critical_weekly:
            critical_weekly[x] = {'count': 0, 'week': y}
        if x not in high_weekly:
            high_weekly[x] = {'count': 0, 'week': y}
        if x not in medium_weekly:
            medium_weekly[x] = {'count': 0, 'week': y}

        if x in open_close_weekly:
            if v.mitigated:
                open_close_weekly[x]['closed'] += 1
            else:
                open_close_weekly[x]['open'] += 1
        else:

            if v.mitigated:
                open_close_weekly[x] = {'closed': 1, 'open': 0, 'accepted': 0}
            else:
                open_close_weekly[x] = {'closed': 0, 'open': 1, 'accepted': 0}
            open_close_weekly[x]['week'] = y

        if x in severity_weekly:
            if v.severity in severity_weekly[x]:
                severity_weekly[x][v.severity] += 1
            else:
                severity_weekly[x][v.severity] = 1
        else:
            severity_weekly[x] = {'Critical': 0, 'High': 0,
                                  'Medium': 0, 'Low': 0, 'Info': 0}
            severity_weekly[x][v.severity] = 1
            severity_weekly[x]['week'] = y

        if v.severity == 'Critical':
            if x in critical_weekly:
                critical_weekly[x]['count'] += 1
            else:
                critical_weekly[x] = {'count': 1, 'week': y}
        elif v.severity == 'High':
            if x in high_weekly:
                high_weekly[x]['count'] += 1
            else:
                high_weekly[x] = {'count': 1, 'week': y}
        elif v.severity == 'Medium':
            if x in medium_weekly:
                medium_weekly[x]['count'] += 1
            else:
                medium_weekly[x] = {'count': 1, 'week': y}

    for a in accepted_findings:
        iso_cal = a.date.isocalendar()
        x = iso_to_gregorian(iso_cal[0], iso_cal[1], 1)
        y = x.strftime("%m/%d
%Y")
        x = (tcalendar.timegm(x.timetuple()) * 1000)

        if x in open_close_weekly:
            open_close_weekly[x]['accepted'] += 1
        else:
            open_close_weekly[x] = {'closed': 0, 'open': 0, 'accepted': 1}
            open_close_weekly[x]['week'] = y

    test_data = {}
    for t in tests:
        if t.test_type.name in test_data:
            test_data[t.test_type.name] += t.verified_finding_count()
        else:
            test_data[t.test_type.name] = t.verified_finding_count()

    return render(request,
                  'dojo/view_product.html',
                  {'prod': prod,
                   'product_metadata': product_metadata,
                   'engs': engs,
                   'endpoints': endpoints,
                   'i_engs': i_engs,
                   'scan_sets': scan_sets,
                   'creds': creds,
                   'verified_findings': verified_findings,
                   'open_findings': open_findings,
                   'closed_findings': closed_findings,
                   'accepted_findings': accepted_findings,
                   'new_findings': new_verified_findings,
                   'start_date': start_date,
                   'punchcard': punchcard,
                   'ticks': ticks,
                   'highest_count': highest_count,
                   'open_close_weekly': open_close_weekly,
                   'severity_weekly': severity_weekly,
                   'critical_weekly': critical_weekly,
                   'high_weekly': high_weekly,
                   'medium_weekly': medium_weekly,
                   'test_data': test_data,
                   'user': request.user,
                   'authorized': auth})

@user_passes_test(lambda u: u.is_staff)
def new_product(request):
    jform = None
    if request.method == 'POST':
        form = ProductForm(request.POST, instance=Product())
        if get_system_setting('enable_jira'):
                jform = JIRAPKeyForm(request.POST, instance=JIRA_PKey())
        else:
            jform = None

        if form.is_valid():
            product = form.save()
            tags = request.POST.getlist('tags')
            t = ", ".join(tags)
            product.tags = t
            messages.add_message(request,
                                 messages.SUCCESS,
                                 'Product added successfully.',
                                 extra_tags='alert-success')
            if get_system_setting('enable_jira'):
                    if jform.is_valid():
                        jira_pkey = jform.save(commit=False)
                        if jira_pkey.conf is not None:
                            jira_pkey.product = product
                            jira_pkey.save()
                            messages.add_message(request,
                                                 messages.SUCCESS,
                                                 'JIRA information added successfully.',
                                                 extra_tags='alert-success')

            return HttpResponseRedirect(reverse('view_product', args=(product.id,)))
    else:
        form = ProductForm()
        if get_system_setting('enable_jira'):
                jform = JIRAPKeyForm()
        else:
            jform = None
    add_breadcrumb(title="New Product", top_level=False, request=request)
    return render(request, 'dojo/new_product.html',
                  {'form': form,
                   'jform': jform})

@user_passes_test(lambda u: u.is_staff)
def edit_product(request, pid):
    prod = Product.objects.get(pk=pid)
    jira_enabled = True
    jira_inst = None
    jform = None
    try:
        jira_inst = JIRA_PKey.objects.get(product=prod)
    except:
        jira_enabled = False
        pass
    if request.method == 'POST':
        form = ProductForm(request.POST, instance=prod)
        if form.is_valid():
            form.save()
            tags = request.POST.getlist('tags')
            t = ", ".join(tags)
            prod.tags = t
            messages.add_message(request,
                                 messages.SUCCESS,
                                 'Product updated successfully.',
                                 extra_tags='alert-success')

            if get_system_setting('enable_jira') and jira_enabled:
                jform = JIRAPKeyForm(request.POST, instance=jira_inst)
                #need to handle delete
                try:
                    jform.save()
                except:
                    pass
            else:
                jform = JIRAPKeyForm(request.POST)
                if  jform.is_valid():
                    new_conf = jform.save(commit=False)
                    new_conf.product_id = pid
                    new_conf.save()
                    messages.add_message(request,
                                            messages.SUCCESS,
                                            'JIRA information updated successfully.',
                                            extra_tags='alert-success')

            return HttpResponseRedirect(reverse('view_product', args=(pid,)))
    else:
        form = ProductForm(instance=prod,
                           initial={'auth_users': prod.authorized_users.all(),
                                    'tags': get_tag_list(Tag.objects.get_for_object(prod))})

        if get_system_setting('enable_jira') and jira_enabled:
            if jira_enabled:
                jform = JIRAPKeyForm(instance=jira_inst)
            else:
                jform = JIRAPKeyForm()
        else:
            jform = None
    add_breadcrumb(parent=prod, title="Edit", top_level=False, request=request)

    return render(request,
                  'dojo/edit_product.html',
                  {'form': form,
                   'jform': jform,
                   'product': prod,
                   })

@user_passes_test(lambda u: u.is_staff)
def delete_product(request, pid):
    product = get_object_or_404(Product, pk=pid)
    form = DeleteProductForm(instance=product)

    from django.contrib.admin.utils import NestedObjects
    from django.db import DEFAULT_DB_ALIAS

    collector = NestedObjects(using=DEFAULT_DB_ALIAS)
    collector.collect([product])
    rels = collector.nested()

    if request.method == 'POST':
        if 'id' in request.POST and str(product.id) == request.POST['id']:
            form = DeleteProductForm(request.POST, instance=product)
            if form.is_valid():
                if product.tags:
                    del product.tags
                product.delete()
                messages.add_message(request,
                                     messages.SUCCESS,
                                     'Product and relationships removed.',
                                     extra_tags='alert-success')
                return HttpResponseRedirect(reverse('product'))

    add_breadcrumb(parent=product, title="Delete", top_level=False, request=request)

    return render(request, 'dojo/delete_product.html',
                  {'product': product,
                   'form': form,
                   'rels': rels,
                   })

"""
Greg
Status: in production
"""

@user_passes_test(lambda u: u.is_staff)
def all_product_findings(request, pid):
    p = get_object_or_404(Product, id=pid)
    result = ProductFindingFilter(
        request.GET,
        queryset=Finding.objects.filter(test__engagement__product=p,
                                        active=True,
                                        verified=True))
    page = get_page_items(request, result.qs, 25)

    add_breadcrumb(title="Open findings", top_level=False, request=request)

    return render(request,
                  "dojo/all_product_findings.html",
                  {"findings": page,
                   "product": p,
                   "filtered": result,
                   "user": request.user,
                   })

@user_passes_test(lambda u: u.is_staff)
def new_eng_for_app(request, pid):
    jform = None
    prod = Product.objects.get(id=pid)
    if request.method == 'POST':
        form = EngForm(request.POST)
        if form.is_valid():
            new_eng = form.save(commit=False)
            new_eng.product = prod
            new_eng.lead = request.user
            if new_eng.threat_model:
                new_eng.progress = 'threat_model'
            else:
                new_eng.progress = 'other'
            new_eng.save()
            if get_system_setting('enable_jira'):
                    #Test to make sure there is a Jira project associated the product
                    try:
                        jform = JIRAFindingForm(request.POST, prefix='jiraform',
                                            enabled=JIRA_PKey.objects.get(product=prod).push_all_issues)
                        if jform.is_valid():
                            add_epic_task.delay(new_eng, jform.cleaned_data.get('push_to_jira'))
                    except JIRA_PKey.DoesNotExist:
                        pass

            #else:
            #    print >>sys.stderr, 'no prefix is found'

            messages.add_message(request,
                                 messages.SUCCESS,
                                 'Engagement added successfully.',
                                 extra_tags='alert-success')

            create_notification(event='engagement_added', title='Engagement added', engagement=new_eng, url=request.build_absolute_uri(reverse('view_engagement', args=(new_eng.id,))), objowner=new_eng.lead)

            if "_Add Tests" in request.POST:
                return HttpResponseRedirect(reverse('add_tests', args=(new_eng.id,)))
            else:
                return HttpResponseRedirect(reverse('view_engagement', args=(new_eng.id,)))
    else:
        form = EngForm(initial={})
        if(get_system_setting('enable_jira')):
                if JIRA_PKey.objects.filter(product=prod).count() != 0:
                    jform = JIRAFindingForm(prefix='jiraform', enabled=JIRA_PKey.objects.get(product=prod).push_all_issues)

    add_breadcrumb(parent=prod, title="New Engagement", top_level=False, request=request)

    return render(request, 'dojo/new_eng.html',
                  {'form': form, 'pid': pid,
                   'jform': jform
                   })

@user_passes_test(lambda u: u.is_staff)
def add_meta_data(request, pid):
    prod = Product.objects.get(id=pid)
    ct = ContentType.objects.get_for_model(prod)
    if request.method == 'POST':
        form = ProductMetaDataForm(request.POST)
        if form.is_valid():
            cf, created = CustomField.objects.get_or_create(name=form.cleaned_data['name'],
                                                   content_type=ct,
                                                   field_type='a')
            cf.save()
            cfv, created = CustomFieldValue.objects.get_or_create(field=cf,
                                                         object_id=prod.id)
            cfv.value = form.cleaned_data['value']
            cfv.clean()
            cfv.save()
            messages.add_message(request,
                                 messages.SUCCESS,
                                 'Metadata added successfully.',
                                 extra_tags='alert-success')
            if 'add_another' in request.POST:
                return HttpResponseRedirect(reverse('add_meta_data', args=(pid,)))
            else:
                return HttpResponseRedirect(reverse('view_product', args=(pid,)))
    else:
        form = ProductMetaDataForm(initial={'content_type': prod})

    add_breadcrumb(parent=prod, title="Add Metadata", top_level=False, request=request)

    return render(request,
                  'dojo/add_product_meta_data.html',
                  {'form': form,
                   'product': prod,
                   })

@user_passes_test(lambda u: u.is_staff)
def edit_meta_data(request, pid):
    prod = Product.objects.get(id=pid)
    ct = ContentType.objects.get_for_model(prod)

    product_cf = CustomField.objects.filter(content_type=ct)
    product_metadata = {}

    for cf in product_cf:
        cfv = CustomFieldValue.objects.filter(field=cf, object_id=prod.id)
        if len(cfv):
            product_metadata[cf] = cfv[0]

    if request.method == 'POST':
        for key, value in request.POST.iteritems():
            if key.startswith('cfv_'):
                cfv_id = int(key.split('_')[1])
                cfv = get_object_or_404(CustomFieldValue, id=cfv_id)

                value = value.strip()
                if value:
                    cfv.value = value
                    cfv.save()
                else:
                    cfv.delete()
        messages.add_message(request,
                             messages.SUCCESS,
                             'Metadata edited successfully.',
                             extra_tags='alert-success')
        return HttpResponseRedirect(reverse('view_product', args=(pid,)))

    add_breadcrumb(parent=prod, title="Edit Metadata", top_level=False, request=request)

    return render(request,
                  'dojo/edit_product_meta_data.html',
                  {'product': prod,
                   'product_metadata': product_metadata,
                   })

@user_passes_test(lambda u: u.is_staff)
def ad_hoc_finding(request, pid):
    prod = Product.objects.get(id=pid)
    test = None
    try:
        eng = Engagement.objects.get(product=prod, name="Ad Hoc Engagement")
        tests = Test.objects.filter(engagement=eng)

        if len(tests) != 0:
            test = tests[0]
        else:
            test = Test(engagement=eng, test_type=Test_Type.objects.get(name="Pen Test"),
                        target_start=datetime.now(tz=localtz), target_end=datetime.now(tz=localtz))
            test.save()
    except:
        eng = Engagement(name="Ad Hoc Engagement", target_start=datetime.now(tz=localtz),
                         target_end=datetime.now(tz=localtz), active=False, product=prod)
        eng.save()
        test = Test(engagement=eng, test_type=Test_Type.objects.get(name="Pen Test"),
                    target_start=datetime.now(tz=localtz), target_end=datetime.now(tz=localtz))
        test.save()
    form_error = False
    enabled = False
    jform = None
    form = AdHocFindingForm(initial={'date': datetime.now(tz=localtz).date()})
    if get_system_setting('enable_jira'):
            if JIRA_PKey.objects.filter(product=test.engagement.product).count() != 0:
                enabled = JIRA_PKey.objects.get(product=test.engagement.product).push_all_issues
                jform = JIRAFindingForm(enabled=enabled, prefix='jiraform')
    else:
        jform = None
    if request.method == 'POST':
        form = AdHocFindingForm(request.POST)
        if form.is_valid():
            new_finding = form.save(commit=False)
            new_finding.test = test
            new_finding.reporter = request.user
            new_finding.numerical_severity = Finding.get_numerical_severity(
                new_finding.severity)
            if new_finding.false_p or new_finding.active is False:
                new_finding.mitigated = datetime.now(tz=localtz)
                new_finding.mitigated_by = request.user
            create_template = new_finding.is_template
            # always false now since this will be deprecated soon in favor of new Finding_Template model
            new_finding.is_template = False
            new_finding.save()
            new_finding.endpoints = form.cleaned_data['endpoints']
            new_finding.save()
            if 'jiraform-push_to_jira' in request.POST:
                jform = JIRAFindingForm(request.POST, prefix='jiraform', enabled=enabled)
                if jform.is_valid():
                    add_issue_task.delay(new_finding, jform.cleaned_data.get('push_to_jira'))
                messages.add_message(request,
                                     messages.SUCCESS,
                                     'Finding added successfully.',
                                     extra_tags='alert-success')
            if create_template:
                templates = Finding_Template.objects.filter(title=new_finding.title)
                if len(templates) > 0:
                    messages.add_message(request,
                                         messages.ERROR,
                                         'A finding template was not created.  A template with this title already '
                                         'exists.',
                                         extra_tags='alert-danger')
                else:
                    template = Finding_Template(title=new_finding.title,
                                                cwe=new_finding.cwe,
                                                severity=new_finding.severity,
                                                description=new_finding.description,
                                                mitigation=new_finding.mitigation,
                                                impact=new_finding.impact,
                                                references=new_finding.references,
                                                numerical_severity=new_finding.numerical_severity)
                    template.save()
                    messages.add_message(request,
                                         messages.SUCCESS,
                                         'A finding template was also created.',
                                         extra_tags='alert-success')
            if '_Finished' in request.POST:
                return HttpResponseRedirect(reverse('view_test', args=(test.id,)))
            else:
                return HttpResponseRedirect(reverse('add_findings', args=(test.id,)))
        else:
            if 'endpoints' in form.cleaned_data:
                form.fields['endpoints'].queryset = form.cleaned_data['endpoints']
            else:
                form.fields['endpoints'].queryset = Endpoint.objects.none()
            form_error = True
            messages.add_message(request,
                                 messages.ERROR,
                                 'The form has errors, please correct them below.',
                                 extra_tags='alert-danger')
    add_breadcrumb(parent=prod, title="Add Finding", top_level=False, request=request)
    return render(request, 'dojo/ad_hoc_findings.html',
                  {'form': form,
                   'temp': False,
                   'tid' : test.id,
                   'pid': pid,
                   'form_error': form_error,
                   'jform': jform,
                   })


text:
package org.jkiss.dbeaver.model.impl;

import org.jkiss.dbeaver.Log;
import org.jkiss.dbeaver.model.data.DBDContentCached;
import org.jkiss.dbeaver.model.data.DBDContentStorage;
import org.jkiss.dbeaver.model.runtime.DBRProgressMonitor;
import org.jkiss.utils.IOUtils;

import java.io.*;
import java.nio.charset.Charset;

/**
 * Memory content storage
 */
public class BytesContentStorage implements DBDContentStorage, DBDContentCached {

    private static final Log log = Log.getLog(BytesContentStorage.class);

    private byte[] data;
    private String encoding;

    public BytesContentStorage(byte[] data, String encoding)
    {
        this.data = data;
        this.encoding = encoding;
    }

    public BytesContentStorage(byte[] data, Charset charset)
    {
        this.data = data;
        this.encoding = charset.name();
    }

    @Override
    public InputStream getContentStream()
        throws IOException
    {
        return new ByteArrayInputStream(data);
    }

    @Override
    public Reader getContentReader()
        throws IOException
    {
        return new InputStreamReader(
            getContentStream(),
            encoding);
    }

    @Override
    public long getContentLength()
    {
        return data.length;
    }

    @Override
    public String getCharset()
    {
        return encoding;
    }

    @Override
    public DBDContentStorage cloneStorage(DBRProgressMonitor monitor)
        throws IOException
    {
        return new BytesContentStorage(data, encoding);
    }

    @Override
    public void release()
    {
        data = null;
    }

    public static BytesContentStorage createFromStream(
        InputStream stream,
        long contentLength,
        String encoding)
        throws IOException
    {
        if (contentLength > Integer.MAX_VALUE) {
            throw new IOException("Too big content length for memory storage: " + contentLength);
        }
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        IOUtils.copyStream(stream, baos);
        byte[] result = baos.toByteArray();
        if (result.length != contentLength) {
            log.warn("Actual content length (" + result.length + ") is less than declared: " + contentLength);
        }
        return new BytesContentStorage(result, encoding);
    }

    @Override
    public Object getCachedValue() {
        return data;
    }
}


text:
import datetime
import logging
import os

from integration_tests import network_metrics
from telemetry.page import page_test
from telemetry.value import scalar

class ChromeProxyMetricException(page_test.MeasurementFailure):
  pass

CHROME_PROXY_VIA_HEADER = 'Chrome-Compression-Proxy'
CHROME_PROXY_VIA_HEADER_DEPRECATED = '1.1 Chrome Compression Proxy'

PROXY_SETTING_HTTPS = 'proxy.googlezip.net:443'
PROXY_SETTING_HTTPS_WITH_SCHEME = 'https://' + PROXY_SETTING_HTTPS
PROXY_DEV_SETTING_HTTPS_WITH_SCHEME = 'http://proxy-dev.googlezip.net:80'
PROXY_SETTING_HTTP = 'compress.googlezip.net:80'
PROXY_SETTING_DIRECT = 'direct://'

# The default Chrome Proxy bypass time is a range from one to five mintues.
# See ProxyList::UpdateRetryInfoOnFallback in net/proxy/proxy_list.cc.
DEFAULT_BYPASS_MIN_SECONDS = 60
DEFAULT_BYPASS_MAX_SECONDS = 5 * 60

def GetProxyInfoFromNetworkInternals(tab, url='chrome://net-internals#proxy'):
  tab.Navigate(url)
  with open(os.path.join(os.path.dirname(__file__),
                         'chrome_proxy_metrics.js')) as f:
    js = f.read()
    tab.ExecuteJavaScript(js)
  tab.WaitForJavaScriptExpression('performance.timing.loadEventStart', 300)
  info = tab.EvaluateJavaScript('window.__getChromeProxyInfo()')
  return info

def ProxyRetryTimeInRange(retry_time, low, high, grace_seconds=30):
  return (retry_time >= low and
          (retry_time < high + datetime.timedelta(seconds=grace_seconds)))

class ChromeProxyResponse(network_metrics.HTTPResponse):
  """ Represents an HTTP response from a timeleine event."""
  def __init__(self, event):
    super(ChromeProxyResponse, self).__init__(event)

  def ShouldHaveChromeProxyViaHeader(self):
    resp = self.response
    # Ignore https and data url
    if resp.url.startswith('https') or resp.url.startswith('data:'):
      return False
    # Ignore 304 Not Modified and cache hit.
    if resp.status == 304 or resp.served_from_cache:
      return False
    # Ignore invalid responses that don't have any header. Log a warning.
    if not resp.headers:
      logging.warning('response for %s does not any have header '
                      '(refer=%s, status=%s)',
                      resp.url, resp.GetHeader('Referer'), resp.status)
      return False
    return True

  def HasChromeProxyViaHeader(self):
    via_header = self.response.GetHeader('Via')
    if not via_header:
      return False
    vias = [v.strip(' ') for v in via_header.split(',')]
    # The Via header is valid if it is the old format or the new format
    # with 4-character version prefix, for example,
    # "1.1 Chrome-Compression-Proxy".
    return (CHROME_PROXY_VIA_HEADER_DEPRECATED in vias or
            any(v[4:] == CHROME_PROXY_VIA_HEADER for v in vias))

  def IsValidByViaHeader(self):
    return (not self.ShouldHaveChromeProxyViaHeader() or
            self.HasChromeProxyViaHeader())

  def IsSafebrowsingResponse(self):
    if (self.response.status == 307 and
        self.response.GetHeader('X-Malware-Url') == '1' and
        self.IsValidByViaHeader() and
        self.response.GetHeader('Location') == self.response.url):
      return True
    return False

class ChromeProxyMetric(network_metrics.NetworkMetric):
  """A Chrome proxy timeline metric."""

  def __init__(self):
    super(ChromeProxyMetric, self).__init__()
    self.compute_data_saving = True
    self.effective_proxies = {
        "proxy": PROXY_SETTING_HTTPS_WITH_SCHEME,
        "proxy-dev": PROXY_DEV_SETTING_HTTPS_WITH_SCHEME,
        "fallback": PROXY_SETTING_HTTP,
        "direct": PROXY_SETTING_DIRECT,
        }

  def SetEvents(self, events):
    """Used for unittest."""
    self._events = events

  def ResponseFromEvent(self, event):
    return ChromeProxyResponse(event)

  def AddResults(self, tab, results):
    raise NotImplementedError

  def AddResultsForDataSaving(self, tab, results):
    resources_via_proxy = 0
    resources_from_cache = 0
    resources_direct = 0

    super(ChromeProxyMetric, self).AddResults(tab, results)
    for resp in self.IterResponses(tab):
      if resp.response.served_from_cache:
        resources_from_cache += 1
      if resp.HasChromeProxyViaHeader():
        resources_via_proxy += 1
      else:
        resources_direct += 1

    results.AddValue(scalar.ScalarValue(
        results.current_page, 'resources_via_proxy', 'count',
        resources_via_proxy))
    results.AddValue(scalar.ScalarValue(
        results.current_page, 'resources_from_cache', 'count',
        resources_from_cache))
    results.AddValue(scalar.ScalarValue(
        results.current_page, 'resources_direct', 'count', resources_direct))

  def AddResultsForHeaderValidation(self, tab, results):
    via_count = 0
    bypass_count = 0
    for resp in self.IterResponses(tab):
      if resp.IsValidByViaHeader():
        via_count += 1
      else:
        bypassed, _ = self.IsProxyBypassed(tab)
        if tab and bypassed:
          logging.warning('Proxy bypassed for %s', resp.response.url)
          bypass_count += 1
        else:
          r = resp.response
          raise ChromeProxyMetricException, (
              '%s: Via header (%s) is not valid (refer=%s, status=%d)' % (
                  r.url, r.GetHeader('Via'), r.GetHeader('Referer'), r.status))
    results.AddValue(scalar.ScalarValue(
        results.current_page, 'checked_via_header', 'count', via_count))
    results.AddValue(scalar.ScalarValue(
        results.current_page, 'request_bypassed', 'count', bypass_count))

  def AddResultsForClientVersion(self, tab, results):
    for resp in self.IterResponses(tab):
      r = resp.response
      if resp.response.status != 200:
        raise ChromeProxyMetricException, ('%s: Response is not 200: %d' %
                                           (r.url, r.status))
      if not resp.IsValidByViaHeader():
        raise ChromeProxyMetricException, ('%s: Response missing via header' %
                                           (r.url))
    results.AddValue(scalar.ScalarValue(
        results.current_page, 'version_test', 'count', 1))

  def IsProxyBypassed(self, tab):
    """ Returns True if all configured proxies are bypassed."""
    if not tab:
      return False, []

    info = GetProxyInfoFromNetworkInternals(tab)
    if not info['enabled']:
      raise ChromeProxyMetricException, (
          'Chrome proxy should be enabled. proxy info: %s' % info)

    bad_proxies = [str(p['proxy']) for p in info['badProxies']]
    bad_proxies.sort()
    proxies = [self.effective_proxies['proxy'],
               self.effective_proxies['fallback']]
    proxies.sort()
    proxies_dev = [self.effective_proxies['proxy-dev'],
                   self.effective_proxies['fallback']]
    proxies_dev.sort()
    if bad_proxies == proxies:
      return True, proxies
    elif bad_proxies == proxies_dev:
      return True, proxies_dev
    return False, []

  @staticmethod
  def VerifyBadProxies(
      badProxies, expected_proxies,
      retry_seconds_low = DEFAULT_BYPASS_MIN_SECONDS,
      retry_seconds_high = DEFAULT_BYPASS_MAX_SECONDS):
    """Verify the bad proxy list and their retry times are expected. """
    if not badProxies or (len(badProxies) != len(expected_proxies)):
      return False

    # Check all expected proxies.
    proxies = [p['proxy'] for p in badProxies]
    expected_proxies.sort()
    proxies.sort()
    if not expected_proxies == proxies:
      raise ChromeProxyMetricException, (
          'Bad proxies: got %s want %s' % (
              str(badProxies), str(expected_proxies)))

    # Check retry time
    for p in badProxies:
      retry_time_low = (datetime.datetime.now() +
                        datetime.timedelta(seconds=retry_seconds_low))
      retry_time_high = (datetime.datetime.now() +
                        datetime.timedelta(seconds=retry_seconds_high))
      got_retry_time = datetime.datetime.fromtimestamp(int(p['retry'])/1000)
      if not ProxyRetryTimeInRange(
          got_retry_time, retry_time_low, retry_time_high):
        raise ChromeProxyMetricException, (
            'Bad proxy %s retry time (%s) should be within range (%s-%s).' % (
                p['proxy'], str(got_retry_time), str(retry_time_low),
                str(retry_time_high)))
    return True

  def AddResultsForBypass(self, tab, results):
    bypass_count = 0
    for resp in self.IterResponses(tab):
      if resp.HasChromeProxyViaHeader():
        r = resp.response
        raise ChromeProxyMetricException, (
            '%s: Should not have Via header (%s) (refer=%s, status=%d)' % (
                r.url, r.GetHeader('Via'), r.GetHeader('Referer'), r.status))
      bypass_count += 1

    if tab:
      info = GetProxyInfoFromNetworkInternals(tab)
      if not info['enabled']:
        raise ChromeProxyMetricException, (
            'Chrome proxy should be enabled. proxy info: %s' % info)
      _, expected_bad_proxies = self.IsProxyBypassed(tab)
      self.VerifyBadProxies(info['badProxies'], expected_bad_proxies)

    results.AddValue(scalar.ScalarValue(
        results.current_page, 'bypass', 'count', bypass_count))

  def AddResultsForBlockOnce(self, tab, results):
    eligible_response_count = 0
    bypass_count = 0
    for resp in self.IterResponses(tab):
      if resp.ShouldHaveChromeProxyViaHeader():
        eligible_response_count += 1
        if not resp.HasChromeProxyViaHeader():
          bypass_count += 1

    if tab:
      info = GetProxyInfoFromNetworkInternals(tab)
      if not info['enabled']:
        raise ChromeProxyMetricException, (
            'Chrome proxy should be enabled. proxy info: %s' % info)
      self.VerifyBadProxies(info['badProxies'], [])

    if eligible_response_count <= 1:
      raise ChromeProxyMetricException, (
          'There should be more than one DRP eligible response '
          '(eligible_response_count=%d, bypass_count=%d)\n' % (
              eligible_response_count, bypass_count))
    elif bypass_count != 1:
      raise ChromeProxyMetricException, (
          'Exactly one response should be bypassed. '
          '(eligible_response_count=%d, bypass_count=%d)\n' % (
              eligible_response_count, bypass_count))
    else:
      results.AddValue(scalar.ScalarValue(
          results.current_page, 'eligible_responses', 'count',
          eligible_response_count))
      results.AddValue(scalar.ScalarValue(
          results.current_page, 'bypass', 'count', bypass_count))

  def AddResultsForSafebrowsing(self, tab, results):
    count = 0
    safebrowsing_count = 0
    for resp in self.IterResponses(tab):
      count += 1
      if resp.IsSafebrowsingResponse():
        safebrowsing_count += 1
      else:
        r = resp.response
        raise ChromeProxyMetricException, (
            '%s: Not a valid safe browsing response.\n'
            'Reponse: status=(%d, %s)\nHeaders:\n %s' % (
                r.url, r.status, r.status_text, r.headers))
    if count == safebrowsing_count:
      results.AddValue(scalar.ScalarValue(
          results.current_page, 'safebrowsing', 'boolean', True))
    else:
      raise ChromeProxyMetricException, (
          'Safebrowsing failed (count=%d, safebrowsing_count=%d)\n' % (
              count, safebrowsing_count))

  def AddResultsForHTTPFallback(
      self, tab, results, expected_proxies=None, expected_bad_proxies=None):
    info = GetProxyInfoFromNetworkInternals(tab)
    if not 'enabled' in info or not info['enabled']:
      raise ChromeProxyMetricException, (
          'Chrome proxy should be enabled. proxy info: %s' % info)

    if not expected_proxies:
      expected_proxies = [self.effective_proxies['fallback'],
                          self.effective_proxies['direct']]
    if not expected_bad_proxies:
      expected_bad_proxies = []

    proxies = info['proxies']
    if proxies != expected_proxies:
      raise ChromeProxyMetricException, (
          'Wrong effective proxies (%s). Expect: "%s"' % (
          str(proxies), str(expected_proxies)))

    bad_proxies = []
    if 'badProxies' in info and info['badProxies']:
      bad_proxies = [p['proxy'] for p in info['badProxies']
                     if 'proxy' in p and p['proxy']]
    if bad_proxies != expected_bad_proxies:
      raise ChromeProxyMetricException, (
          'Wrong bad proxies (%s). Expect: "%s"' % (
          str(bad_proxies), str(expected_bad_proxies)))
    results.AddValue(scalar.ScalarValue(
        results.current_page, 'http_fallback', 'boolean', True))


text:
cask 'tistoryeditor' do
  version '0.6.9'
  sha256 'c52327c39467667cc463a372048c10d4470ebdfafcf8aa5ea2b78fc2606e5a97'

  # github.com/joostory/tistory-editor was verified as official when first introduced to the cask
  url "https://github.com/joostory/tistory-editor/releases/download/v#{version}/TistoryEditor-#{version}-mac.zip"
  appcast 'https://github.com/joostory/tistory-editor/releases.atom'
  name 'TistoryEditor'
  homepage 'https://tistory-editor.tistory.com/'

  app 'TistoryEditor.app'

  zap trash: [
               '~/Library/Preferences/net.joostory.tistory-editor.helper.plist',
               '~/Library/Preferences/net.joostory.tistory-editor.plist',
               '~/Library/Saved Application State/net.joostory.tistory-editor.savedState',
             ]
end


text:
'use strict';

module.exports = {
  require: [require.resolve('./test/setup.js')],
  // forbids use of .only() in CI
  forbidOnly: Boolean(process.env.CI),
  color: true,
};


text:
CBMCFLAGS +=

PROOF_UID = s2n_stuffer_write_uint8
HARNESS_ENTRY = $(PROOF_UID)_harness
HARNESS_FILE = $(HARNESS_ENTRY).c

PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE)
PROOF_SOURCES += $(PROOF_SOURCE)/cbmc_utils.c
PROOF_SOURCES += $(PROOF_SOURCE)/make_common_datastructures.c

PROJECT_SOURCES += $(SRCDIR)/stuffer/s2n_stuffer.c
PROJECT_SOURCES += $(SRCDIR)/stuffer/s2n_stuffer_network_order.c
PROJECT_SOURCES += $(SRCDIR)/utils/s2n_blob.c
PROJECT_SOURCES += $(SRCDIR)/utils/s2n_ensure.c
PROJECT_SOURCES += $(SRCDIR)/utils/s2n_mem.c
PROJECT_SOURCES += $(SRCDIR)/utils/s2n_safety.c
PROJECT_SOURCES += $(SRCDIR)/utils/s2n_result.c

# We abstract these functions because manual inspection demonstrates they are unreachable.
REMOVE_FUNCTION_BODY += s2n_blob_slice
REMOVE_FUNCTION_BODY += __CPROVER_file_local_s2n_mem_c_s2n_mem_cleanup_impl
REMOVE_FUNCTION_BODY += s2n_stuffer_wipe

UNWINDSET +=

include ../Makefile.common


text:
require 'spec_helper'

describe ReputationObserver do
  subject { ReputationObserver.instance }

  context 'when Answer created' do
    it "calls Reputation.created_answer" do
      answer = build(:answer)
      expect(Reputation).to receive(:created_answer).with(answer)
      answer.save!
    end
  end

  context 'when updating Answer' do
    let(:question) { create(:question) }
    let(:answer) { create(:answer, accepted: true) }

    it 'calls Reputation.accepted when answer accepted' do
      expect(Reputation).to receive(:accepted_answer).with(answer, false)
      answer.toggle_accepted_from question
    end

    it 'calls Reputation.accepted when answer not accepted' do
      answer.update(accepted: false)
      expect(Reputation).to receive(:accepted_answer).with(answer, true)
      answer.toggle_accepted_from question
    end
  end

  context 'when Vote created' do
    [:question, :answer].each do |object|
      context "on #{object}" do
        context 'when upvote' do
          it "calls Reputation.voted" do
            vote = build(:vote, voteable: create(object), value: 1)
            expect(Reputation).to receive(:voted).with(vote)
            vote.save!
          end
        end

        context 'when downvote' do
          it "calls Reputation.voted" do
            vote = build(:vote, voteable: create(object), value: -1)
            expect(Reputation).to receive(:voted).with(vote)
            vote.save!
          end
        end
      end
    end
  end
end


text: Usefull codes for php applications


text:
/*jshint expr: true, nonew: false*/
var PluginApi = require('../lib/PluginApi.js');

describe('PluginApi', function () {

    describe('constructor', function () {
        it('should not throw if no arguments', function () {
            expect(function () {
                new PluginApi();
            }).not.to.throw();
        });
    });

    describe('events', function () {
        var api;

        beforeEach(function () {
            api = new PluginApi();
        });

        afterEach(function () {
            api._reset();
        });

        it('should exist', function () {
            expect(api.on).to.exist;
            expect(api.trigger).to.exist;
        });

        it('should allow adding listener', function () {
            var spy = sinon.spy();
            api.on('test', spy);
            expect(spy).to.not.have.been.called;
        });

        it('should trigger listeners', function () {
            var spy = sinon.spy();
            api.on('test', spy);
            api.trigger('test');
            expect(spy).to.have.been.calledOnce;
        });

        it('should pass arguments from trigger to the listeners', function () {
            var spy = sinon.spy();
            api.on('test', spy);
            api.trigger('test', 1, 2, 3);
            expect(spy).to.have.been.calledWith(1, 2, 3);
        });

        it('should allow multiple listeners', function () {
            var spy = sinon.spy();
            var spy2 = sinon.spy();
            api.on('test', spy);
            api.on('test', spy2);
            api.trigger('test');
            expect(spy).to.have.been.calledOnce;
            expect(spy2).to.have.been.calledOnce;
        });

        it('should allow triggering events without listeners', function () {
            expect(function () {
                api.trigger('foo');
            }).not.to.throw();
        });
    });
});


text:
layout: example.html
title: Semi-Transparent Layer
shortdesc: Example of a map with a semi-transparent layer.
docs: >
  This example will display a tiled MaxBox layer semi-transparently over a MapQuest background.
tags: "transparent, mapquest, tilejson"
---


text:
package fcm

import (
	"bytes"
	"testing"

	a "github.com/stretchr/testify/assert"
)

// -- helpers
func thAssertInstanceInMapper(t *testing.T, m *MemoryMapper, i *Instance) {
	for _, r := range i.RegistrationIDS {
		if !a.Contains(t, m.regIDs, r, "missing registrationID: %s, instanceID: %s", r, i.ID) {
			continue
		}
		a.Equal(t, i, m.regIDs[r], "instance does not match on registrationID: %s, instanceID: %s", r, i.ID)
	}
}

// -- suite setup
func tsMemoryStorageSetup() (*MemoryStorage, func()) {
	s := NewMemoryStorage()
	closer := func() {}
	return s, closer
}

func tsMemoryStorageWitAppsSetup() (*MemoryStorage, func()) {
	s, closer := tsMemoryStorageSetup()
	s.AppSave(&TFAppA)
	s.AppSave(&TFAppB)
	s.AppSave(&TFAppC)
	return s, closer
}

func TSMemoryStorageWitAppsAndInstancesSetup() (*MemoryStorage, func()) {
	s, closer := tsMemoryStorageSetup()
	s.AppSave(&TFAppA)
	s.AppSave(&TFAppB)
	s.AppSave(&TFAppC)

	s.InstanceSave(&TFInsAA)
	s.InstanceSave(&TFInsAB)
	s.InstanceSave(&TFInsAC)
	s.InstanceSave(&TFInsAZ)
	s.InstanceSave(&TFInsBA)
	s.InstanceSave(&TFInsBB)
	s.InstanceSave(&TFInsBC)

	return s, closer
}

func tsMapperSetup() (*MemoryMapper, func()) {
	m := NewMemoryMapper()
	closer := func() {}
	return m, closer
}

func tsGeneratorSetup(at, ipa int, up float64, rpi int) (*Generator, func()) {
	m := NewGenerator(at, ipa, up, rpi)
	closer := func() {}
	return m, closer
}

func tsImporterSetup() (*JSONImporter, *MemoryStorage, *MemoryMapper, func()) {
	s := NewMemoryStorage()
	m := NewMemoryMapper()
	i := NewJSONImporter(s, m)
	return i, s, m, func() {}
}

func tsExporterSetup() (*JSONExporter, *bytes.Buffer, *bytes.Buffer, func()) {
	aw := &bytes.Buffer{}
	iw := &bytes.Buffer{}
	e := NewJSONExporter(aw, iw)
	return e, aw, iw, func() {}
}


text:
<!DOCTYPE html>

    <title>MobileToys</title>
    <link rel="stylesheet" href="../../../lib/jquery.mobile.css"/>

Start

Page2


text:
#include "ch.h"
#include "hal.h"

#include "chprintf.h"
#include "shell.h"

#include "usbcfg.h"

/* Virtual serial port over USB.*/
SerialUSBDriver SDU1;

/*===========================================================================*/
/* Command line related.                                                     */
/*===========================================================================*/

#define SHELL_WA_SIZE   THD_WA_SIZE(2048)
#define TEST_WA_SIZE    THD_WA_SIZE(256)

static void cmd_mem(BaseSequentialStream *chp, int argc, char *argv[]) {
  size_t n, size;

  (void)argv;
  if (argc > 0) {
    chprintf(chp, "Usage: mem\r\n");
    return;
  }
  n = chHeapStatus(NULL, &size);
  chprintf(chp, "core free memory : %u bytes\r\n", chCoreStatus());
  chprintf(chp, "heap fragments   : %u\r\n", n);
  chprintf(chp, "heap free total  : %u bytes\r\n", size);
}

static void cmd_threads(BaseSequentialStream *chp, int argc, char *argv[]) {
  static const char *states[] = {THD_STATE_NAMES};
  Thread *tp;

  (void)argv;
  if (argc > 0) {
    chprintf(chp, "Usage: threads\r\n");
    return;
  }
  chprintf(chp, "    addr    stack prio refs     state time\r\n");
  tp = chRegFirstThread();
  do {
    chprintf(chp, "%.8lx %.8lx %4lu %4lu %9s %lu\r\n",
            (uint32_t)tp, (uint32_t)tp->p_ctx.r13,
            (uint32_t)tp->p_prio, (uint32_t)(tp->p_refs - 1),
            states[tp->p_state], (uint32_t)tp->p_time);
    tp = chRegNextThread(tp);
  } while (tp != NULL);
}

static const ShellCommand commands[] = {
  {"mem", cmd_mem},
  {"threads", cmd_threads},
  {NULL, NULL}
};

static const ShellConfig shell_cfg1 = {
  (BaseSequentialStream *)&SDU1,
  commands
};

/*
 * Blinker thread
 */
static WORKING_AREA(waThread1, 256);

static msg_t Thread1(void *arg) {

  (void)arg;
  chRegSetThreadName("thread1");

  while (TRUE) {
    palSetPad(GPIOD, GPIOD_LED3);
    chThdSleepMilliseconds(200);
    palSetPad(GPIOD, GPIOD_LED4);
    chThdSleepMilliseconds(200);
    palSetPad(GPIOD, GPIOD_LED5);
    chThdSleepMilliseconds(200);
    palSetPad(GPIOD, GPIOD_LED6);
    chThdSleepMilliseconds(200);

    palClearPad(GPIOD, GPIOD_LED6);
    chThdSleepMilliseconds(200);
    palClearPad(GPIOD, GPIOD_LED5);
    chThdSleepMilliseconds(200);
    palClearPad(GPIOD, GPIOD_LED4);
    chThdSleepMilliseconds(200);
    palClearPad(GPIOD, GPIOD_LED3);
    chThdSleepMilliseconds(200);
  }

  return RDY_OK;
}

/*===========================================================================*/
/* Initialization and main thread.                                           */
/*===========================================================================*/

/*
 * Application entry point.
 */
int main(void) {
  Thread *shelltp = NULL;

  /*
   * System initializations.
   * - HAL initialization, this also initializes the configured device drivers
   *   and performs the board-specific initializations.
   * - Kernel initialization, the main() function becomes a thread and the
   *   RTOS is active.
   */
  halInit();
  chSysInit();

  /*
   * Shell manager initialization.
   */
  shellInit();

  /*
   * Initializes a serial-over-USB CDC driver.
   */
  sduObjectInit(&SDU1);
  sduStart(&SDU1, &serusbcfg);

  /*
   * Activates the USB driver and then the USB bus pull-up on D+.
   * Note, a delay is inserted in order to not have to disconnect the cable
   * after a reset.
   */
  usbDisconnectBus(serusbcfg.usbp);
  chThdSleepMilliseconds(1000);
  usbStart(serusbcfg.usbp, &usbcfg);
  usbConnectBus(serusbcfg.usbp);

  /*
   * Creates the example thread.
   */
  chThdCreateStatic(waThread1, sizeof(waThread1),
                    NORMALPRIO + 10, Thread1, NULL);

  /*
   * Normal main() thread activity, in this demo it just performs
   * a shell respawn upon its termination.
   */
  while (TRUE) {
    if (!shelltp) {
      if (SDU1.config->usbp->state == USB_ACTIVE) {
        /* Spawns a new shell.*/
        shelltp = shellCreate(&shell_cfg1, SHELL_WA_SIZE, NORMALPRIO);
      }
    }
    else {
      /* If the previous shell exited.*/
      if (chThdTerminated(shelltp)) {
        /* Recovers memory of the previous shell.*/
        chThdRelease(shelltp);
        shelltp = NULL;
      }
    }
    chThdSleepMilliseconds(500);
  }
}


text:
class PagesController < ActionController::Base
  include Rails.application.routes.url_helpers
  sessioner

  def index
    render plain: "OK"
  end
end


text:
Largely only for debugging usage during the evaluation of comm models in nteract.

Intended to be used with `@nteract/transforms`. Note that this transform is bundled
in `@nteract/transforms-full`.

## Installation

```
npm install @nteract/transform-model-debug
```


text:
#ifndef QT3DINPUT_INPUT_INPUTSETTINGS_H
#define QT3DINPUT_INPUT_INPUTSETTINGS_H

//
//  W A R N I N G
//  -------------
//
// This file is not part of the Qt API.  It exists for the convenience
// of other Qt classes.  This header file may change from version to
// version without notice, or even be removed.
//
// We mean it.
//

#include <QPointer>
#include <Qt3DCore/qbackendnode.h>

QT_BEGIN_NAMESPACE

namespace Qt3DInput {

namespace Input {

class InputHandler;

class InputSettings : public Qt3DCore::QBackendNode
{
public:
    InputSettings();
    inline QObject *eventSource() const { return m_eventSource; }

protected:
    void sceneChangeEvent(const Qt3DCore::QSceneChangePtr &e) Q_DECL_OVERRIDE;

private:
    void initializeFromPeer(const Qt3DCore::QNodeCreatedChangeBasePtr &change) Q_DECL_FINAL;

    QPointer<QObject> m_eventSource;
};

class InputSettingsFunctor : public Qt3DCore::QBackendNodeMapper
{
public:
    explicit InputSettingsFunctor(InputHandler *handler);

    Qt3DCore::QBackendNode *create(const Qt3DCore::QNodeCreatedChangeBasePtr &change) const Q_DECL_OVERRIDE;
    Qt3DCore::QBackendNode *get(Qt3DCore::QNodeId id) const Q_DECL_OVERRIDE;
    void destroy(Qt3DCore::QNodeId id) const Q_DECL_OVERRIDE;

private:
    InputHandler *m_handler;
};

} // Input

} // Qt3DInput

QT_END_NAMESPACE

#endif // QT3DINPUT_INPUT_INPUTSETTINGS_H


text:
          <%= polyglot.t('CreateStore') %>

      Home

        <%= polyglot.t('Pages') %>

                  <a class="btn btn-txt btn-togRight js-homeListingsAll tooltip tooltip-left colorGrey marginTop6"
                     data-tooltip="<%= polyglot.t('ShowAllListingsTooltip') %>">
                    <%= polyglot.t('Random') %>

                  <a class="btn btn-txt btn-togRight active js-homeListingsFollowed tooltip tooltip-left colorGrey marginTop6"
                     data-tooltip="<%= polyglot.t('ShowOnlyFollowerListingsTooltip') %>">
                    <%= polyglot.t('ShowOnlyFollowerListings') %>

                  <% if(!localStorage.getItem('safeListingsWarningDissmissed')){ %>

<%= polyglot.t('TryRandom') %>

                  <% } %>

              <div class="js-loadingText" data-default-text="<%= polyglot.t('discover.searchDefaultText') %>" data-searching-text="<%= polyglot.t('discover.searchingText') %>"><%= polyglot.t('discover.searchDefaultText') %>

  <%= ob.backToTopTmpl() %>

<%= polyglot.t('ComingSoon') %>

      <%= polyglot.t('FeedPlaceholder') %>