text:
package main

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/sha256"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/base64"
	"encoding/json"
	"encoding/pem"
	"fmt"
	"github.com/RobinUS2/golang-jresp"
	"github.com/julienschmidt/httprouter"
	"github.com/nu7hatch/gouuid"
	"github.com/spf13/cast"
	"github.com/RobinUS2/indispenso/data_table"
	"io/ioutil"
	"math/big"
	"net"
	"net/http"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

// Server methods (you probably only need one or two in HA failover mode)

type Server struct {
	clientsMux sync.RWMutex
	clients    map[string]*RegisteredClient

	Tags    map[string]bool
	tagsMux sync.RWMutex

	userStore            *UserStore
	templateStore        *TemplateStore
	consensus            *Consensus
	executionCoordinator *ExecutionCoordinator
	httpCheckStore       *HttpCheckStore
	authService          *AuthService
	notifications        *NotificationManager

	InstanceId string // Unique ID generated at startup of the server, used for re-authentication and client-side refresh after and update/restart
}

// Register client
func (s *Server) RegisterClient(clientId string, tags []string) {
	s.clientsMux.RLock()
	if s.clients[clientId] == nil {
		s.clientsMux.RUnlock()

		// Write lock
		s.clientsMux.Lock()
		s.clients[clientId] = newRegisteredClient(clientId)
		s.clientsMux.Unlock()
		log.Printf("Client %s registered with tags %s", clientId, tags)
	} else {
		s.clientsMux.RUnlock()
	}

	// Update client
	s.clients[clientId].mux.Lock()
	s.clients[clientId].LastPing = time.Now()
	s.clients[clientId].Tags = tags
	s.clients[clientId].mux.Unlock()

	// Update tags
	s.tagsMux.Lock()
	for _, tag := range tags {
		s.Tags[tag] = true
	}
	s.tagsMux.Unlock()
}

func (s *Server) GetClient(clientId string) *RegisteredClient {
	s.clientsMux.Lock()
	defer s.clientsMux.Unlock()
	return s.clients[clientId]
}

// Scan for old clients
func (s *Server) CleanupClients() {
	s.clientsMux.Lock()
	for k, client := range s.clients {
		if time.Now().Sub(client.LastPing).Seconds() > float64(CLIENT_PING_INTERVAL*5) {
			// Disconnect
			log.Printf("Client %s disconnected", client.ClientId)
			delete(s.clients, k)
		}
	}
	s.clientsMux.Unlock()
}

// Submit command to registered client using channel notify system
func (client *RegisteredClient) Submit(cmd *Cmd) {
	client.mux.Lock()

	// Command in pending list, this will be polled of within milliseconds
	client.Cmds[cmd.Id] = cmd

	// Keep track of command status
	client.DispatchedCmds[cmd.Id] = cmd

	client.mux.Unlock()

	// Log
	audit.Log(nil, "Execute", fmt.Sprintf("Command '%s' on client %s with id %s", cmd.Command, client.ClientId, cmd.Id))

	// Signal for work
	client.CmdChan <- true
}

// A client that is registered with the server
type RegisteredClient struct {
	mux       sync.RWMutex
	ClientId  string
	AuthToken string `json:"-"` // Do not add to JSON
	LastPing  time.Time
	Tags      []string

	// Dispatched commands to the client
	DispatchedCmds map[string]*Cmd

	// Pending commands
	Cmds map[string]*Cmd

	// Channel used to trigger the long poll to fire a command to the client
	CmdChan chan bool `json:"-"`
}

// Get list of dispatched commands
// will automatically purge commands older than X days
func (c *RegisteredClient) GetDispatchedCmds() map[string]*Cmd {
	// Max age
	maxAge := time.Now().Unix() - (14 * 86400)

	// Is this one dirty? Meaning it contains too old data?
	dirty := false

	// Placeholder of list
	newMap := make(map[string]*Cmd, 0)

	// Build list
	c.mux.RLock()
	for k, d := range c.DispatchedCmds {
		if d.Created >= maxAge {
			newMap[k] = d
		} else {
			dirty = true
		}
	}
	c.mux.RUnlock()

	// Swap?
	if dirty {
		c.mux.Lock()
		if conf.Debug {
			log.Printf("Cleaning up dispatched commands of client %s size went from %d to %d", c.ClientId, len(c.DispatchedCmds), len(newMap))
		}
		c.DispatchedCmds = newMap
		c.mux.Unlock()
	}

	return newMap
}

// Does this register client have this tag?
func (c *RegisteredClient) HasTag(s string) bool {
	if c.Tags == nil {
		return false
	}
	if len(c.Tags) == 0 {
		return false
	}
	for _, tag := range c.Tags {
		if tag == s {
			return true
		}
	}
	return false
}

// Generate keys
func (s *Server) _prepareTlsKeys() error {
	if _, err := os.Stat(conf.GetSslCertFile()); os.IsNotExist(err) {
		if !conf.AutoGenerateCert {
			log.Printf("Cannot locat certificate file(%s) doesn't exist, provide one or enable automatic self signed cert generation", conf.GetSslCertFile())
			return err
		}
		privateKey, err := _readOrGeneratePrivateKey(conf.GetSslPrivateKeyFile())
		if err != nil {
			return err
		}
		return _generateCertificate(conf.GetSslCertFile(), privateKey, 365*24*time.Hour)
	}
	return nil
}

func _readOrGeneratePrivateKey(fileName string) (*rsa.PrivateKey, error) {
	if _, err := os.Stat(fileName); os.IsNotExist(err) {
		privateKey, _ := rsa.GenerateKey(rand.Reader, 2048)
		keyOut, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
		defer keyOut.Close()
		if err != nil {
			log.Printf("failed to open %s for writing: %s", fileName, err)
			return privateKey, err
		}
		pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)})
		return privateKey, nil
	} else {
		log.Printf("Private key (%s) exists and will be used", fileName)
		certArray, err := ioutil.ReadFile(fileName)
		if err != nil {
			return nil, err
		}
		block, _ := pem.Decode(certArray)
		return x509.ParsePKCS1PrivateKey(block.Bytes)
	}
}

/**
Function that provide Serial number for certificate.

The serial number is chosen by the CA which issued the certificate.
It is just written in the certificate. The CA can choose the serial
number in any way as it sees fit.
*/
func _getCertSerialNumber() *big.Int {
	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
	if err != nil {
		log.Printf("failed to generate serial number: %s", err)
	}
	return serialNumber
}

func _generateCertificateTmpl(subject pkix.Name, validPeriod time.Duration) x509.Certificate {
	currentTime := time.Now()
	return x509.Certificate{
		SerialNumber: _getCertSerialNumber(),
		Subject:      subject,

		NotBefore: currentTime,
		NotAfter:  currentTime.Add(validPeriod),

		KeyUsage:    x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
		ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
		IsCA:        false,
		BasicConstraintsValid: true,
		IPAddresses:           []net.IP{net.IPv4(127, 0, 0, 1).To4()},
	}
}

func _generateCertificate(fileName string, privateKey *rsa.PrivateKey, validPeriod time.Duration) error {

	log.Println("Autogeneration of selfsigned certificate key...")
	subject := pkix.Name{
		Organization:       []string{"Indispenso"},
		Country:            []string{"NL"},
		CommonName:         "ssl.indispenso.org",
		OrganizationalUnit: []string{"IT"},
		Province:           []string{"Indispenso"},
		Locality:           []string{"Indispenso"},
	}

	tmpl := _generateCertificateTmpl(subject, validPeriod)
	certBytes, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &privateKey.PublicKey, privateKey)
	if err != nil {
		log.Printf("Failed to create certificate: %s\n", err)
		return err
	}
	log.Println("Public key generated sucessfully")

	certFile, err := os.Create(fileName)
	if err != nil {
		log.Printf("Cannot create cert file : %s\n", err)
		return err
	}

	pem.Encode(certFile, &pem.Block{Type: "CERTIFICATE", Bytes: certBytes})
	certFile.Close()
	log.Printf("Successfully written certificate: %s\n", fileName)
	return err
}

// Start server
func (s *Server) Start() bool {
	// Users
	s.userStore = newUserStore(conf.HomeFile("users.json"))

	s.authService = createAuthService(s.userStore)

	// Templates
	s.templateStore = newTemplateStore()

	// Consensus handler
	s.consensus = newConsensus()

	// Coordinator
	s.executionCoordinator = newExecutionCoordinator()

	// HTTP checks
	s.httpCheckStore = newHttpCheckStore()

	//Notifications
	s.notifications = newNotificationManager()

	// Print info
	log.Printf("Starting server at https://localhost:%d/", conf.ServerPort)

	// Start webserver
	go func() {
		router := httprouter.New()

		// Homepage that redirects to /console
		router.GET("/", Home)

		// For uptime checks
		router.GET("/ping", Ping)

		// List tags
		router.GET("/tags", GetTags)

		// Client commands
		router.GET("/client/:clientId/ping", ClientPing)
		router.GET("/client/:clientId/cmds", ClientCmds)
		router.PUT("/client/:clientId/cmd/:cmd/state", PutClientCmdState)
		router.PUT("/client/:clientId/cmd/:cmd/logs", PutClientCmdLogs)
		router.GET("/client/:clientId/cmd/:cmd/logs", GetClientCmdLogs)
		router.POST("/client/:clientId/auth", PostClientAuth)

		// Auth endpoint
		router.POST("/auth", PostAuth)

		// Templates
		router.GET("/templates", GetTemplate)
		router.POST("/template/:templateid/validation", PostTemplateValidation)
		router.DELETE("/template/:templateid/validation/:id", DeleteTemplateValidation)
		router.POST("/template", PostTemplate)
		router.DELETE("/template", DeleteTemplate)

		// Update password
		router.PUT("/user/password", PutUserPassword)

		// List clients (~ slaves)
		router.GET("/clients", GetClients)

		// List users
		router.GET("/users", GetUsers)

		// List user names by ids
		router.GET("/users/names", GetUsersNames)

		// Create user
		router.POST("/user", PostUser)

		// Remove user
		router.DELETE("/user", DeleteUser)

		//Change user
		router.PUT("/user", PutUser)

		// Consensus requests
		router.POST("/consensus/request", PostConsensusRequest)
		router.DELETE("/consensus/request", DeleteConsensusRequest)
		router.POST("/consensus/approve", PostConsensusApprove)
		router.GET("/consensus/pending", GetConsensusPending)

		// Dispatched commands list
		router.POST("/dispatched", data_table.DefaultStoreHandler(DispatchedCmdQuery))

		// Http checks
		router.GET("/http-check/:id", GetHttpCheck)
		router.GET("/http-checks", GetHttpChecks)
		router.POST("/http-check", PostHttpCheck)
		router.DELETE("/http-check", DeleteHttpCheck)

		// Two factor auth
		router.GET("/user/2fa", GetUser2fa)
		router.PUT("/user/2fa", PutUser2fa)

		// Backup
		router.GET("/backup/configs.zip", GetBackupConfigs)

		// Console endpoint for interface
		router.ServeFiles("/console/*filepath", http.Dir("console"))

		// Auto generate key
		if err := s._prepareTlsKeys(); err != nil {
			log.Printf("TLS preperation failed due to : %s", err)
			log.Fatal("Unable to start server")
		}

		// Start server
		log.Printf("Failed to start server %v", http.ListenAndServeTLS(fmt.Sprintf(":%d", conf.ServerPort), conf.GetSslCertFile(), conf.GetSslPrivateKeyFile(), router))
	}()

	// Minutely cleanups etc
	go func() {
		c := time.Tick(1 * time.Minute)
		for _ = range c {
			server.CleanupClients()
		}
	}()

	return true
}

func (s *Server) SetupNotifications(conf *Conf) {
	for _, n := range conf.GetNotifications() {
		if !n.IsEnabled() {
			continue
		}
		if err := n.IsValid(); err != nil {
			log.Printf("Invalid notification service config: %s", err)
		}
		service, err := n.GetService()
		if err != nil {
			log.Printf("Cannot instantiate notification service: %s", err)
		}
		s.notifications.register(service)
		log.Printf("Succesfully registerd notification %s", n.GetName())
	}
}

func createAuthService(us *UserStore) *AuthService {
	as := newAuthService(us, DefaultFirstFactorAuth, newGAuthAuthenticator())

	if conf.EnableLdap {
		as.appendFirstFactor(newLdapAuthenticator(conf.ldapConfig, us))
	}

	return as
}

// Get logs from dispatched job
func GetClientCmdLogs(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !authUser(r) {
		jr.Error("User not authorized for GetClientCmdLogs ")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Get client
	registeredClient := server.GetClient(ps.ByName("clientId"))
	if registeredClient == nil {
		jr.Error("Client not registered")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Command
	cmdId := ps.ByName("cmd")
	registeredClient.mux.RLock()
	cmd := registeredClient.DispatchedCmds[cmdId]
	registeredClient.mux.RUnlock()
	if cmd == nil {
		jr.Error("Command not found")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	jr.Set("log_output", cmd.BufOutput)
	jr.Set("log_error", cmd.BufOutputErr)

	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// Enable user two factor
func PutUser2fa(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !authUser(r) {
		jr.Error("User not authorized for PutUser2fa")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// User
	user := getUser(r)
	if len(user.TotpSecret) < 1 {
		jr.Error("Two factor authentication not setup")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}
	if user.HasTwoFactor() {
		jr.Error("Two factor authentication already setup")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Gather values
	value1 := r.PostFormValue("token_1")
	value2 := r.PostFormValue("token_2")
	if value1 == value2 || strings.TrimSpace(value1) == strings.TrimSpace(value2) {
		jr.Error("The two values should not be the same. Wait for the next token (in a few seconds) to show up and provide that.")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Validate
	valid1, _ := user.ValidateTotp(value1)
	valid2, _ := user.ValidateTotp(value2)
	res := valid1 && valid2 // Both must match
	if res == false {
		jr.Error("The two tokens do not match. Make sure that the clock is set correctly on your mobile device and the Indispenso server.")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Enable
	if res {
		user.TotpSecretValidated = true
		user.AuthType |= AUTH_TYPE_TWO_FACTOR
		server.userStore.save()
	}

	jr.Set("enabled", res)
	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// Get user two factor data
func GetUser2fa(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !authUser(r) {
		jr.Error("User not authorized for GetUser2fa")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// User
	user := getUser(r)
	if user.HasTwoFactor() {
		jr.Error("Two factor authentication already setup")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	if err := user.GenerateOTPSecret(); err != nil {
		jr.Error(err.Error())
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	server.userStore.save()

	qrImageBytes, err := user.TotpQrImage()
	if err != nil {
		jr.Error(err.Error())
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	jr.Set("Secret", user.TotpSecret)
	jr.Set("Png", qrImageBytes)
	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

func DispatchedCmdQuery(tableStore *data_table.DefaultStore) *data_table.DefaultStore {
	// Fetch and create
	server.clientsMux.RLock()
	for _, client := range server.clients {
		for _, d := range client.GetDispatchedCmds() {
			commandTime := time.Unix(d.Created, 0)
			row := make(map[string]interface{})
			row["created"] = commandTime.Format("2006-01-02 15:04:05")

			template := server.templateStore.Get(d.TemplateId)
			if template != nil {
				row["template"] = template.Title
			} else {
				row["template"] = "-"
			}

			user := server.userStore.ById(d.RequestUserId)
			if user != nil {
				row["user"] = user.Username
			} else {
				row["user"] = "-"
			}

			row["client"] = client.ClientId
			row["state"] = d.State
			row["link"] = fmt.Sprintf("logs?id=%s&client=%s", d.Id, client.ClientId)
			rowObj := tableStore.CreateRow(row)
			if time.Since(commandTime).Hours() > 24 {
				rowObj.RowClass = "history-old"
			}
			tableStore.AddRow(rowObj)
		}
	}
	server.clientsMux.RUnlock()

	return tableStore
}

// Get pending execution request
func GetConsensusPending(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !authUser(r) {
		jr.Error("User not authorized for GetConsensusPending")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}
	user := getUser(r)

	server.consensus.pendingMux.RLock()
	pending := make([]*ConsensusRequest, 0)
	work := make([]*ConsensusRequest, 0)
	for _, req := range server.consensus.Pending {
		// Ignore already executed
		if req.Executed {
			continue
		}

		// Ignore self
		if req.RequestUserId == user.Id {
			pending = append(pending, req)
			continue
		}

		// Voted?
		if req.ApproveUserIds[user.Id] == true {
			pending = append(pending, req)
			continue
		}

		work = append(work, req)
	}
	jr.Set("requests", pending)
	jr.Set("server_instance_id", server.InstanceId)
	jr.Set("work", work)
	server.consensus.pendingMux.RUnlock()

	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// Approve execution request
func PostConsensusApprove(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !authUser(r) {
		jr.Error("User not authorized for PostConsensusApprove")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	user := getUser(r)
	if !user.HasRole("approver") {
		jr.Error("User not allowed for PostConsensusApprove")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Vote
	id := strings.TrimSpace(r.PostFormValue("id"))
	req := server.consensus.Get(id)
	if req == nil {
		jr.Error("Request not found")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}
	res := req.Approve(user)
	server.consensus.save()

	jr.Set("approved", res)
	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// Cancel execution request
func DeleteConsensusRequest(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !authUser(r) {
		jr.Error("User not authorized for  DeleteConsensusRequest")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	user := getUser(r)
	if !user.HasRole("requester") {
		jr.Error("User not allowed to DeleteConsensusRequest")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Get template
	id := strings.TrimSpace(r.URL.Query().Get("id"))
	req := server.consensus.Get(id)
	if req == nil {
		jr.Error("Request not found")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Did we request this? Or are we admin
	isAdmin := user.HasRole("admin")
	isCreator := req.RequestUserId == user.Id
	if !isAdmin && !isCreator {
		jr.Error("Only the creator or admins can cancel a request")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Create request
	res := req.Cancel(user)
	server.consensus.save()

	jr.Set("cancelled", res)

	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// Create execution request
func PostConsensusRequest(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !authUser(r) {
		jr.Error("User not authorized for PostConsensusRequest")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Are we allow to request execution?
	user := getUser(r)
	if !user.HasRole("requester") {
		jr.Error("User not allowed to PostConsensusRequest")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Verify two factor for, so that a hacked account can not request or execute anything without getting access to the 2fa device
	if res, _ := user.ValidateTotp(r.PostFormValue("totp")); res == false {
		jr.Error("Invalid two factor token")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Reason
	reason := strings.TrimSpace(r.PostFormValue("reason"))
	if len(reason) < 4 {
		jr.Error("Please provide a valid reason")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Template
	templateId := strings.TrimSpace(r.PostFormValue("template"))
	clientIds := strings.Split(strings.TrimSpace(r.PostFormValue("clients")), ",")

	// Create request
	cr := server.consensus.AddRequest(templateId, clientIds, user, reason)
	cr.AddCallback(consensusRequestFinishedNotification)
	cr.check() // Check whether it can run straight away
	server.consensus.save()

	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

func consensusRequestFinishedNotification(consensusRequest *ConsensusRequest) {
	msg := fmt.Sprintf("Consesnsus request(id: %s) finished within %d s", consensusRequest.Id, consensusRequest.CompleteTime-consensusRequest.StartTime)
	server.notifications.Notify(&Message{Type: EXECUTION_DONE, Content: msg, Url: conf.ServerRequest("/console/#!history")})
}

// Create validation rule for templates
func PostTemplateValidation(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !authUser(r) {
		jr.Error("User not authorized for PostTemplateValidation")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Get template
	id := ps.ByName("templateid")
	template := server.templateStore.Get(id)
	if template == nil {
		jr.Error("Template not found")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Input
	txt := r.PostFormValue("text")
	isFatal := r.PostFormValue("fatal") == "1"
	mustContain := r.PostFormValue("must_contain") == "1"
	streamId := 1 // Default process output stream only

	// Text must have length
	if len(strings.TrimSpace(txt)) < 1 {
		jr.Error("Text can not be empty")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Create rule
	rule := newExecutionValidation(txt, isFatal, mustContain, streamId)

	// Add rule
	template.AddValidationRule(rule)

	// Save
	res := server.templateStore.save()

	// Done
	jr.Set("saved", res)
	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// Delete validation rule from template
func DeleteTemplateValidation(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !authUser(r) {
		jr.Error("User not authorized for DeleteTemplateValidation")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Get template
	templateId := ps.ByName("templateid")
	template := server.templateStore.Get(templateId)
	if template == nil {
		jr.Error("Template not found")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Validaton rule id
	id := ps.ByName("id")

	// Delete rule
	template.DeleteValidationRule(id)

	// Save
	res := server.templateStore.save()

	// Done
	jr.Set("saved", res)
	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// Get templates
func GetTemplate(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !authUser(r) {
		jr.Error("User not authorized for GetTemplate")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	server.templateStore.templateMux.RLock()
	jr.Set("templates", server.templateStore.Templates)
	server.templateStore.templateMux.RUnlock()
	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// Create template
func PostTemplate(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !authUser(r) {
		jr.Error("User not authorized for PostTemplate")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	user := getUser(r)
	if !user.HasRole("admin") {
		jr.Error("User not allowed to PostTemplate")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	title := strings.TrimSpace(r.PostFormValue("title"))
	description := strings.TrimSpace(r.PostFormValue("description"))
	command := r.PostFormValue("command")
	includedTags := r.PostFormValue("includedTags")
	excludedTags := r.PostFormValue("excludedTags")
	executionStrategyStr := r.PostFormValue("executionStrategy")

	// Create strategy
	var executionStrategy *ExecutionStrategy
	switch executionStrategyStr {
	case "simple":
		executionStrategy = newExecutionStrategy(SimpleExecutionStrategy)
		break
	case "one-test":
		executionStrategy = newExecutionStrategy(OneTestExecutionStrategy)
		break
	case "rolling":
		executionStrategy = newExecutionStrategy(RollingExecutionStrategy)
		break
	case "exponential-rolling":
		executionStrategy = newExecutionStrategy(ExponentialRollingExecutionStrategy)
		break
	default:
		jr.Error("Strategy not found")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Minimum authorizations
	minAuthStr := strings.TrimSpace(r.PostFormValue("minAuth"))
	minAuth, minAuthE := strconv.ParseInt(minAuthStr, 10, 0)
	if len(minAuthStr) < 1 {
		jr.Error("Fill in min auth")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	} else if minAuthE != nil {
		jr.Error(fmt.Sprintf("%s", minAuthE))
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	} else if minAuth < 1 {
		jr.Error("Min auth must be at least 1")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Timeout
	timeoutStr := strings.TrimSpace(r.PostFormValue("timeout"))
	timeout, timeoutE := strconv.ParseInt(timeoutStr, 10, 0)
	if len(timeoutStr) < 1 {
		jr.Error("Fill in timeout")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	} else if timeoutE != nil {
		jr.Error(fmt.Sprintf("%s", timeoutE))
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	} else if timeout < 1 {
		jr.Error("Timeout must be at least 1 second")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Validate template
	template := newTemplate(title, description, command, true, strings.Split(includedTags, ","), strings.Split(excludedTags, ","), uint(minAuth), int(timeout), executionStrategy)
	valid, err := template.IsValid()
	if !valid {
		jr.Error(fmt.Sprintf("%s", err))
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	server.templateStore.Add(template)
	server.templateStore.save()
	jr.Set("template", template)
	jr.Set("saved", true)
	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// Login
func PostAuth(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()

	authReq := &AuthRequest{
		login:      strings.TrimSpace(r.PostFormValue("username")),
		credential: strings.TrimSpace(r.PostFormValue("password")),
		token:      strings.TrimSpace(r.PostFormValue("2fa")),
	}

	user, err := server.authService.authUser(authReq)
	if err != nil {
		log.Printf("%s\n", err)
		jr.Error("Username / password / two-factor combination invalid") // Message must be constant to not leak information
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Start setssion
	token := user.StartSession()
	user.TouchSession(getIp(r))
	server.userStore.save() // Call save to persist token

	// Return token
	jr.Set("session_token", token)

	// User roles
	roles := make([]string, 0)
	for role := range user.Roles {
		roles = append(roles, role)
	}
	jr.Set("user_roles", roles)
	jr.Set("user_id", user.Id)
	jr.Set("two_factor_enabled", user.HasTwoFactor())
	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// List of all tags
func GetTags(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !authUser(r) {
		jr.Error("User not authorized for GetTags")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}
	server.tagsMux.RLock()
	tags := make([]string, 0)
	for tag := range server.Tags {
		tags = append(tags, tag)
	}
	jr.Set("tags", tags)
	server.tagsMux.RUnlock()
	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// Change password
func PutUserPassword(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !authUser(r) {
		jr.Error("User not authorized for PutUserPassword")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Validate password
	newPwd := r.PostFormValue("password")
	if len(newPwd) < 16 {
		jr.Error("Password must be at least 16 characters, please pick a strong one!")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Match passwords
	newPwd2 := r.PostFormValue("password2")
	if newPwd != newPwd2 {
		jr.Error("Please confirm your password")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Get user
	user := getUser(r)
	if user == nil {
		return
	}

	// Change password
	user.PasswordHash, _ = server.userStore.HashPassword(newPwd)
	server.userStore.save()

	jr.Set("saved", true)
	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// User from request
func getUser(r *http.Request) *User {
	// Username
	usr := r.Header.Get("X-Auth-User")

	// Get user
	user := server.userStore.ByName(usr)
	if user == nil {
		return nil
	}

	// Has token?
	if len(user.SessionToken) < 1 {
		return nil
	}

	// Enabled?
	if user.Enabled == false {
		return nil
	}

	// Token expired
	if time.Now().Sub(user.SessionLastTimestamp) > time.Duration(30*time.Minute) {
		return nil
	}

	// Validate token match
	if r.Header.Get("X-Auth-Session") != user.SessionToken {
		return nil
	}
	return user
}

// Delete template
func DeleteTemplate(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !authUser(r) {
		jr.Error("User not authorized for DeleteTemplate")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}
	usr := getUser(r)
	if !usr.HasRole("admin") {
		jr.Error("User not allowed to DeleteTemplate")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Username
	id := strings.TrimSpace(r.URL.Query().Get("id"))

	// Make sure it's not used by an HTTP check
	if len(server.httpCheckStore.FindByTemplate(id)) > 0 {
		jr.Error("This template is used by one or multiple http checks. You need to remove those first before deleting the template.")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Remove
	server.templateStore.Remove(id)
	server.templateStore.save()

	jr.Set("saved", true)
	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// Delete user
func DeleteUser(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !authUser(r) {
		jr.Error("User not authorized for DeleteUser")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}
	usr := getUser(r)
	if !usr.HasRole("admin") {
		jr.Error("User not allowed to DeleteUser")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Verify two factor for deletion of a user
	if res, _ := usr.ValidateTotp(r.URL.Query().Get("admin_totp")); res == false {
		jr.Error("Invalid two factor token")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Username
	username := strings.TrimSpace(r.URL.Query().Get("username"))

	// Can not remove yourself
	if usr.Username == username {
		jr.Error("You can not remove yourself. If you want to achieve this, make a new admin account. Login as that new account and then remove the old account.")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Get user
	server.userStore.RemoveByName(username)
	server.userStore.save()

	jr.Set("saved", true)
	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// Create user
func PostUser(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !authUser(r) {
		jr.Error("User not authorized for PostUser")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}
	usr := getUser(r)
	if !usr.HasRole("admin") {
		jr.Error("User not allowed to PostUser")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Verify two factor for creation of new user, so that a hacked admin can not create a new user and use that to sign of for new commands
	if res, _ := usr.ValidateTotp(r.PostFormValue("admin_totp")); res == false {
		jr.Error("Invalid two factor token")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Username
	username := r.PostFormValue("username")
	email := r.PostFormValue("email")

	// Validate password
	newPwd := r.PostFormValue("password")
	if len(newPwd) < 16 {
		jr.Error("Password must be at least 16 characters, please pick a strong one!")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Match passwords
	newPwd2 := r.PostFormValue("password2")
	if newPwd != newPwd2 {
		jr.Error("Please confirm your password")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Roles
	roles := strings.Split(r.PostFormValue("roles"), ",")

	// Create user
	res := server.userStore.CreateUser(username, newPwd, email, roles)
	server.userStore.save()

	jr.Set("saved", res)
	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// Modify user
func PutUser(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !authUser(r) {
		jr.Error("User not authorized for Change User")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}
	admin := getUser(r)
	if !admin.HasRole("admin") {
		jr.Error("User not allowed to Change User")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Verify two factor for change user
	if res, _ := admin.ValidateTotp(r.PostFormValue("token")); res == false {
		jr.Error("Invalid two factor token")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Username
	username := r.PostFormValue("username")
	user := server.userStore.ByName(username)
	if user == nil {
		jr.Error("Cannot find user to modify")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}
	for key, _ := range r.PostForm {
		switch key {
		case "enable":
			user.Enabled = cast.ToBool(r.PostFormValue(key))
		case "username", "token":
			continue
		default:
			jr.Error("Invalid change request")
			fmt.Fprint(w, jr.ToString(conf.Debug))
			return
		}
	}

	server.userStore.save()
	jr.Set("changed", true)
	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// Get user names
func GetUsersNames(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !authUser(r) {
		jr.Error("User not authorized for GetUsersNames")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}
	// Availble to anyone
	server.userStore.usersMux.RLock()
	users := make([]map[string]interface{}, 0)
	for _, userPtr := range server.userStore.Users {
		user := make(map[string]interface{})
		user["Id"] = userPtr.Id
		user["Username"] = userPtr.Username
		users = append(users, user)
	}
	jr.Set("users", users)
	server.userStore.usersMux.RUnlock()
	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// List users
func GetUsers(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !authUser(r) {
		jr.Error("User not authorized for GetUsers")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}
	usr := getUser(r)
	if !usr.HasRole("admin") {
		jr.Error("User not allowed to GetUsers")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}
	server.userStore.usersMux.RLock()
	users := make([]User, 0)
	for _, userPtr := range server.userStore.Users {
		user := *userPtr
		// Hide sensitive fields
		user.PasswordHash = ""
		user.SessionToken = ""
		user.TotpSecret = ""
		users = append(users, user)
	}
	jr.Set("users", users)
	jr.Set("authTypes", server.userStore.AuthTypes())
	server.userStore.usersMux.RUnlock()
	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// List clients
func GetClients(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !authUser(r) {
		jr.Error("User not authorized for GetClients")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Filters
	tagsInclude := strings.Split(r.URL.Query().Get("filter_tags_include"), ",")
	tagsExclude := strings.Split(r.URL.Query().Get("filter_tags_exclude"), ",")
	if len(tagsInclude) == 1 && tagsInclude[0] == "" {
		tagsInclude = make([]string, 0)
	}
	if len(tagsExclude) == 1 && tagsExclude[0] == "" {
		tagsExclude = make([]string, 0)
	}

	clients := make([]RegisteredClient, 0)
	server.clientsMux.RLock()
outer:
	for _, clientPtr := range server.clients {
		// Excluded? One match is enough to skip this one
		if len(tagsExclude) > 0 {
			for _, exclude := range tagsExclude {
				if clientPtr.HasTag(exclude) {
					continue outer
				}
			}
		}

		// Included? Must have all
		var match bool = true
		for _, include := range tagsInclude {
			if !clientPtr.HasTag(include) {
				match = false
				break
			}
		}
		if len(tagsInclude) > 0 && match == false {
			continue
		}

		// Deref, so we can modify the object without modifying the real one
		client := *clientPtr

		// Clear out the dispatched commands history (massive logs etc)
		client.DispatchedCmds = nil
		client.Cmds = nil

		// Add to list
		clients = append(clients, client)
	}
	server.clientsMux.RUnlock()

	jr.Set("clients", clients)
	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// Register client with token, this is used for signing commands towards the client which will then verify them
func PostClientAuth(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !auth(r) {
		jr.Error("User not authorized for PostClientAuth")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Get client
	registeredClient := server.GetClient(ps.ByName("clientId"))
	if registeredClient == nil {
		jr.Error("Client not registered")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Generate token and return
	token, tokenE := secureRandomString(32)
	if tokenE != nil {
		jr.Error("Failed to generate token")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Store token
	log.Printf(fmt.Sprintf("Client %s authenticated", registeredClient.ClientId))
	registeredClient.mux.Lock()
	registeredClient.AuthToken = token
	registeredClient.mux.Unlock()

	// Sign token based of our secure token
	hasher := sha256.New()
	hasher.Write([]byte(token))
	hasher.Write([]byte(conf.Token))
	tokenSignature := base64.URLEncoding.EncodeToString(hasher.Sum(nil))

	// Return token
	jr.Set("token", token)
	jr.Set("token_signature", tokenSignature)
	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// Set command logs
func PutClientCmdLogs(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !auth(r) {
		jr.Error("Client not authorized for PutClientCmdLogs")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Get client
	registeredClient := server.GetClient(ps.ByName("clientId"))
	if registeredClient == nil {
		jr.Error("Client not registered")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Command
	cmdId := ps.ByName("cmd")
	registeredClient.mux.RLock()
	cmd := registeredClient.DispatchedCmds[cmdId]
	registeredClient.mux.RUnlock()
	if cmd == nil {
		jr.Error("Command not found")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Read body
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		jr.Error("Failed to read body")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Decode json
	type LogStruct struct {
		Output []string `json:"output"`
		Error  []string `json:"error"`
	}
	var m *LogStruct
	je := json.Unmarshal(body, &m)
	if je != nil {
		jr.Error("Failed to parse json")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Append buffers
	if m.Output != nil {
		for _, line := range m.Output {
			cmd.BufOutput = append(cmd.BufOutput, line)
		}
	}

	// Append buffers
	if m.Error != nil {
		for _, line := range m.Error {
			cmd.BufOutputErr = append(cmd.BufOutputErr, line)
		}
	}

	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// Set command state
func PutClientCmdState(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !auth(r) {
		jr.Error("Client not authorized for PutClientCmdState")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Get client
	registeredClient := server.GetClient(ps.ByName("clientId"))
	if registeredClient == nil {
		jr.Error("Client not registered")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Command
	cmdId := ps.ByName("cmd")
	registeredClient.mux.RLock()
	cmd := registeredClient.DispatchedCmds[cmdId]
	registeredClient.mux.RUnlock()
	if cmd == nil {
		jr.Error("Command not found")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// State
	state := r.URL.Query().Get("state")

	// Save state in local server
	cmd.SetState(state)

	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// Commands
func ClientCmds(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !auth(r) {
		jr.Error("Client not authorized for ClientCmds")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Get client
	clientId := ps.ByName("clientId")
	registeredClient := server.GetClient(clientId)
	if registeredClient == nil {
		jr.Error(fmt.Sprintf("Client %s not registered", clientId))
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Do we have a token? If not, ignore as the client will discard the commands without hmac signatures
	if len(registeredClient.AuthToken) < 1 {
		jr.Error(fmt.Sprintf("Client %s auth token not available", registeredClient.ClientId))
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}

	// Read from channel and dispatch before timeout
	select {
	case <-registeredClient.CmdChan:
		cmds := make([]*Cmd, 0)
		registeredClient.mux.Lock()
		for _, cmd := range registeredClient.Cmds {
			if cmd.Pending {
				cmds = append(cmds, cmd)
				cmd.Pending = false
			}
		}
		registeredClient.mux.Unlock()
		jr.Set("cmds", cmds)
	case <-time.After(time.Second * LONG_POLL_TIMEOUT):
		// No commands
		jr.Set("cmds", make([]string, 0))
	}
	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// Ping
func ClientPing(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	jr := jresp.NewJsonResp()
	if !auth(r) {
		jr.Error("Client not authorized for ClientPing")
		fmt.Fprint(w, jr.ToString(conf.Debug))
		return
	}
	tags := strings.Split(r.URL.Query().Get("tags"), ",")
	server.RegisterClient(ps.ByName("clientId"), tags)
	jr.Set("ack", true)
	jr.Set("server_instance_id", server.InstanceId)
	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// Home
func Home(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	// Redirect to console
	http.Redirect(w, r, r.URL.String()+"console/", 301)
}

// Ping
func Ping(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	jr := jresp.NewJsonResp()
	jr.Set("ping", "pong")
	jr.OK()
	fmt.Fprint(w, jr.ToString(conf.Debug))
}

// Auth
func auth(r *http.Request) bool {
	// Signed token
	uri := r.URL.String()
	hasher := sha256.New()
	hasher.Write([]byte(uri))
	hasher.Write([]byte(conf.Token))
	signedToken := base64.URLEncoding.EncodeToString(hasher.Sum(nil))

	// Validate
	if r.Header.Get("X-Auth") != signedToken {
		return false
	}
	return true
}

// Auth user
func authUser(r *http.Request) bool {
	// Username
	user := getUser(r)
	if user == nil {
		return false
	}

	user.TouchSession(getIp(r))
	return true
}

// Get ip
func getIp(r *http.Request) string {
	return r.RemoteAddr
}

// Create new server
func newServer() *Server {
	id, _ := uuid.NewV4()
	return &Server{
		clients:    make(map[string]*RegisteredClient),
		Tags:       make(map[string]bool),
		InstanceId: id.String(),
	}
}

// New registered client
func newRegisteredClient(clientId string) *RegisteredClient {
	return &RegisteredClient{
		ClientId:       clientId,
		Cmds:           make(map[string]*Cmd),
		CmdChan:        make(chan bool),
		DispatchedCmds: make(map[string]*Cmd),
	}
}


text:
<!DOCTYPE html>

    <meta charset="utf-8">
    <meta name="description" content="Convert SVG to Android VectorDrawable">
    <meta name="keywords" content="android,svg,convert,vectordrawable,drawable,lollipop">
    <meta name="author" content="Juraj Novák">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Android SVG to VectorDrawable</title>
    <link href="css/bootstrap.min.css" rel="stylesheet">
    <link href="css/ribbons.min.css" rel="stylesheet">
    <link href="css/main.css" rel="stylesheet">
    <link rel="image_src" href="img/icon.png"/>

Android SVG to VectorDrawable

...one VectorDrawable to rule all screen densities

        Last update:

Sorry but your browser does not support some of new HTML5 features! Try using latest version of Google Chrome or
            Firefox.

            Drop file here (or multiple files) to load content or click on this box to open file dialog.

No file will be uploaded - uses only JavaScript HTML5 FileReader.

                Keep in mind that Android does support svg files only partially.

                Please report wrongly converted SVGs by filling an issue on github.

            <button type="button" class="btn btn-primary btn-sm" onclick="download(this)">Download
            </button>
            <button type="button" class="btn btn-default btn-sm" onclick="selectAll(this)">Select all
            </button>

        Success: create xml file and copy all contents into res/drawable directory

                    <input type="checkbox" class="opt-id-as-name" onchange="useIdAsName(this);">Import IDs from SVG as name

                    <input type="checkbox" class="clear-groups" onchange="clearGroups(this);">Remove empty groups without attributes

                    <input type="checkbox" class="wordwrap-pathdata" onchange="enableWordWrapPathData(this);">Word-wrap pathData

                    <input type="checkbox" class="bake-transforms" onchange="bakeTransforms(this);">Bake transforms into path (experimental)

                    <input type="checkbox" class="enable-clippaths" onchange="enableClipPaths(this);">Enable support for clip-path (work in progress)

            Android Support Library 23.2 has added support for VectorDrawable on Android API 7+ (more info)

                <button type="button" class="close" data-dismiss="modal" aria-label="Close">×</button>

Multiple SVG files ( files)

                <button type="button" class="btn btn-default" data-dismiss="modal">Cancel</button>
                <button type="button" class="btn btn-primary">Export</button>

        © 2018, Juraj Novák, inloop.eu

        | other projects |

<input type="file" id="dropzone-dialog" style="opacity: 0;display:none">

    Fork me on GitHub

redpajama_set_name: RedPajamaGithub


text:
from __future__ import division
from __future__ import print_function

"""
 ------------ Follow The Regularized Leader - Proximal ------------

FTRL-P is an online classification algorithm that combines both L1 and L2
norms, particularly suited for large data sets with extremely high dimensionality.

This implementation follow the algorithm by H. B. McMahan et. al. It minimizes
the LogLoss function iteratively with a combination of L2 and L1 (centralized
at the current point) norms and adaptive, per coordinate learning rates.

This algorithm is efficient at obtaining sparsity and has proven to perform
very well in massive Click-Through-Rate prediction tasks.

This module contains two objects...

References:
    * Follow-the-Regularized-Leader and Mirror Descent: Equivalent Theorems
      and L1 Regularization, H. Brendan McMahan
    * Ad Click Prediction: a View from the Trenches, H. Brendan McMahan et. al.

"""

from math import log, exp, fabs, sqrt
from csv import DictReader
from datetime import datetime
from random import random

def log_loss(y, p):
    """
    --- Log_loss computing function
    A function to compute the log loss of a predicted probability p given
    a true target y.

    :param y: True target value
    :param p: Predicted probability
    :return: Log loss.
    """
    p = max(min(p, 1. - 10e-15), 10e-15)
    return -log(p) if y == 1 else -log(1. - p)

class DataGen(object):
    """
    DataGen is an object to generate the data that is fed to the
    classifier.

    It reads the data file one row at a time, hashes it
    and returns it.

    The names and types of columns must be passed to it, so that categorical,
    target, numerical and identification columns can be treated differently.

    It also keeps track of the name and position of all features to allow
    the classifier to keep track of the coefficients by feature.
    """

    def __init__(self, max_features, target, descriptive=(), categorical=(), numerical=None, transformation=None):
        """
        The object initialized with the maximum number of features to be generated and the
        names of the appropriate columns.

        Categorical columns are hashed while numerical columns are kept as is, therefore
        care must be taken with normalization and pre processing.

        :param max_features: The maximum number of features to generate. It includes all
                             numerical and categorical features. Must be greater than the
                             number of numerical features.

        :param target: The name of the target variable. It must be a binary variable taking
                       values in {0, 1}.

        :param descriptive: Descriptive features that are used to identify the samples but
                            are not to be used for modelling, such as IDs, public
                            identifiers, etc.

        :param categorical: Categorical variable to be hashed.

        :param numerical: Numerical variable. These will not be hashed but will be used in
                          the modelling phase.

        """

        # --- Instance variables.
        # Instance variables are created for columns names and the number of numerical
        # columns in addition to all of the object's parameters.

        # Stores the maximum number of features to generate while hashing
        self.mf = max_features

        # Stores the name of the target variable.
        self.y = target

        # Stores a list with the names of all descriptive variables.
        self.ids = descriptive

        # Stores a list with the names of all categorical variables.
        self.cat = categorical

        # Stores a list with the names of all numerical variables.
        self.num = numerical

        # Stores a dictionary with the names of numerical variable to apply a given function to.
        self.tra = transformation if transformation is not None else {}

        # Dictionary to store names
        self.names = {}

        # --- Numerical features
        # Numerical features are indexed in sorted order. The number
        # of features is determined by the variable size. The value
        # of each feature is just the value read from the file. Start
        # by defining what is numeric. If the user does not pass the
        # names of all numerical features, the code will assume
        # every columns that is not id, target or categorical is
        # numeric and find their name when the training process begin.
        if self.num is not None:
            self.num_cols = sorted(self.num)

            # Store the names in our names dictionary
            self.names.update(dict(zip(self.num_cols, range(len(self.num_cols)))))
        else:
            self.num_cols = []

        # --- Something to build model on
        # Make sure the user passed some information on the columns to
        # be used to build the model upon
        assert len(self.cat) + len(self.num_cols) > 0, 'At least one categorical or numerical feature must ' \
                                                       'be provided.'

    def _fetch(self, path):
        """
        This method is the core reason this object exists. It is a python generator
        that hashes categorical variables, combines them to numerical variables and
        yields all the relevant information, row by row.

        :param path: Path of the data file to be read.

        :return: YIELDS the current row, ID information, feature values and the target value.
                 even if the file does not contain a target field it returns a target value
                 of zero anyway.
        """

        for t, row in enumerate(DictReader(open(path))):
            # --- Variables
            #   t: The current line being read
            # row: All the values in this line

            # --- Ids and other descriptive fields
            # Process any descriptive fields and put it all in a list.
            ids = []
            for ID in self.ids:
                ids.append(row[ID])
                del row[ID]

            # --- Target
            # Process target and delete its entry from row if it exists
            # otherwise just ignore and move along
            y = 0.
            if self.y in row:
                if row[self.y] == '1':
                    y = 1.
                del row[self.y]

            # --- Features
            # Initialize an empty dictionary to hold feature
            # indexes and their corresponding values.
            #
            x = {}

            # --- Enough features?
            # For the very first row make sure we have enough features (max features
            # is large enough) by computing the number of numerical columns and
            # asserting that the maximum number of features is larger than it.
            if t == 0:
                # --- Hash size
                # Computes a constant to add to hash index, it dictates the
                # number of features that will not be hashed
                num_size = len(self.num_cols)
                size = num_size + len(self.tra)

                # Make sure there is enough space for hashing
                assert self.mf > size, 'Not enough dimensions to fit all features.'

            # --- Numerical Variables
            # Now we loop over numerical variables
            for i, key in enumerate(self.num_cols):
                # --- No transformation
                # If no transformation is necessary, just store the actual value
                # of the variable.
                x[i] = float(row[key])

            # --- Transformations
            # Create on the fly transformed variables. The user passes a map of the
            # name of the new variable to a tuple containing the name of the original
            # variable to be transformed and the function to be applied to it.
            # Once completed the new name is appended to the names dictionary with its
            # corresponding index.#
            for i, key in enumerate(self.tra):
                # Start by addition to the data array x the new transformed values
                # by looping over new_features and applying the transformation to the
                # desired old feature.
                x[num_size + i] = self.tra[key][1](row[self.tra[key][0]])

                # Create a key in names dictionary with the new name and its
                # corresponding index.
                self.names[key] = num_size + i

            # --- Categorical features
            # Categorical features are hashed. For each different kind a
            # hashed index is created and a value of 1 is 'stored' in that
            # position.
            for key in self.cat:
                # --- Category
                # Get the categorial variable from row
                value = row[key]

                # --- Hash
                # One-hot encode everything with hash trick
                index = (abs(hash(key + '_' + value)) % (self.mf - size)) + size
                x[index] = 1.

                # --- Save Name
                # Save the name and index to the names dictionary if its a new feature
                # AND if there's still enough space.
                if key + '_' + value not in self.names and len(self.names) < self.mf:
                    self.names[key + '_' + value] = index

            # Yield everything.
            yield t, ids, x, y

    def train(self, path):
        """
        The train method is just a wrapper around the _fetch generator to comply
        with sklearn's API.

        :param path: The path for the training file.

        :return: YIELDS row, features, target value
        """

        # --- Generates train data
        # This is just a generator on top of the basic _fetch. If this was python 3 I
        # could use 'yield from', but I don't think this syntax exists in python 2.7,
        # so I opted to use the explicit, less pythonic way.
        for t, ids, x, y in self._fetch(path):
            # --- Variables
            #   t: Current row
            # ids: List of ID information
            #   x: Feature values
            #   y: Target values

            yield t, x, y

    def test(self, path):
        """
        The test method is just a wrapper around the _fetch generator to comply
        with sklearn's API.

        :param path: The path for the test file.

        :return: YIELDS row, features
        """

        # --- Generates test data
        # This is just a generator on top of the basic _fetch. If this was python 3 I
        # could use 'yield from', but I don't think this syntax exists in python 2.7,
        # so I opted to use the explicit, less pythonic way.
        for t, ids, x, y in self._fetch(path):
            # --- Variables
            #   t: Current row
            # ids: List of ID information
            #   x: Feature values
            #   y: Target values

            yield t, x

class FTRLP(object):
    """
    --- Follow The Regularized Leader - Proximal ---

    FTRL-P is an online classification algorithm that combines both L1 and L2
    norms, particularly suited for large data sets with extremely high dimensionality.

    This implementation follow the algorithm by H. B. McMahan et. al. It minimizes
    the LogLoss function iteratively with a combination of L2 and L1 (centralized
    at the current point) norms and adaptive, per coordinate learning rates.

    This algorithm is efficient at obtaining sparsity and has proven to perform
    very well in massive Click-Through-Rate prediction tasks.

    References:
        * Follow-the-Regularized-Leader and Mirror Descent: Equivalent Theorems
          and L1 Regularization, H. Brendan McMahan
        * Ad Click Prediction: a View from the Trenches, H. Brendan McMahan et. al.
    """

    def __init__(self, alpha=1, beta=1, l1=1, l2=1, subsample=1, epochs=1, rate=0):
        """
        Initializes the classifier's learning rate constants alpha and beta,
        the regularization constants L1 and L2, and the maximum number of
        features (limiting factor of the hash function).

        The per feature learning rate is given by:
                eta = alpha / ( beta + sqrt( sum g**g ) )

        :param alpha: Learning rate's proportionality constant.

        :param beta: Learning rate's parameter.

        :param l1: l1 regularization constant.

        :param l2: l2 regularization constant.

        :return:
        """

        # --- Classifier Parameters
        # The FTRLP algorithm has four free parameters that can be tuned as pleased.

        # Learning rate's proportionality constant.
        self.alpha = alpha
        # Learning rate's parameter.
        self.beta = beta
        # L1 regularization constant.
        self.l1 = l1
        # L2 regularization constant.
        self.l2 = l2

        # --- Log likelihood
        # Stores the log likelihood during the whole
        # fitting process.
        self.log_likelihood_ = 0
        self.loss = []

        # --- Weight parameters.
        # Lists and dictionaries to hold the weights. Initiate
        # the weight vector z and learning rate n as None so that
        # when self.train is called multiple times it will not
        # overwrite the stored values. This essentially allows epoch
        # training to take place, albeit a little bit ugly.
        self.z = None
        self.n = None
        # The weight vector used for prediction is constructed on the fly
        # and, in order to keep the memory cost low, it is a dictionary
        # that receives values and keys as needed.

        # --- Coefficients
        # Lists to store the coefficients and their corresponding names.
        # Initialized to None and constructed once the training method is
        # completed. In case of multiple epochs, these quantities will be
        # computed multiple times.
        self.coef_ = {}
        self.cname = None

        # --- Target Ratio
        # Store the ratio of each class of a binnary target variable to use
        # it to make weighted discrete label predictions.
        self.target_ratio = 0.

        # --- Printing Rate
        # Number of samples to train and predict on before printing
        # current status
        self.rate = rate

        # --- Subsample
        # While online methods can't be shuffle, combining subsampling of
        # the training set with multiple epoch training gives similar results.
        self.subsample = subsample

        # --- Epochs
        # something...
        self.epochs = epochs

        # --- Flag for partial fit
        # Keeps a flag to allow the user to train multiple times
        # without overwriting the object.
        self.fit_flag = False

    def _build_p(self, data_gen, path):
        # Maybe is worth migrating the weight construction algorithm
        # to here, I think it could clean up the code a little a bit
        # in both train and predict methods.
        pass

    def _clear_params(self):
        """
        If the fit method is called multiple times, all trained parameters
        must be cleared allowing for a fresh start. This function simply
        resets everything back to square one.

        :return: Nothing
        """

        # All models parameters are set to their original value (see
        # __init__ description
        self.log_likelihood_ = 0
        self.loss = []
        self.z = None
        self.n = None
        self.coef_ = {}
        self.cname = None

    def get_params(self, deep=True):
        """
        A function to return a map of parameters names and values.

        :param deep: Not sure yet, gotta check sklearn usage.

        :return: Dictionary mapping parameters names to their values
        """

        ps = {'alpha': self.alpha,
              'beta': self.beta,
              'l1': self.l1,
              'l2': self.l2,
              'subsample': self.subsample,
              'epochs': self.epochs,
              'rate': self.rate}

        return ps

    def set_params(self, **params):
        """

        :param params:
        :return:
        """

        for key, value in params.iteritems():
            setattr(self, key, value)

    def _update(self, y, p, x, w):
        """
        # --- Update weight vector and learning rate.
        # With the prediction round completed we can proceed to
        # updating the weight vector z and the learning rate eta
        # based on the last observed label.

        # To do so we will use the computed probability and target
        # value to find the gradient loss and continue from there.

        # The gradient for the log likelihood for round t can easily
        # be shown to be:
        #               g_i = (p - y) * x_i, (round t)

        # The remaining quantities are updated according to the
        # minimization procedure outlined in [2].

        :param y: True target variable
        :param p: Predicted probability for the current sample
        :param x: Non zero feature values
        :param w: Weights
        :return: Nothing
        """

        # --- Update loop
        # Loop over all relevant indexes and update all values
        # accordingly.
        for i in x.keys():
            # --- Compute Gradient of LogLoss
            g = (p - y) * x[i]

            # --- Update constant sigma
            # Note that this upgrade is equivalent to
            #       (eta_(t, i))^-1 - (eta_(t - 1, i))^-1
            # as discussed in [2].
            s = (sqrt(self.n[i] + g * g) - sqrt(self.n[i])) / self.alpha

            # --- Increment changes
            # Finally,  increment the appropriate changes to weights and
            # learning rate vectors.
            self.z[i] += g - s * w[i]
            self.n[i] += g * g

    def _train(self, data_gen, path):
        """
        --- Fitting method ---

        Online fitting method. It takes one sample at a time, builds
        the weight vector on the fly and computes the dot product of
        weight vector and values and a prediction is made.

        Then the true label of the target variable is observed and the
        loss is added.

        Once this is completed the weights are updated based on the
        previously observed values.

        :param data_gen: An instance of the DataGen class
        :param path: The path to the training set
        :return:
        """

        # Best way? Proper coding means no access to protected members...
        if self.z is None and self.n is None:
            self.z = [0.] * data_gen.mf
            self.n = [0.] * data_gen.mf

        # --- Start the clock!
        start_time = datetime.now()

        for t, x, y in data_gen.train(path):
            # --- Variables
            #   t: Current row
            #   x: Feature values
            #   y: Target values

            # --- Target Ratio Update
            # Rolling calculation of the target average
            self.target_ratio = (1.0 * (t * self.target_ratio + y)) / (t + 1)

            # --- Stochastic sample selection
            # Chose whether or not to use a sample in
            # training time. Since online methods can't
            # really be shuffle we can use this combined
            # with multiple epochs to create heterogeneity.
            #if random() > self.subsample and ((t + 1) % self.rate != 0):
            if random() > self.subsample and (t + 1) % self.rate != 0:
                continue

            # --- Dot product init.
            # The dot product is computed as the weights are calculated,
            # here it is initiated at zero.
            wtx = 0

            # --- Real time weights
            # Initialize an empty dictionary to hold the weights
            w = {}

            # --- Weights and prediction
            # Computes the weights for numerical features using the
            # indexes and values present in the x dictionary. And make
            # a prediction.

            # This first loop build the weight vector on the fly. Since
            # we expect most weights to be zero, the weight vector can
            # be constructed in real time. Furthermore, there is no
            # reason to store it, neither to clear it, since at each
            # iteration only the relevant indexes are populated and used.
            for indx in x.keys():
                # --- Loop over indicator I
                # x.keys() carries all the indexes of the feature
                # vector with non-zero entries. Therefore, we can
                # simply loop over it since anything else will not
                # contribute to the dot product w.x, and, consequently
                # to the prediction.
                if fabs(self.z[indx]) <= self.l1:
                    # --- L1 regularization
                    # If the condition on the absolute value of the
                    # vector Z is not met, the weight coefficient is
                    # set exactly to zero.
                    w[indx] = 0
                else:
                    # --- Non zero weight
                    # Provided abs(z_i) is large enough, the weight w_i
                    # is computed. First, the sign of z_i is determined.
                    sign = 1. if self.z[indx] >= 0 else -1.

                    # Then the value of w_i if computed and stored. Note
                    # that any previous value w_i may have had will be
                    # overwritten here. Which is fine since it will not
                    # be used anywhere outside this (t) loop.
                    w[indx] = - (self.z[indx] - sign * self.l1) / \
                                (self.l2 + (self.beta + sqrt(self.n[indx])) / self.alpha)

                # --- Update dot product
                # Once the value of w_i is computed we can use to compute
                # the i-th contribution to the dot product w.x. Which, here
                # is being done inside the index loop, compute only coordinates
                # that could possible be non-zero.
                wtx += w[indx] * x[indx]

            # --- Make a prediction
            # With the w.x dot product in hand we can compute the output
            # probability by putting wtx through the sigmoid function.
            # We limit wtx value to lie in the [-35, 35] interval to
            # avoid round off errors.
            p = 1. / (1. + exp(-max(min(wtx, 35.), -35.)))

            # --- Update the loss function
            # Now we look at the target value and use it, together with the
            # output probability that was just computed to find the loss we
            # suffer this round.
            self.log_likelihood_ += log_loss(y, p)

            # --- Verbose section
            if (self.rate > 0) and (t + 1) % self.rate == 0:
                # Append to the loss list.
                self.loss.append(self.log_likelihood_)

                # Print all the current information
                print('Training Samples: {0:9} | '
                      'Loss: {1:11.2f} | '
                      'Time taken: {2:4} seconds'.format(t + 1,
                                                         self.log_likelihood_,
                                                         (datetime.now() - start_time).seconds))

            # --- Update weights
            # Finally, we now how well we did this round and move on to
            # updating the weights based on the current status of our
            # knowledge.
            self._update(y, p, x, w)

        # --- Coefficient names and indexes
        # Bind the feature names to their corresponding coefficient obtained from
        # the regression.
        self.coef_.update(dict([[key, self.z[data_gen.names[key]]] for key in data_gen.names.keys()]))

    def fit(self, data_gen, path):
        """
        Epoch wrapper around the main fitting method _train

        :param data_gen: An instance of the DataGen class
        :param path: The path to the training set
        :return:
        """

        # --- Check fit flag
        # Make sure the fit methods is starting from a clean slate by
        # checking the fit_flag variable and calling the _clear_params
        # function if necessary.
        # While always calling _clear_params would do the job, by setting
        # this flag we are also able to call fit multiple times WITHOUT
        # clearing all parameters --- See partial_fit.
        if self.fit_flag:
            self._clear_params()

        # --- Start the clock!
        total_time = datetime.now()

        # Train epochs
        for epoch in range(self.epochs):
            # --- Start the clock!
            epoch_time = datetime.now()

            # --- Verbose
            # Print epoch if verbose is turned on
            if self.rate > 0:
                print('TRAINING EPOCH: {0:2}'.format(epoch + 1))
                print('-' * 18)

            self._train(data_gen, path)

            # --- Verbose
            # Print time taken if verbose is turned on
            if self.rate > 0:
                print('EPOCH {0:2} FINISHED IN {1} seconds'.format(epoch + 1,
                                                                   (datetime.now() - epoch_time).seconds))
                print()

        # --- Verbose
        # Print fit information if verbose is on
        if self.rate > 0:
            print(' --- TRAINING FINISHED IN '
                  '{0} SECONDS WITH LOSS {1:.2f} ---'.format((datetime.now() - total_time).seconds,
                                                             self.log_likelihood_))
            print()

        # --- Fit Flag
        # Set fit_flag to true. If fit is called again this is will trigger
        # the call of _clean_params. See partial_fit for different usage.
        self.fit_flag = True

    def partial_fit(self, data_gen, path):
        """
        Simple solution to allow multiple fit calls without overwriting
        previously calculated weights, losses and etc.

        :param data_gen: An instance of the DataGen class
        :param path: The path to the training set
        :return:
        """

        # --- Fit Flag
        # Start by reseting fit_flag to false to "trick"
        # the fit method into keep training without overwriting
        # previously calculated quantities.
        self.fit_flag = False

        # --- Fit
        # Call the fit method and proceed as normal
        self.fit(data_gen, path)

    def predict_proba(self, data_gen, path):
        """
        --- Predicting Probabilities method ---

        Predictions...

        :param data_gen: An instance of the DataGen class
        :param path: The path to the test set

        :return: A list with predicted probabilities
        """

        # --- Results
        # Initialize an empty list to hold predicted values.
        result = []

        # --- Start the clock!
        start_time = datetime.now()

        for t, x in data_gen.test(path):
            # --- Variables
            #   t: Current row
            #   x: Feature values

            # --- Dot product init.
            # The dot product is computed as the weights are calculated,
            # here it is initiated at zero.
            wtx = 0

            # --- Real time weights
            # Initialize an empty dictionary to hold the weights
            w = {}

            # --- Weights and prediction
            # Computes the weights for numerical features using the
            # indexes and values present in the x dictionary. And make
            # a prediction.

            # This first loop build the weight vector on the fly. Since
            # we expect most weights to be zero, the weight vector can
            # be constructed in real time. Furthermore, there is no
            # reason to store it, neither to clear it, since at each
            # iteration only the relevant indexes are populated and used.
            for indx in x.keys():
                # --- Loop over indicator I
                # x.keys() carries all the indexes of the feature
                # vector with non-zero entries. Therefore, we can
                # simply loop over it since anything else will not
                # contribute to the dot product w.x, and, consequently
                # to the prediction.
                if fabs(self.z[indx]) <= self.l1:
                    # --- L1 regularization
                    # If the condition on the absolute value of the
                    # vector Z is not met, the weight coefficient is
                    # set exactly to zero.
                    w[indx] = 0
                else:
                    # --- Non zero weight
                    # Provided abs(z_i) is large enough, the weight w_i
                    # is computed. First, the sign of z_i is determined.
                    sign = 1. if self.z[indx] >= 0 else -1.

                    # Then the value of w_i if computed and stored. Note
                    # that any previous value w_i may have had will be
                    # overwritten here. Which is fine since it will not
                    # be used anywhere outside this (t) loop.
                    w[indx] = - (self.z[indx] - sign * self.l1) / \
                                (self.l2 + (self.beta + sqrt(self.n[indx])) / self.alpha)

                # --- Update dot product
                # Once the value of w_i is computed we can use to compute
                # the i-th contribution to the dot product w.x. Which, here
                # is being done inside the index loop, compute only coordinates
                # that could possible be non-zero.
                wtx += w[indx] * x[indx]

            # --- Make a prediction
            # With the w.x dot product in hand we can compute the output
            # probability by putting wTx through the sigmoid function.
            # We limit wTx value to lie in the [-35, 35] interval to
            # avoid round off errors.
            result.append(1. / (1. + exp(-max(min(wtx, 35.), -35.))))

            # Verbose section - Still needs work...
            if (t + 1) % self.rate == 0:
                # print some stuff
                print('Test Samples: {0:8} | '
                      'Time taken: {1:3} seconds'.format(t + 1,
                                                         (datetime.now() - start_time).seconds))

        # All done, return the predictions!
        return result

    def predict(self, data_gen, path):
        """
        --- Predicting method ---

        Predictions...

        :param data_gen: An instance of the DataGen class
        :param path: The path to the test set

        :return: A list with predicted probabilities
        """

        # --- Probabilities
        # Compute probabilities by invoking the predict_proba method
        probs = self.predict_proba(data_gen, path)

        # --- Return
        # Return binary labels. The threshold is set using the mean value of the
        # target variable.
        return map(lambda x: 0 if x <= self.target_ratio else 1, probs)


text:
package daemon // import "github.com/docker/docker/daemon"

import (
	"context"
	"time"

	containerpkg "github.com/docker/docker/container"
	"github.com/docker/docker/errdefs"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// ContainerStop looks for the given container and stops it.
// In case the container fails to stop gracefully within a time duration
// specified by the timeout argument, in seconds, it is forcefully
// terminated (killed).
//
// If the timeout is nil, the container's StopTimeout value is used, if set,
// otherwise the engine default. A negative timeout value can be specified,
// meaning no timeout, i.e. no forceful termination is performed.
func (daemon *Daemon) ContainerStop(name string, timeout *int) error {
	container, err := daemon.GetContainer(name)
	if err != nil {
		return err
	}
	if !container.IsRunning() {
		return containerNotModifiedError{running: false}
	}
	if timeout == nil {
		stopTimeout := container.StopTimeout()
		timeout = &stopTimeout
	}
	if err := daemon.containerStop(container, *timeout); err != nil {
		return errdefs.System(errors.Wrapf(err, "cannot stop container: %s", name))
	}
	return nil
}

// containerStop sends a stop signal, waits, sends a kill signal.
func (daemon *Daemon) containerStop(container *containerpkg.Container, seconds int) error {
	if !container.IsRunning() {
		return nil
	}

	stopSignal := container.StopSignal()
	// 1. Send a stop signal
	if err := daemon.killPossiblyDeadProcess(container, stopSignal); err != nil {
		// While normally we might "return err" here we're not going to
		// because if we can't stop the container by this point then
		// it's probably because it's already stopped. Meaning, between
		// the time of the IsRunning() call above and now it stopped.
		// Also, since the err return will be environment specific we can't
		// look for any particular (common) error that would indicate
		// that the process is already dead vs something else going wrong.
		// So, instead we'll give it up to 2 more seconds to complete and if
		// by that time the container is still running, then the error
		// we got is probably valid and so we force kill it.
		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
		defer cancel()

		if status := <-container.Wait(ctx, containerpkg.WaitConditionNotRunning); status.Err() != nil {
			logrus.Infof("Container failed to stop after sending signal %d to the process, force killing", stopSignal)
			if err := daemon.killPossiblyDeadProcess(container, 9); err != nil {
				return err
			}
		}
	}

	// 2. Wait for the process to exit on its own
	ctx := context.Background()
	if seconds >= 0 {
		var cancel context.CancelFunc
		ctx, cancel = context.WithTimeout(ctx, time.Duration(seconds)*time.Second)
		defer cancel()
	}

	if status := <-container.Wait(ctx, containerpkg.WaitConditionNotRunning); status.Err() != nil {
		logrus.Infof("Container %v failed to exit within %d seconds of signal %d - using the force", container.ID, seconds, stopSignal)
		// 3. If it doesn't, then send SIGKILL
		if err := daemon.Kill(container); err != nil {
			// Wait without a timeout, ignore result.
			<-container.Wait(context.Background(), containerpkg.WaitConditionNotRunning)
			logrus.Warn(err) // Don't return error because we only care that container is stopped, not what function stopped it
		}
	}

	daemon.LogContainerEvent(container, "stop")
	return nil
}


text:
package com.yiibai.struts2;

public class HelloWorldAction {
	private String name;

	public String execute() throws Exception {
		System.out.println("Inside action....");
		return "success";
	}

	public String getName() {
		return name;
	}

	public void setName(String name) {
		this.name = name;
	}
}


text:
// HEADER
#include <dali-toolkit/internal/text/controller/text-controller-background-actor.h>

// EXTERNAL INCLUDES
#include <dali/public-api/rendering/renderer.h>

// INTERNAL INCLUDES
#include <dali-toolkit/devel-api/controls/control-depth-index-ranges.h>
#include <dali-toolkit/internal/graphics/builtin-shader-extern-gen.h>
#include <dali-toolkit/internal/text/cursor-helper-functions.h>
#include <dali-toolkit/internal/text/rendering/styles/character-spacing-helper-functions.h>
#include <dali-toolkit/internal/text/text-view.h>

namespace Dali::Toolkit::Text
{
namespace
{
struct BackgroundVertex
{
  Vector2 mPosition; ///< Vertex posiiton
  Vector4 mColor;    ///< Vertex color
};

struct BackgroundMesh
{
  Vector<BackgroundVertex> mVertices; ///< container of vertices
  Vector<unsigned short>   mIndices;  ///< container of indices
};
} // unnamed namespace

Length CalculateBackgroundLineHeight(LineRun lineRun)
{
  Length height = lineRun.ascender + -(lineRun.descender);

  if(lineRun.lineSpacing > 0)
  {
    height += lineRun.lineSpacing;
  }

  return height;
}

Actor CreateControllerBackgroundActor(const View& textView, const VisualModelPtr& textVisualModel, const LogicalModelPtr& textLogicalModel, Shader& textShaderBackground)
{
  // NOTE: Currently we only support background color for left-to-right text.

  Actor actor;

  Length numberOfGlyphs = textView.GetNumberOfGlyphs();
  if(numberOfGlyphs > 0u)
  {
    Vector<GlyphInfo> glyphs;
    glyphs.Resize(numberOfGlyphs);

    Vector<Vector2> positions;
    positions.Resize(numberOfGlyphs);

    // Get the line where the glyphs are laid-out.
    const LineRun* lineRun         = textVisualModel->mLines.Begin();
    float          alignmentOffset = lineRun->alignmentOffset;
    numberOfGlyphs                 = textView.GetGlyphs(glyphs.Begin(),
                                        positions.Begin(),
                                        alignmentOffset,
                                        0u,
                                        numberOfGlyphs);

    glyphs.Resize(numberOfGlyphs);
    positions.Resize(numberOfGlyphs);

    const GlyphInfo* const glyphsBuffer    = glyphs.Begin();
    const Vector2* const   positionsBuffer = positions.Begin();

    BackgroundMesh mesh;
    mesh.mVertices.Reserve(4u * glyphs.Size());
    mesh.mIndices.Reserve(6u * glyphs.Size());

    const Vector2 textSize = textView.GetLayoutSize();

    const float offsetX = alignmentOffset + textSize.width * 0.5f;
    const float offsetY = textSize.height * 0.5f;

    const Vector4* const    backgroundColorsBuffer       = textView.GetBackgroundColors();
    const ColorIndex* const backgroundColorIndicesBuffer = textView.GetBackgroundColorIndices();
    const Vector4&          defaultBackgroundColor       = textVisualModel->IsBackgroundEnabled() ? textVisualModel->GetBackgroundColor() : Color::TRANSPARENT;
    const float             modelCharacterSpacing        = textVisualModel->GetCharacterSpacing();
    Vector<CharacterIndex>& glyphToCharacterMap          = textVisualModel->mGlyphsToCharacters;
    const CharacterIndex*   glyphToCharacterMapBuffer    = glyphToCharacterMap.Begin();
    float                   calculatedAdvance            = 0.f;

    // Get the character-spacing runs.
    const Vector<CharacterSpacingGlyphRun>& characterSpacingGlyphRuns = textVisualModel->GetCharacterSpacingGlyphRuns();

    Vector4   quad;
    uint32_t  numberOfQuads = 0u;
    Length    yLineOffset   = 0;
    Length    prevLineIndex = 0;
    LineIndex lineIndex;

    for(uint32_t i = 0, glyphSize = glyphs.Size(); i < glyphSize; ++i)
    {
      const GlyphInfo& glyph = *(glyphsBuffer + i);

      // Get the background color of the character.
      // The color index zero is reserved for the default background color (i.e. Color::TRANSPARENT)
      const bool       isMarkupBackground       = textView.IsMarkupBackgroundColorSet();
      const ColorIndex backgroundColorIndex     = isMarkupBackground ? *(backgroundColorIndicesBuffer + i) : 0u;
      const bool       isDefaultBackgroundColor = (0u == backgroundColorIndex);
      const Vector4&   backgroundColor          = isDefaultBackgroundColor ? defaultBackgroundColor : *(backgroundColorsBuffer + backgroundColorIndex - 1u);

      lineIndex         = textVisualModel->GetLineOfGlyph(i);
      Length lineHeight = CalculateBackgroundLineHeight(lineRun[lineIndex]);

      if(lineIndex != prevLineIndex)
      {
        yLineOffset += CalculateBackgroundLineHeight(lineRun[prevLineIndex]);

        if(lineRun[prevLineIndex].lineSpacing < 0)
        {
          yLineOffset += lineRun[prevLineIndex].lineSpacing;
        }
      }

      // Only create quads for glyphs with a background color
      if(backgroundColor != Color::TRANSPARENT)
      {
        const float characterSpacing = GetGlyphCharacterSpacing(i, characterSpacingGlyphRuns, modelCharacterSpacing);

        const Vector2 position = *(positionsBuffer + i);
        calculatedAdvance      = GetCalculatedAdvance(*(textLogicalModel->mText.Begin() + (*(glyphToCharacterMapBuffer + i))), characterSpacing, glyph.advance);

        if(i == 0u && glyphSize == 1u) // Only one glyph in the whole text
        {
          quad.x = position.x;
          quad.y = yLineOffset;
          quad.z = quad.x + std::max(calculatedAdvance, glyph.xBearing + glyph.width);
          quad.w = lineHeight;
        }
        else if((lineIndex != prevLineIndex) || (i == 0u)) // The first glyph in the line
        {
          quad.x = position.x;
          quad.y = yLineOffset;
          quad.z = quad.x - glyph.xBearing + calculatedAdvance;
          quad.w = quad.y + lineHeight;
        }
        else if(i == glyphSize - 1u) // The last glyph in the whole text
        {
          quad.x = position.x - glyph.xBearing;
          quad.y = yLineOffset;
          quad.z = quad.x + std::max(calculatedAdvance, glyph.xBearing + glyph.width);
          quad.w = quad.y + lineHeight;
        }
        else // The glyph in the middle of the text
        {
          quad.x = position.x - glyph.xBearing;
          quad.y = yLineOffset;
          quad.z = quad.x + calculatedAdvance;
          quad.w = quad.y + lineHeight;
        }

        BackgroundVertex vertex;

        // Top left
        vertex.mPosition.x = quad.x - offsetX;
        vertex.mPosition.y = quad.y - offsetY;
        vertex.mColor      = backgroundColor;
        mesh.mVertices.PushBack(vertex);

        // Top right
        vertex.mPosition.x = quad.z - offsetX;
        vertex.mPosition.y = quad.y - offsetY;
        vertex.mColor      = backgroundColor;
        mesh.mVertices.PushBack(vertex);

        // Bottom left
        vertex.mPosition.x = quad.x - offsetX;
        vertex.mPosition.y = quad.w - offsetY;
        vertex.mColor      = backgroundColor;
        mesh.mVertices.PushBack(vertex);

        // Bottom right
        vertex.mPosition.x = quad.z - offsetX;
        vertex.mPosition.y = quad.w - offsetY;
        vertex.mColor      = backgroundColor;
        mesh.mVertices.PushBack(vertex);

        // Six indices in counter clockwise winding
        mesh.mIndices.PushBack(1u + 4 * numberOfQuads);
        mesh.mIndices.PushBack(0u + 4 * numberOfQuads);
        mesh.mIndices.PushBack(2u + 4 * numberOfQuads);
        mesh.mIndices.PushBack(2u + 4 * numberOfQuads);
        mesh.mIndices.PushBack(3u + 4 * numberOfQuads);
        mesh.mIndices.PushBack(1u + 4 * numberOfQuads);

        numberOfQuads++;
      }

      if(lineIndex != prevLineIndex)
      {
        prevLineIndex = lineIndex;
      }
    }

    // Only create the background actor if there are glyphs with background color
    if(mesh.mVertices.Count() > 0u)
    {
      Property::Map quadVertexFormat;
      quadVertexFormat["aPosition"] = Property::VECTOR2;
      quadVertexFormat["aColor"]    = Property::VECTOR4;

      VertexBuffer quadVertices = VertexBuffer::New(quadVertexFormat);
      quadVertices.SetData(&mesh.mVertices[0], mesh.mVertices.Size());

      Geometry quadGeometry = Geometry::New();
      quadGeometry.AddVertexBuffer(quadVertices);
      quadGeometry.SetIndexBuffer(&mesh.mIndices[0], mesh.mIndices.Size());

      if(!textShaderBackground)
      {
        textShaderBackground = Shader::New(SHADER_TEXT_CONTROLLER_BACKGROUND_SHADER_VERT, SHADER_TEXT_CONTROLLER_BACKGROUND_SHADER_FRAG);
      }

      Dali::Renderer renderer = Dali::Renderer::New(quadGeometry, textShaderBackground);
      renderer.SetProperty(Dali::Renderer::Property::BLEND_MODE, BlendMode::ON);
      renderer.SetProperty(Dali::Renderer::Property::DEPTH_INDEX, DepthIndex::CONTENT);

      actor = Actor::New();
      actor.SetProperty(Dali::Actor::Property::NAME, "TextBackgroundColorActor");
      actor.SetProperty(Actor::Property::PARENT_ORIGIN, ParentOrigin::TOP_LEFT);
      actor.SetProperty(Actor::Property::ANCHOR_POINT, AnchorPoint::TOP_LEFT);
      actor.SetProperty(Actor::Property::SIZE, textSize);
      actor.SetProperty(Actor::Property::COLOR_MODE, USE_OWN_MULTIPLY_PARENT_COLOR);
      actor.AddRenderer(renderer);
    }
  }

  return actor;
}

} // namespace Dali::Toolkit::Text


text:
:mod:`foolbox.zoo`
==================

.. automodule:: foolbox.zoo

Get Model
---------

.. autofunction:: get_model

Fetch Weights
-------------

.. autofunction:: fetch_weights


text:
ACCEPTED

#### According to
International Plant Names Index

#### Published in
null

#### Original name
Caralluma acutiloba N.E.Br.

### Remarks
null


text:
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/google/syzkaller/pkg/bisect"
	"github.com/google/syzkaller/pkg/config"
	"github.com/google/syzkaller/pkg/debugtracer"
	"github.com/google/syzkaller/pkg/mgrconfig"
	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/pkg/vcs"
)

var (
	flagConfig            = flag.String("config", "", "bisect config file")
	flagCrash             = flag.String("crash", "", "dir with crash info")
	flagFix               = flag.Bool("fix", false, "search for crash fix")
	flagKernelCommit      = flag.String("kernel_commit", "", "original kernel commit")
	flagKernelCommitTitle = flag.String("kernel_commit_title", "", "original kernel commit title")
	flagSyzkallerCommit   = flag.String("syzkaller_commit", "", "original syzkaller commit")
)

type Config struct {
	Compiler string `json:"compiler"`
	// Currently either 'gcc' or 'clang'. Note that pkg/bisect requires
	// explicit plumbing for every os/compiler combination.
	CompilerType string `json:"compiler_type"`
	// BinDir must point to a dir that contains compilers required to build
	// older versions of the kernel. For linux, it needs to include several
	// compiler versions.
	BinDir        string `json:"bin_dir"`
	Ccache        string `json:"ccache"`
	KernelRepo    string `json:"kernel_repo"`
	KernelBranch  string `json:"kernel_branch"`
	SyzkallerRepo string `json:"syzkaller_repo"`
	// Directory with user-space system for building kernel images
	// (for linux that's the input to tools/create-gce-image.sh).
	Userspace string `json:"userspace"`
	// Sysctl/cmdline files used to build the image which was used to crash the kernel, e.g. see:
	// dashboard/config/upstream.sysctl
	// dashboard/config/upstream-selinux.cmdline
	Sysctl  string `json:"sysctl"`
	Cmdline string `json:"cmdline"`

	KernelConfig         string `json:"kernel_config"`
	KernelBaselineConfig string `json:"kernel_baseline_config"`

	// Manager config that was used to obtain the crash.
	Manager json.RawMessage `json:"manager"`
}

func main() {
	flag.Parse()
	os.Setenv("SYZ_DISABLE_SANDBOXING", "yes")
	mycfg := new(Config)
	if err := config.LoadFile(*flagConfig, mycfg); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	mgrcfg, err := mgrconfig.LoadData(mycfg.Manager)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	if mgrcfg.Workdir == "" {
		mgrcfg.Workdir, err = ioutil.TempDir("", "syz-bisect")
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to create temp dir: %v\n", err)
			os.Exit(1)
		}
		defer os.RemoveAll(mgrcfg.Workdir)
	}
	cfg := &bisect.Config{
		Trace: &debugtracer.GenericTracer{
			TraceWriter: os.Stdout,
			OutDir:      *flagCrash,
		},
		Fix:             *flagFix,
		DefaultCompiler: mycfg.Compiler,
		CompilerType:    mycfg.CompilerType,
		BinDir:          mycfg.BinDir,
		Ccache:          mycfg.Ccache,
		Kernel: bisect.KernelConfig{
			Repo:        mycfg.KernelRepo,
			Branch:      mycfg.KernelBranch,
			Commit:      *flagKernelCommit,
			CommitTitle: *flagKernelCommitTitle,
			Userspace:   mycfg.Userspace,
			Sysctl:      mycfg.Sysctl,
			Cmdline:     mycfg.Cmdline,
		},
		Syzkaller: bisect.SyzkallerConfig{
			Repo:   mycfg.SyzkallerRepo,
			Commit: *flagSyzkallerCommit,
		},
		Manager: mgrcfg,
	}
	loadFile("", mycfg.KernelConfig, &cfg.Kernel.Config, true)
	loadFile("", mycfg.KernelBaselineConfig, &cfg.Kernel.BaselineConfig, false)
	loadFile(*flagCrash, "repro.prog", &cfg.Repro.Syz, false)
	loadFile(*flagCrash, "repro.cprog", &cfg.Repro.C, false)
	loadFile(*flagCrash, "repro.opts", &cfg.Repro.Opts, false)

	if len(cfg.Repro.Syz) == 0 && len(cfg.Repro.C) == 0 {
		fmt.Fprintf(os.Stderr, "no repro.cprog or repro.prog found\n")
		os.Exit(1)
	}

	if cfg.Syzkaller.Commit == "" {
		cfg.Syzkaller.Commit = vcs.HEAD
	}
	if cfg.Kernel.Commit == "" {
		cfg.Kernel.Commit = vcs.HEAD
	}

	result, err := bisect.Run(cfg)
	if err != nil {
		fmt.Fprintf(os.Stderr, "bisection failed: %v\n", err)
		os.Exit(1)
	}

	saveResultCommits(result.Commits)
}

func loadFile(path, file string, dst *[]byte, mandatory bool) {
	filename := filepath.Join(path, file)
	if !mandatory && !osutil.IsExist(filename) {
		return
	}
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	*dst = data
}

func saveResultCommits(commits []*vcs.Commit) {
	var result string
	if len(commits) > 0 {
		for _, commit := range commits {
			result = result + commit.Hash + "\n"
		}
	} else if *flagFix {
		result = "the crash still happens on HEAD\n"
	} else {
		result = "the crash already happened on the oldest tested release\n"
	}

	var fileName string
	if *flagFix {
		fileName = filepath.Join(*flagCrash, "fix.commit")
	} else {
		fileName = filepath.Join(*flagCrash, "cause.commit")
	}
	osutil.WriteFile(fileName, []byte(result))
}


text:
package net.sansa_stack.rdf.common.qualityassessment.utils

import collection.JavaConverters._
import com.typesafe.config.{ Config, ConfigFactory }

/*
 * DataSet Utils.
 * @author Gezim Sejdiu
 */
object DatasetUtils extends Serializable {

  @transient lazy val conf: Config = ConfigFactory.load("metrics.conf")

  val prefixes = conf.getStringList("rdf.qualityassessment.dataset.prefixes").asScala.toList

  /*
   * Subject Class URI
   * @return Class of subjects for which property value is checked.
   */
  val subject: String = conf.getString("rdf.qualityassessment.dataset.subject")

  /*
   * Property URI
   * @return Property to be checked.
   */
  val property: String = conf.getString("rdf.qualityassessment.dataset.property")

  /*
   * LowerBound
   * Lower bound to evaluate.
   */
  val lowerBound: Double = conf.getDouble("rdf.qualityassessment.dataset.lowerBound")

  /*
   * UpperBound
   * Upper bound to evaluate.
   */
  val upperBound: Double = conf.getDouble("rdf.qualityassessment.dataset.upperBound")

  val shortURIThreshold: Double = conf.getDouble("rdf.qualityassessment.dataset.shortUri.threshold")
}