text:
//go:build windows
// +build windows

package main

import (
	"fmt"
	"os"
	"time"

	"github.com/alecthomas/kong"
	"github.com/scjalliance/resourceful/enforcer"
	"github.com/scjalliance/resourceful/policy"
	"github.com/scjalliance/resourceful/provider/fsprov"
	"golang.org/x/sys/windows/svc"
)

// Commands that we accept.
const (
	cmdsAccepted = svc.AcceptStop | svc.AcceptShutdown | svc.AcceptPauseAndContinue
)

// Handler is a windows service handler.
type Handler struct {
	Name    string
	Conf    EnforceConfig
	ConfErr error
	Logger  enforcer.Logger
}

// Run causes the service to run under the given name until it is instructed
// to stop by the operating system.
func (h Handler) Run() error {
	return svc.Run(h.Name, h)
}

// Execute performs service request processing for windows.
func (h Handler) Execute(args []string, requests <-chan svc.ChangeRequest, changes chan<- svc.Status) (ssec bool, errno uint32) {
	// In all circumstances, indicate that we're stopping when we exit
	defer func() {
		changes <- svc.Status{State: svc.StopPending}
	}()

	// Track progress
	var checkpoint uint32

	// Indicate to the system that we're starting up
	checkpoint = sendProgress(changes, checkpoint)

	h.log("Program Arguments: %#v", os.Args)

	// Check for errors in the arguments provided by the OS
	if h.ConfErr != nil {
		h.log("Invalid Program Arguments: %v", h.ConfErr)
		return false, 1
	}

	// Parse arguments provided by the service manager
	if len(args) > 0 {
		h.Name = args[0]
	}
	if len(args) > 1 {
		args = args[1:]
		h.log("Service Arguments: %#v", args)
		var cmd ServiceCmd
		parser := kong.Must(&cmd)
		if _, err := parser.Parse(args); err != nil {
			h.log("Invalid Service Arguments: %v", err)
			return false, 1
		}
		/*
			} else if command != enforceCmd.FullCommand() {
				h.log("Invalid Service Command: %s", command)
				return false, 1
			}
		*/
		if cmd.Server != "" {
			h.Conf.Server = cmd.Server
		}
		if cmd.Passive {
			h.Conf.Passive = true
		}
	}
	checkpoint = sendProgress(changes, checkpoint)

	// Determine how to invoke a UI process within a session
	executable, err := os.Executable()
	if err != nil {
		h.log("Failed to query executable: %v", err)
		return false, 1
	}
	uiCommand := enforcer.Command{
		Path: executable,
		Args: []string{"ui"},
	}
	checkpoint = sendProgress(changes, checkpoint)

	// Examine the environment
	environment, err := buildEnvironment()
	if err != nil {
		h.log("Failed to collect environment: %v\n", err)
		return false, 1
	}
	checkpoint = sendProgress(changes, checkpoint)

	// Prepare a guardian client
	client := newClient(h.Conf.Server)
	checkpoint = sendProgress(changes, checkpoint)

	// Determine the policy cache directory
	polDir, err := cacheDir()
	if err != nil {
		h.log("Failed to locate cache directory: %v\n", err)
	}

	// Prepare a filesystem policy provider that can act as a policy cache
	var cache policy.Cache
	if polDir != "" {
		prov := fsprov.New(polDir)
		defer prov.Close()
		cache = prov
	}

	// Prepare the service
	service := enforcer.New(client, time.Second, time.Minute, uiCommand, environment, cache, h.Conf.Passive, h.Logger)
	checkpoint = sendProgress(changes, checkpoint)

	// Start the service
	if err := service.Start(); err != nil {
		h.log("Failed to start service: %v.", err)
		return false, 1
	}

	// Stop the service when we exit
	defer service.Stop()

	// Switch to the running state
	changes <- svc.Status{State: svc.Running, Accepts: cmdsAccepted}

	// Main service loop
	for {
		req := <-requests
		switch req.Cmd {
		case svc.Interrogate:
			changes <- req.CurrentStatus
			// Testing deadlock from https://code.google.com/p/winsvc/issues/detail?id=4
			time.Sleep(100 * time.Millisecond)
			changes <- req.CurrentStatus
		case svc.Stop, svc.Shutdown:
			changes <- svc.Status{State: svc.StopPending}
			h.log("Service shutting down.")
			return false, 0
		case svc.Pause:
			changes <- svc.Status{State: svc.Paused, Accepts: cmdsAccepted}
			h.log("Service paused.")
			service.Stop()
		case svc.Continue:
			changes <- svc.Status{State: svc.Running, Accepts: cmdsAccepted}
			h.log("Service unpaused.")
			service.Start()
		}
	}
}

func (h *Handler) log(format string, v ...interface{}) {
	if h.Logger == nil {
		return
	}
	h.Logger.Log(enforcer.ServiceEvent{
		Msg: fmt.Sprintf(format, v...),
	})
}

func (h *Handler) debug(format string, v ...interface{}) {
	if h.Logger == nil {
		return
	}
	h.Logger.Log(enforcer.ServiceEvent{
		Msg:   fmt.Sprintf(format, v...),
		Debug: true,
	})
}

func sendProgress(changes chan<- svc.Status, checkpoint uint32) uint32 {
	changes <- svc.Status{State: svc.StartPending, CheckPoint: checkpoint}
	return checkpoint + 1
}


text:
QuickSpecBegin(SDLHMIPermissionsSpec)

describe(@"Getter/Setter Tests", ^ {
    it(@"Should set and get correctly", ^ {
        SDLHMIPermissions* testStruct = [[SDLHMIPermissions alloc] init];

        testStruct.allowed = [@[SDLHMILevelBackground, SDLHMILevelFull] copy];
        testStruct.userDisallowed = [@[SDLHMILevelNone, SDLHMILevelLimited] copy];

        expect(testStruct.allowed).to(equal([@[SDLHMILevelBackground, SDLHMILevelFull] copy]));
        expect(testStruct.userDisallowed).to(equal([@[SDLHMILevelNone, SDLHMILevelLimited] copy]));
    });

    it(@"Should get correctly when initialized", ^ {
        NSMutableDictionary* dict = [@{SDLNameAllowed:[@[SDLHMILevelBackground, SDLHMILevelFull] copy],
                                       SDLNameUserDisallowed:[@[SDLHMILevelNone, SDLHMILevelLimited] copy]} mutableCopy];
        SDLHMIPermissions* testStruct = [[SDLHMIPermissions alloc] initWithDictionary:dict];

        expect(testStruct.allowed).to(equal([@[SDLHMILevelBackground, SDLHMILevelFull] copy]));
        expect(testStruct.userDisallowed).to(equal([@[SDLHMILevelNone, SDLHMILevelLimited] copy]));
    });

    it(@"Should return nil if not set", ^ {
        SDLHMIPermissions* testStruct = [[SDLHMIPermissions alloc] init];

        expect(testStruct.allowed).to(beNil());
        expect(testStruct.userDisallowed).to(beNil());
    });
});

QuickSpecEnd

redpajama_set_name: RedPajamaGithub


text:
import { QAcademicoStrategy } from './index';
import { asyncTest } from '../../../test-utils';
import { StrategyFactory, StrategyType } from '../factory';
import { QBoletim } from './qboletim';

describe('QBoletim', () => {

  let strategy: QAcademicoStrategy;

  beforeAll(asyncTest(async () => {
    strategy = await StrategyFactory.build(StrategyType.QACADEMICO, 'http://localhost:9595') as any;
    await strategy.login('test', '12345');
    spyOn(strategy, 'getUrl').and.callThrough();
  }));

  it('deve buscar pagina de boletim conforme periodo', asyncTest(async () => {
    await QBoletim.getBoletim(strategy, {
      nome: '',
      codigo: '2017_1'
    });
    expect(strategy.getUrl).toHaveBeenCalledWith('http://localhost:9595/index.asp?t=2032&cmbanos=2017&cmbperiodos=1');
  }));

  it('deve ler todos os dados da tabela', asyncTest(async () => {
    const ret = await QBoletim.getBoletim(strategy, {
      nome: '',
      codigo: '2017_1'
    });
    expect(ret.length).toBe(12);
    ret.forEach(disciplina => {
      const keys = [
        'Componente Curricular', 'CH', 'Turma', 'T. Faltas', 'M Final',
        '1E', 'F', '1R1E', 'F', 'NF1E', '2E',
        'F', '1R2E', 'F', 'Situação'
      ];
      const objKeys = Object.keys(disciplina);
      keys.forEach(key => {
        expect(objKeys).toContain(key);
        expect(disciplina[key]).toBeDefined();
      });
    });
  }));

});


text:
/************************************************************************
*
* Flood Project © (2008-201x)
* Licensed under the simplified BSD license. All rights reserved.
*
************************************************************************/

#include "Entity.h"
#include "Component.h"
#include "Geometry.h"
#include "Memory.h"
#include "Transform.h"

using namespace System;
using namespace System::Runtime::InteropServices;

Flood::Entity::Entity(::Entity* native)
{
    NativePtr = native;
}

Flood::Entity::Entity(System::IntPtr native)
{
    auto __native = (::Entity*)native.ToPointer();
    NativePtr = __native;
}

Flood::Entity::Entity()
{
    NativePtr = new ::Entity();
}

Flood::Entity::Entity(System::String^ name)
{
    auto arg0 = clix::marshalString<clix::E_UTF8>(name);
    NativePtr = new ::Entity(arg0);
}

bool Flood::Entity::IsVisible()
{
    auto __ret = ((::Entity*)NativePtr)->isVisible();
    return __ret;
}

bool Flood::Entity::GetTag(int index)
{
    auto arg0 = (::int32)(::int32_t)index;
    auto __ret = ((::Entity*)NativePtr)->getTag(arg0);
    return __ret;
}

void Flood::Entity::SetTag(int index, bool state)
{
    auto arg0 = (::int32)(::int32_t)index;
    ((::Entity*)NativePtr)->setTag(arg0, state);
}

bool Flood::Entity::AddComponent(Flood::Component^ component)
{
    auto arg0 = (::Component*)component->NativePtr;
    auto __ret = ((::Entity*)NativePtr)->addComponent(arg0);
    return __ret;
}

bool Flood::Entity::RemoveComponent(Flood::Component^ component)
{
    auto arg0 = (::Component*)component->NativePtr;
    auto __ret = ((::Entity*)NativePtr)->removeComponent(arg0);
    return __ret;
}

Flood::Component^ Flood::Entity::GetComponent(System::String^ name)
{
    auto _arg0 = clix::marshalString<clix::E_UTF8>(name);
    auto arg0 = _arg0.c_str();
    auto __ret = ((::Entity*)NativePtr)->getComponent(arg0);
    return (__ret.get() == nullptr) ? nullptr : gcnew Flood::Component((::Component*)__ret.get());
}

bool Flood::Entity::AddTransform()
{
    auto __ret = ((::Entity*)NativePtr)->addTransform();
    return __ret;
}

void Flood::Entity::Update(float delta)
{
    ((::Entity*)NativePtr)->update(delta);
}

void Flood::Entity::SendEvents()
{
    ((::Entity*)NativePtr)->sendEvents();
}

bool Flood::Entity::Equals(System::Object^ object)
{
    if (!object) return false;
    auto obj = dynamic_cast<Entity^>(object);

    if (!obj) return false;
    return __Instance == obj->__Instance;
}

int Flood::Entity::GetHashCode()
{
    return (int)NativePtr;
}

Flood::Entity^ Flood::Entity::Create(Flood::Allocator^ _0)
{
    auto arg0 = (::Allocator*)_0->NativePtr;
    auto __ret = ::EntityCreate(arg0);
    if (__ret == nullptr) return nullptr;
    return (__ret == nullptr) ? nullptr : gcnew Flood::Entity((::Entity*)__ret);
}

System::IntPtr Flood::Entity::__Instance::get()
{
    return System::IntPtr(NativePtr);
}

void Flood::Entity::__Instance::set(System::IntPtr object)
{
    NativePtr = (::Entity*)object.ToPointer();
}

System::String^ Flood::Entity::Name::get()
{
    auto &__ret = ((::Entity*)NativePtr)->getName();
    return (System::String^)(clix::marshalString<clix::E_UTF8>(__ret));
}

void Flood::Entity::Name::set(System::String^ name)
{
    auto arg0 = clix::marshalString<clix::E_UTF8>(name);
    ((::Entity*)NativePtr)->setName(arg0);
}

void Flood::Entity::Visible::set(bool v)
{
    ((::Entity*)NativePtr)->setVisible(v);
}

Flood::Entity^ Flood::Entity::Parent::get()
{
    auto __ret = ((::Entity*)NativePtr)->getParent();
    if (__ret == nullptr) return nullptr;
    return (__ret == nullptr) ? nullptr : gcnew Flood::Entity((::Entity*)__ret);
}

void Flood::Entity::Parent::set(Flood::Entity^ v)
{
    auto arg0 = (::Entity*)v->NativePtr;
    ((::Entity*)NativePtr)->setParent(arg0);
}

Flood::Transform^ Flood::Entity::Transform::get()
{
    auto __ret = ((::Entity*)NativePtr)->getTransform();
    return (__ret.get() == nullptr) ? nullptr : gcnew Flood::Transform((::Transform*)__ret.get());
}

System::Collections::Generic::List<Flood::Geometry^>^ Flood::Entity::Geometry::get()
{
    auto __ret = ((::Entity*)NativePtr)->getGeometry();
    auto _tmp__ret = gcnew System::Collections::Generic::List<Flood::Geometry^>();
    for(auto _element : __ret)
    {
        auto _marshalElement = (_element.get() == nullptr) ? nullptr : gcnew Flood::Geometry((::Geometry*)_element.get());
        _tmp__ret->Add(_marshalElement);
    }
    return _tmp__ret;
}

void Flood::Entity::ComponentAdded::add(System::Action<Flood::Component^>^ evt)
{
    if (!_ComponentAddedDelegateInstance)
    {
        _ComponentAddedDelegateInstance = gcnew _ComponentAddedDelegate(this, &Flood::Entity::_ComponentAddedRaise);
        auto _fptr = (void (*)(const ::ComponentPtr&))Marshal::GetFunctionPointerForDelegate(_ComponentAddedDelegateInstance).ToPointer();
        ((::Entity*)NativePtr)->onComponentAdded.Connect(_fptr);
    }
    _ComponentAdded = static_cast<System::Action<Flood::Component^>^>(System::Delegate::Combine(_ComponentAdded, evt));
}

void Flood::Entity::ComponentAdded::remove(System::Action<Flood::Component^>^ evt)
{
    _ComponentAdded = static_cast<System::Action<Flood::Component^>^>(System::Delegate::Remove(_ComponentAdded, evt));
}

void Flood::Entity::ComponentAdded::raise(Flood::Component^ _0)
{
    _ComponentAdded(_0);
}

void Flood::Entity::_ComponentAddedRaise(const ::ComponentPtr& _0)
{
    ComponentAdded::raise((_0.get() == nullptr) ? nullptr : gcnew Flood::Component((::Component*)_0.get()));
}

void Flood::Entity::ComponentRemoved::add(System::Action<Flood::Component^>^ evt)
{
    if (!_ComponentRemovedDelegateInstance)
    {
        _ComponentRemovedDelegateInstance = gcnew _ComponentRemovedDelegate(this, &Flood::Entity::_ComponentRemovedRaise);
        auto _fptr = (void (*)(const ::ComponentPtr&))Marshal::GetFunctionPointerForDelegate(_ComponentRemovedDelegateInstance).ToPointer();
        ((::Entity*)NativePtr)->onComponentRemoved.Connect(_fptr);
    }
    _ComponentRemoved = static_cast<System::Action<Flood::Component^>^>(System::Delegate::Combine(_ComponentRemoved, evt));
}

void Flood::Entity::ComponentRemoved::remove(System::Action<Flood::Component^>^ evt)
{
    _ComponentRemoved = static_cast<System::Action<Flood::Component^>^>(System::Delegate::Remove(_ComponentRemoved, evt));
}

void Flood::Entity::ComponentRemoved::raise(Flood::Component^ _0)
{
    _ComponentRemoved(_0);
}

void Flood::Entity::_ComponentRemovedRaise(const ::ComponentPtr& _0)
{
    ComponentRemoved::raise((_0.get() == nullptr) ? nullptr : gcnew Flood::Component((::Component*)_0.get()));
}


text:
/* ----------------------------------------------------------------------------
 * This file was automatically generated by SWIG (http://www.swig.org).
 * Version 3.0.5
 *
 * This file is not intended to be easily readable and contains a number of
 * coding conventions designed to improve portability and efficiency. Do not make
 * changes to this file unless you know what you are doing--modify the SWIG
 * interface file instead.
 * ----------------------------------------------------------------------------- */

#ifndef SWIG_libtorrent_WRAP_H_
#define SWIG_libtorrent_WRAP_H_

struct SwigDirector_swig_plugin : public swig_plugin, public Swig::Director {

public:
    void swig_connect_director(JNIEnv *jenv, jobject jself, jclass jcls, bool swig_mem_own, bool weak_global);
    SwigDirector_swig_plugin(JNIEnv *jenv);
    virtual ~SwigDirector_swig_plugin();
    virtual swig_torrent_plugin *new_torrent(libtorrent::torrent *t);
    virtual void added();
    virtual void on_alert(libtorrent::alert const *a);
    virtual bool on_unknown_torrent(libtorrent::sha1_hash const &info_hash, libtorrent::peer_connection *pc, libtorrent::add_torrent_params &p);
    virtual void on_tick();
    virtual bool on_optimistic_unchoke(std::vector< libtorrent::policy::peer * > &peers);
    virtual void save_state(libtorrent::entry &e) const;
    virtual void load_state(libtorrent::lazy_entry const &n);
public:
    bool swig_overrides(int n) {
      return (n < 8 ? swig_override[n] : false);
    }
protected:
    bool swig_override[8];
};

struct SwigDirector_swig_torrent_plugin : public swig_torrent_plugin, public Swig::Director {

public:
    void swig_connect_director(JNIEnv *jenv, jobject jself, jclass jcls, bool swig_mem_own, bool weak_global);
    SwigDirector_swig_torrent_plugin(JNIEnv *jenv);
    virtual ~SwigDirector_swig_torrent_plugin();
    virtual swig_peer_plugin *new_peer_connection(libtorrent::peer_connection *pc);
    virtual swig_peer_plugin *new_bt_peer_connection(libtorrent::bt_peer_connection *pc);
    virtual swig_peer_plugin *new_web_peer_connection(libtorrent::web_peer_connection *pc);
    virtual swig_peer_plugin *new_http_seed_connection(libtorrent::http_seed_connection *pc);
    virtual void on_piece_pass(int index);
    virtual void on_piece_failed(int index);
    virtual void tick();
    virtual bool on_pause();
    virtual bool on_resume();
    virtual void on_files_checked();
    virtual void on_state(int s);
    virtual void on_unload();
    virtual void on_load();
    virtual void on_add_peer(tcp::endpoint const &endp, int src, int flags);
public:
    bool swig_overrides(int n) {
      return (n < 14 ? swig_override[n] : false);
    }
protected:
    bool swig_override[14];
};

struct SwigDirector_swig_peer_plugin : public swig_peer_plugin, public Swig::Director {

public:
    void swig_connect_director(JNIEnv *jenv, jobject jself, jclass jcls, bool swig_mem_own, bool weak_global);
    SwigDirector_swig_peer_plugin(JNIEnv *jenv);
    virtual ~SwigDirector_swig_peer_plugin();
    virtual char const *type() const;
    virtual void add_handshake(libtorrent::entry &e);
    virtual void on_disconnect(boost::system::error_code const &ec);
    virtual void on_connected();
    virtual bool on_handshake(char const *reserved_bits);
    virtual bool on_extension_handshake(libtorrent::lazy_entry const &n);
    virtual bool on_choke();
    virtual bool on_unchoke();
    virtual bool on_interested();
    virtual bool on_not_interested();
    virtual bool on_have(int index);
    virtual bool on_dont_have(int index);
    virtual bool on_bitfield(libtorrent::bitfield const &bitfield);
    virtual bool on_have_all();
    virtual bool on_have_none();
    virtual bool on_allowed_fast(int index);
    virtual bool on_request(libtorrent::peer_request const &r);
    virtual bool on_piece(libtorrent::peer_request const &piece, libtorrent::disk_buffer_holder &data);
    virtual bool on_cancel(libtorrent::peer_request const &r);
    virtual bool on_reject(libtorrent::peer_request const &r);
    virtual bool on_suggest(int index);
    virtual void sent_unchoke();
    virtual void sent_payload(int bytes);
    virtual bool can_disconnect(boost::system::error_code const &ec);
    virtual void on_piece_pass(int index);
    virtual void on_piece_failed(int index);
    virtual void tick();
    virtual bool write_request(libtorrent::peer_request const &r);
public:
    bool swig_overrides(int n) {
      return (n < 28 ? swig_override[n] : false);
    }
protected:
    bool swig_override[28];
};

#endif


text:
package com.helger.commons.exception.mock;

/**
 * This is a marker interface for mock exceptions. It should be used to easily
 * identify mock exceptions from regular exceptions.
 *
 * @author Philip Helger
 */
public interface IMockException
{
  /* empty */
}


text:
layout: post
title: Notes on Gradient Descent Optimization
category: researchnotes
---

## Source
---
> [An overview of gradient descent optimization
algorithms](https://arxiv.org/abs/1609.04747)

## Short summary for each algorithm

Here I summarize few modern gradient descent algorithms as they are so common to be seen in research papers. I do not understand why many researchers don't appreciate the work in this line. However, to me, they do look amazing and intelligent, and that's why I would like to write a note down here.

Most of the content comes from the above source paper, which is the best summary of this sort of algorithms I have seen in recent years. I recommend that you read it.

### Momentum

Accelerate along the slope.

$$\begin{align*}
  v_t &= \gamma v_{t-1} + \eta \nabla_{\theta} J(\theta) \\
  \theta &= \theta - v_t
\end{align*}$$

### Nesterov Accelerated Gradient

Slow down the velocities before hitting the local optimum by looking ahead the estimated update $$\theta - \gamma v_{t-1}$$.

$$\begin{align*}
  v_t &= \gamma v_{t-1} + \eta \nabla_{\theta} J(\theta - \gamma v_{t-1}) \\
  \theta &= \theta - v_t
\end{align*}$$

### AdaGrad

It introduces per-parameter updates or learning rates, which is good for sparse data, since it can run a bigger step for the parameters that are rarely updated, and vice versa. On successful story for this is that *Glove*, a well-known word embedding model, uses this for updating rare words' embedding.

$$\begin{align*}
  \theta_{t+1} &= \theta_t - \frac{\eta}{\sqrt{G_t + \epsilon}} \odot g_t
\end{align*}$$

where $$G_t$$ is the sum of squared historical gradients.

### AdaDelta

The weaknesses of AdaGrad are (1) that the learning rate will eventually become infinitesimally small and (2) that the updates based on the gradients and the learning rates are different in "units".

To solve the first problem, it uses a decaying average gradients:
\\[
  G_t = \rho G_{t-1} + (1-\rho) g_t^2
\\]
And for the second problem, they create another variable $$\Delta x_t$$ and store histories of the variable using the same way as the gradients so that it maintains the hypothetical "units". Note that the $$\Delta x_t$$ can also be noted as $$\Delta \theta_t$$ if that makes you feel more compatible to other algorithms.

$$\begin{align*}
\Delta x_t &= - \frac{\sqrt{D_{t-1} + \epsilon}}{\sqrt{G_t + \epsilon}} g_t \\
D_t &= \eta D_{t-1} + (1-\eta) \Delta x_t^2 \\
x_{t+1} &= x_t + \Delta x_t
\end{align*}$$

Note that this method doesn't even need the initial learning rate. It completely depends on the historical $$\Delta x_t$$.

### RMSprop

This is basically the AdaDelta that skips the unit issue.

$$\begin{align*}
G_t &= \rho G_{t-1} + (1-\rho) g_t^2 \\
\theta_{t+1} &= \theta_t - \frac{\eta}{\sqrt{G_t + \epsilon}} g_t
\end{align*}$$

### Adam

A mix of AdaGrad and RMSProp. Adam calculates a decaying average of the gradient and the squared gradient, controlled by $$\beta_1$$ and $$\beta_2$$.

$$\begin{align*}
m_t &= \beta_1 m_{t-1} + (1-\beta_1) g_t \\
v_t &= \beta_1 v_{t-1} + (1-\beta_1) g_t^2
\end{align*}$$

where $$m_t$$ is the first moment (mean) and $$v_t$$ is the second moment (uncentered variance). Moreover, to avoid that in the early steps they would bias toward zero, a bias correlation mechanism is adopted.

$$\begin{align*}
\hat{m}_t &= \frac{m_t}{1-\beta_1^t} \\
\hat{v}_t &= \frac{v_t}{1-\beta_2^t}
\end{align*}$$

Lastly, the Adam update rule
\\[
  \theta_{t+1} = \theta_t - \frac{\eta}{\sqrt{\hat{v}_t} + \epsilon} \hat{m}_t
\\]

The author also suggests values for each parameter:
- $$\eta$$: 0.1
- $$\beta_1$$: 0.9
- $$\beta_2$$: 0.999
- $$\epsilon$$: 10e-8

## Take home
In modern deep learning frameworks, two commonly suggested optimization algorithms worth trying first are
- SGD + Nesterov
- Adam

## References
1. https://machinelearningmastery.com/adam-optimization-algorithm-for-deep-learning/
2. http://cs231n.github.io/neural-networks-3/
2. [An overview of gradient descent optimization
algorithms](https://arxiv.org/abs/1609.04747)
3. [Adam: A Method for Stochastic Optimization](https://arxiv.org/abs/1412.6980)


text:
"""The logger object and user-friendly interface for constructing it."""
import io
import os
from collections import deque, Sequence

import simplejson as json

from . import utils
from .formatters import simple_formatter
from .serialization import serialize_numpy, loads
from .handlers import (GzipJSONHandler, JSONHandler, PrintHandler,
                       PersistentServerHandler, ServerHandler)

def Logger(filename=None, maxlen=0, stream=False, stream_maxlen=0,
           formatter=simple_formatter, push_port=5557, router_port=5556,
           **kwargs):
    r"""A pseudo-class for easy initialization of a log.

    .. note::

       Note that the `stream` and `stream_maxlen` arguments are
       independent. In order to respond to requests from streaming clients,
       past entries are stored in a separate thread when `stream_maxlen >
       0`, which is not the case for the entries stored when `maxlen > 0`.

    Parameters
    ----------
    filename : str, optional
        The file to save the log to in newline delimited JSON format. If
        the filename ends in `.gz` it will be compressed on the fly.
    maxlen : int or None, optional
        The number of entries to store for later retrieval. By default this
        is 0 i.e. entries are discarded after being fed to the handlers.
        For unlimited memory pass ``None``. When `maxlen > 0` normal list
        indexing can be used e.g. ``log[-1]`` to access the last entry. Use
        ``None`` for unlimited memory.
    stream : bool, optional
        If `True`, log entries will be published over a ZeroMQ socket.
        Defaults to `False`.
    stream_maxlen : int or None, optional
        How many entries the log should keep in memory so that clients of
        the streaming interface can request data that they missed. Ignored
        when `stream` is `False`, defaults to `0`. Use ``None`` for
        unlimited memory.
    formatter : function, optional
        A formatter function that determines how log entries will be
        printed to standard output. If `None`, entries will not be printed
        at all. Defaults to :func:`simple_formatter`.
    push_port : int, optional
        The port over which log entries will be published if `stream` is
        true. Defaults to 5557.
    router_port : int, optional
        The port over which snapshots will be sent if `stream_maxlen > 0`.
        Defaults to 5556.
    \*\*kwargs
        Keyword arguments passed on to ``json.dumps``. By default
        ``ensure_ascii=False`` and ``default=serialize_numpy`` are passed.

    Returns
    -------
    :class:`_Logger`
        A logger object with the correct handlers.

    """
    handlers = []
    if filename:
        root, ext = os.path.splitext(filename)
        # If the file ends in .gz then gzip it
        if ext == '.gz':
            handlers.append(GzipJSONHandler(root))
        else:
            handlers.append(JSONHandler(io.open(filename, 'w')))
    if formatter:
        handlers.append(PrintHandler(formatter))
    if stream:
        if stream_maxlen != 0:
            handlers.append(PersistentServerHandler(
                push_port=push_port, router_port=router_port,
                maxlen=stream_maxlen
            ))
        else:
            handlers.append(ServerHandler(port=push_port))
    return _Logger(handlers, maxlen=maxlen, **kwargs)

class _Logger(Sequence):
    """A logger object.

    Parameters
    ----------
    handlers : list or None
        A list of :class:`Handler` objects, each of which will be called in
        the given order. If `None`, the log entry will simply be ignored.
    maxlen : int or None, optional
        See :func:`Logger`'s `maxlen` argument.
    \*\*kwargs
        Keyword arguments passed on to ``json.dumps``. By default
        ``ensure_ascii=False`` and ``default=serialize_numpy`` are passed.

    Attributes
    ----------
    handlers : list
        The list of handlers, which can be appended to and removed from as
        needed.

    """
    def __init__(self, handlers=None, maxlen=0, **kwargs):
        if not handlers:
            handlers = []
        self.handlers = handlers
        self._entries = deque([], maxlen=maxlen)
        # Ensure that json.dumps returns UTF-8 strings on Python 2
        kwargs.setdefault('ensure_ascii', False)
        kwargs.setdefault('default', serialize_numpy)
        self.json_kwargs = kwargs

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()

    def __getitem__(self, key):
        return self._entries[key]

    def __len__(self):
        return len(self._entries)

    def close(self):
        """Close the handlers."""
        for handler in self.handlers:
            handler.close()

    def load(self, filename, **kwargs):
        """Load log entries from the specified file.

        Parameters
        ----------
        filename : str
            The file to load from. If it ends in ``.gz`` it's assumed to be
            a gzipped file.
        \*\*kwargs
            Arguments passed on ``json.loads``, useful for deserializing
            non-basic objects. By default ``deserialize_numpy`` is passed.

        Returns
        -------
        num_entries : int
            The number of log entries in the file.

        """
        entries = deque([], maxlen=self._entries.maxlen)

        num_entries = 0
        with utils.open(filename, raw_text=True) as f:
            for entry in f:
                num_entries += 1
                entries.append(entry)
        for entry in entries:
            self._entries.append(loads(entry, **kwargs))
        return num_entries

    def log(self, entry):
        """Log an entry.

        Will check if the handler contains filters and apply them if needed
        (so that if multiple handlers have the same filters, they are only
        applied once). Likewise, it will check if the log entry needs to be
        serialized to JSON data, and make sure that the data is only
        serialized once for each set of filters.

        Parameters
        ----------
        entry : dict
            A log entry is a (JSON-compatible) dict.

        """
        # Store entry for retrieval
        self._entries.append(entry)

        # For each set of filters, we store the JSON serialized entry
        filtered_entries = {}
        serialized_entries = {}
        for handler in self.handlers:
            filters = frozenset(handler.filters)

            # If the content needs to be filtered, do so
            if filters:
                if filters not in filtered_entries:
                    filtered_entries[filters] = handler.filter(entry)
                filtered_entry = filtered_entries[filters]
            else:
                filtered_entry = entry

            # If the handler wants JSON data, give it
            if handler.JSON:
                if filters not in serialized_entries:
                    serialized_entries[filters] = json.dumps(
                        filtered_entry, **self.json_kwargs)
                serialized_entry = serialized_entries[filters]
                handler.log(serialized_entry)
            else:
                handler.log(filtered_entry)


text:
"""
The controllers of a typical glim app.

Example - Html rendering
------------------------
from glim import Controller
from glim_extensions.view import View

class HelloController(Controller):
    def hello(self):
        # render the jinja template by binding data
        # renders app/views/hello.html
        return View.render('hello', name = 'Aras')

# for ajax requests, if you tend to send html source
# of a rendered template, use View.source(*args, **kwargs)

Example - JsonResponse
----------------------
from glim import Controller

class BaseController(Controller):
	def hello(self):
		return {'error': None}
		# outputs {"error": null} with application/json header
"""

from glim import Controller

class AppController(Controller):
	def ping(self):
		return {'error': None, 'message': 'pong'}


text:
using Simplic.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace SqlDotNet.Compiler
{
    /// <summary>
    /// Preprocessor to clean and order all tokens before creating the syntax tree.
    /// Tokenizer --> TokenPreProcessor --> SyntaxTreeBuilder
    /// </summary>
    public class TokenPreProcessor
    {
        /// <summary>
        /// Process tokens, by cleaning them up and reorder them
        /// </summary>
        /// <param name="tokens">List of unprocessed tokens</param>
        /// <param name="parserConfig">Parser configuration</param>
        /// <param name="errorListener">Optional Error listener for error reporting</param>
        /// <returns>Dequeue of processed tokens</returns>
        public Dequeue<RawToken> Process(Dequeue<RawToken> tokens, ParserConfiguration parserConfig, IErrorListener errorListener = default(IErrorListener))
        {
            Dequeue<RawToken> processedTokens = new Dequeue<RawToken>();

            RawToken lastToken = null;

            // Token post-process
            while (tokens.Count > 0)
            {
                RawToken token = tokens.PopFirst();

                #region [Create decimal number (1 . 2 ==> 1.2)]
                if (token.Content == ".")
                {
                    if (processedTokens.Count > 0 && parserConfig.IsInt64(processedTokens.PeekLast().Content))
                    {
                        if (tokens.Count > 0)
                        {
                            RawToken lastProcssedToken = processedTokens.PopLast();
                            RawToken nextToken = tokens.PopFirst();

                            if (parserConfig.IsInt64(nextToken.Content) == false && nextToken.Content.EndsWith("d") == false && nextToken.Content.EndsWith("f"))
                            {
                                errorListener.Report("T0001", "Syntax error, exptected numeric value: " + nextToken.Content, nextToken.Index.Item1, nextToken.Index.Item2, nextToken);
                            }

                            token = new RawToken(lastProcssedToken.Content + token.Content + nextToken.Content, null, new Tuple<int, int>(lastProcssedToken.Index.Item1, nextToken.Index.Item2));
                        }
                        else
                        {
                            errorListener.Report("T0005", "Syntax error, expected token behind: " + token.Content, token.Index.Item1, token.Index.Item2, token);
                        }
                    }
                }
                #endregion

                #region [Classify tokens]
                // if a token starts or ends with ", it is only because the tokenizer should
                // handle this as one token. So we cam remove it here:

                if (token.Content.StartsWith("\"") && token.Content.EndsWith("\""))
                {
                    token.Content = token.Content.Substring(1, token.Content.Length - 2);
                }

                string tcontent = token.Content;

                var constType = parserConfig.GetConstDataType(token);

                if (token.Content.StartsWith("'") && token.Content.EndsWith("'"))
                {
                    token.Type = TokenType.Constant;
                }
                else if (token.Content.StartsWith("0x"))
                {
                    token.Type = TokenType.Constant;
                }
                else if (SyntaxTreeFactory.Singleton.StaticTokens.ContainsKey(tcontent))
                {
                    token.Type = SyntaxTreeFactory.Singleton.StaticTokens[tcontent].Type;
                }
                else if (constType != DataType.None)
                {
                    token.Type = TokenType.Constant;
                }
                else if (parserConfig.IsValidLanguageIndependentIdentifier(tcontent) || tcontent.StartsWith("$") || tcontent.StartsWith("@"))
                {
                    token.Type = TokenType.Name;

                    // Name to Owner, when <owner>.<name> is true
                    if (tokens.Count > 0 && tokens.PeekFirst().Content == ".")
                    {
                        tokens.PopFirst(); // Remove dot, because the owner is already detected
                        token.Type = TokenType.Owner;
                    }
                }
                else
                {
                    errorListener?.Report("T0006", "Invalid token: " + token.Content, token.Index.Item1, token.Index.Item2, token);
                }
                #endregion

                // Proof whether a star/multiply is used for select all columns or just for multiply values
                #region [Multiply/Start]
                if (token.Type == TokenType.Multiply)
                {
                    // Use start instead of multiply for the following scenarios: SELECT * or SELECT a.* or SELECT a, *
                    if (lastToken != null && (lastToken.Type == TokenType.Select || lastToken.Type == TokenType.Comma || lastToken.Type == TokenType.Owner))
                    {
                        token.Type = TokenType.Star;
                    }
                }
                #endregion

                lastToken = token;
                processedTokens.PushBack(token);
            }

            return processedTokens;
        }
    }
}


text:
require 'monitor'
require 'net/http'
require 'openssl'
require 'socket'
require 'soap/rpc/driver'
require 'syslog'
require 'yaml'

require 'rubygems'
require 'json'
require 'right_aws'
require 'zookeeper'

$:.unshift File.join(File.dirname(__FILE__), "lib")
require 'helperfunctions'
require 'cron_helper'
require 'haproxy'
require 'collectd'
require 'nginx'
require 'pbserver'
require 'blobstore'
require 'app_controller_client'
require 'user_app_client'
require 'ejabberd'
require 'repo'
require 'zkinterface'
require 'godinterface'

$VERBOSE = nil # to supress excessive SSL cert warnings
APPSCALE_HOME = ENV['APPSCALE_HOME']

FIREWALL_IS_ON = true

WANT_OUTPUT = true
NO_OUTPUT = false

require "#{APPSCALE_HOME}/AppDB/zkappscale/zookeeper_helper"
require "#{APPSCALE_HOME}/Neptune/neptune"

class Exception
  alias real_init initialize
  def initialize(*args)
    real_init *args

    File.open("#{APPSCALE_HOME}/.appscale/exception.log", "w+") { |file|
      file.write(message)
    }

    begin
      #Syslog.open("appscale") { |s| s.debug(message) }
      puts message
    rescue RuntimeError
      # this can occur if we write to syslog at the same time
      # as one of the python components
    end
  end

  alias real_set_backtrace set_backtrace

  def set_backtrace(*args)
    real_set_backtrace *args

    # Only print the last line of the backtrace so the logs arent polluted
    bt = self.backtrace #.last # cgb: need full trace for the moment
    output = "#{self.class}: #{self.message} #{bt}"

    File.open("#{APPSCALE_HOME}/.appscale/exception-full.log", "w+") { |file|
      file.write(output)
    }

    begin
      #Syslog.open("appscale") { |s| s.debug(output) }
    rescue RuntimeError
    end
  end
end

BAD_SECRET_MSG = "false: bad secret"
DJINN_SERVER_PORT = 17443
UA_SERVER_PORT = 4343
SSH_PORT = 22
USE_SSL = true

NEPTUNE_INFO = "/etc/appscale/neptune_info.txt"
STATE_FILE = "/etc/appscale/appcontroller-state.json"
HEALTH_FILE = "/etc/appscale/health.json"

class Djinn
  attr_accessor :job, :nodes, :creds, :app_names, :done_loading
  attr_accessor :port, :apps_loaded, :userappserver_public_ip
  attr_accessor :userappserver_private_ip, :state, :kill_sig_received
  attr_accessor :my_index, :total_boxes, :num_appengines, :restored
  attr_accessor :neptune_jobs, :neptune_nodes, :api_status

  public

  def done(secret)
    return BAD_SECRET_MSG unless valid_secret?(secret)
    return @done_loading
  end

  def kill(secret)
    return BAD_SECRET_MSG unless valid_secret?(secret)
    @kill_sig_received = true

    if is_hybrid_cloud?
      Thread.new {
        sleep(5)
        HelperFunctions.terminate_hybrid_vms(creds)
      }
    elsif is_cloud?
      Thread.new {
        sleep(5)
        infrastructure = creds["infrastructure"]
        keyname = creds["keyname"]
        HelperFunctions.terminate_all_vms(infrastructure, keyname)
      }
    else
      # in xen/kvm deployments we actually want to keep the boxes
      # turned on since that was the state they started in

      stop_ejabberd if my_node.is_login?
      Repo.stop if my_node.is_shadow?

      jobs_to_run = my_node.jobs
      commands = {
        "load_balancer" => "stop_load_balancer",
        "appengine" => "stop_appengine",
        "db_master" => "stop_db_master",
        "db_slave" => "stop_db_slave",
        "zookeeper" => "stop_zookeeper"
      }

      my_node.jobs.each do |job|
        if commands.include?(job)
          Djinn.log_debug("About to run [#{commands[job]}]")
          send(commands[job].to_sym)
        else
          Djinn.log_debug("Unable to find command for job #{job}. Skipping it.")
        end
      end

      if has_soap_server?(my_node)
        stop_soap_server
        stop_pbserver
      end
    end

    GodInterface.shutdown
    FileUtils.rm_rf(STATE_FILE)
    return "OK"
  end

  def set_parameters(djinn_locations, database_credentials, app_names, secret)
    return BAD_SECRET_MSG unless valid_secret?(secret)

    Djinn.log_debug("Djinn locations class: #{djinn_locations.class}")
    Djinn.log_debug("DB Credentials class: #{database_credentials.class}")
    Djinn.log_debug("Apps to load class: #{app_names.class}")

    ["djinn_locations", "database_credentials", "app_names"].each { |param_name|
      param = eval(param_name)
      if param.class != Array
        Djinn.log_debug "#{param_name} wasn't an Array. It was a/an #{param.class}"
        return "Error: #{param_name} wasn't an Array. It was a/an #{param.class}"
      end
    }

    # credentials is an array that we're converting to
    # hash tables, so we need to make sure that every key maps to a value
    # e.g., ['foo', 'bar'] becomes {'foo' => 'bar'}
    # so we need to make sure that the array has an even number of elements

    if database_credentials.length % 2 != 0
      error_msg = "Error: DB Credentials wasn't of even length: Len = " + \
        "#{database_credentials.length}"
      Djinn.log_debug(error_msg)
      return error_msg
    end

    possible_credentials = Hash[*database_credentials]
    if !valid_format_for_credentials(possible_credentials)
      return "Error: Credential format wrong"
    end

    Djinn.log_debug("parameters were valid")

    keyname = possible_credentials["keyname"]
    @nodes = Djinn.convert_location_array_to_class(djinn_locations, keyname)
    @creds = possible_credentials
    @app_names = app_names

    convert_fqdns_to_ips
    @creds = sanitize_credentials

    Djinn.log_debug("Djinn locations: #{@nodes.join(', ')}")
    Djinn.log_debug("DB Credentials: #{HelperFunctions.obscure_creds(@creds).inspect}")
    Djinn.log_debug("Apps to load: #{@app_names.join(', ')}")

    find_me_in_locations
    if @my_index == nil
      return "Error: Couldn't find me in the node map"
    end
    Djinn.log_debug("My index = #{@my_index}")

    ENV['EC2_URL'] = @creds['ec2_url']

    return "OK"
  end

  def set_apps(app_names, secret)
    return BAD_SECRET_MSG unless valid_secret?(secret)

    @app_names = app_names
    return "app names is now #{@app_names.join(', ')}"
  end

  def status(secret)
    return BAD_SECRET_MSG unless valid_secret?(secret)

    stats = get_stats(secret)

    stats_str = <<-STATUS
    Currently using #{stats['cpu']} Percent CPU and #{stats['mem']} Percent Memory
    Hard disk is #{stats['disk']} Percent full
    Is currently: #{stats['roles'].join(', ')}
    Database is at #{stats['db_location']}
    Is in cloud: #{stats['cloud']}
    Current State: #{stats['state']}
    STATUS

    if my_node.is_appengine?
      app_names = []
      stats['apps'].each { |k, v|
        app_names << k
      }

      stats_str << "    Hosting the following apps: #{app_names.join(', ')}"
    end

    return stats_str
  end

  def get_stats(secret)
    return BAD_SECRET_MSG unless valid_secret?(secret)

    usage = HelperFunctions.get_usage
    mem = sprintf("%3.2f", usage['mem'])

    job = my_node.jobs
    job = ["none"] if job.nil?

    # don't use an actual % below, or it will cause a string format exception
    stats = {}
    stats['ip'] = my_node.public_ip
    stats['cpu'] = usage['cpu']
    stats['memory'] = mem
    stats['disk'] = usage['disk']
    stats['roles'] = job
    stats['db_location'] = @userappserver_public_ip
    stats['cloud'] = my_node.cloud
    stats['state'] = @state

    stats['apps'] = {}
    @app_names.each { |name|
      stats['apps'][name] = @apps_loaded.include?(name)
    }

    return stats
  end

  def stop_app(app_name, secret)
    return BAD_SECRET_MSG unless valid_secret?(secret)
    app_name.gsub!(/[^\w\d\-]/, "")
    Djinn.log_debug("Shutting down app named [#{app_name}]")
    result = ""

    # app shutdown process can take more than 30 seconds
    # so run it in a new thread to avoid 'execution expired'
    # error messages and have the tools poll it
    Thread.new {
      # Tell other nodes to shutdown this application
      if @app_names.include?(app_name) and !my_node.is_appengine?
        @nodes.each { |node|
          next if node.private_ip == my_node.private_ip
          if node.is_appengine? or node.is_login?
            ip = node.private_ip
            acc = AppControllerClient.new(ip, @@secret)
            result = acc.stop_app(app_name)
            Djinn.log_debug("Removing application #{app_name} --- #{ip} returned #{result} (#{result.class})")
          end
        }
      end

      # Contact the soap server and remove the application
      if (@app_names.include?(app_name) and !my_node.is_appengine?) or @nodes.length == 1
        ip = HelperFunctions.read_file("#{APPSCALE_HOME}/.appscale/masters")
        uac = UserAppClient.new(ip, @@secret)
        result = uac.delete_app(app_name)
        Djinn.log_debug("Delete app: #{ip} returned #{result} (#{result.class})")
      end

      # may need to stop XMPP listener
      if my_node.is_login?
        pid_files = `ls #{APPSCALE_HOME}/.appscale/xmpp-#{app_name}.pid`.split
        unless pid_files.nil? # not an error here - XMPP is optional
          pid_files.each { |pid_file|
            pid = HelperFunctions.read_file(pid_file)
            `kill -9 #{pid}`
          }

          result = "true"
        end
      end

      if my_node.is_appengine?
        GodInterface.stop(app_name)
        GodInterface.remove(app_name)
        Nginx.remove_app(app_name)
        Collectd.remove_app(app_name)
        HAProxy.remove_app(app_name)
        Nginx.reload
        Collectd.restart
        ZKInterface.remove_app_entry(app_name)
        # TODO God does not shut down the application, so do it here for
        # A temp fix.
        `ps -ef | grep dev_appserver | grep #{app_name} | grep -v grep | grep cookie_secret | awk '{print $2}' | xargs kill -9`
        result = "true"
      end

      @apps_loaded = @apps_loaded - [app_name]
      @app_names = @app_names - [app_name]

      if @apps_loaded.empty?
        @apps_loaded << "none"
      end

      if @app_names.empty?
        @app_names << "none"
      end
    }

    return "true"
  end

  def update(app_names, secret)
    return BAD_SECRET_MSG unless valid_secret?(secret)

    apps = @app_names - app_names + app_names

    @nodes.each_index { |index|
      ip = @nodes[index].private_ip
      acc = AppControllerClient.new(ip, @@secret)
      result = acc.set_apps(apps)
      Djinn.log_debug("#{ip} returned #{result} (#{result.class})")
      @everyone_else_is_done = false if !result
    }

    # now that another app is running we can take out 'none' from the list
    # if it was there (e.g., run-instances with no app given)
    @app_names = @app_names - ["none"]

    return "OK"
  end

  def get_all_public_ips(secret)
    return BAD_SECRET_MSG unless valid_secret?(secret)
    public_ips = []
    @nodes.each { |node|
      public_ips << node.public_ip
    }
    return public_ips
  end

  def job_start(secret)
    return BAD_SECRET_MSG unless valid_secret?(secret)

    if restore_appcontroller_state
      parse_creds
    else
      wait_for_data
      parse_creds
      change_job(secret)
    end

    while !@kill_sig_received do
      @state = "Done starting up AppScale, now in heartbeat mode"
      write_database_info
      write_neptune_info
      backup_appcontroller_state
      update_api_status

      if my_node.is_shadow?
        @nodes.each { |node|
          get_status(node)

          if node.should_destroy?
            Djinn.log_debug("heartbeat - destroying node [#{node}]")
            @nodes.delete(node)
            @neptune_nodes.delete(node)
            infrastructure = @creds["infrastructure"]
            HelperFunctions.terminate_vms([node], infrastructure)
            FileUtils.rm_f("/etc/appscale/status-#{node.private_ip}.json")
          end
        }
        Djinn.log_debug("Finished contacting all other nodes")

        @neptune_nodes.each { |node|
          Djinn.log_debug("Currently examining node [#{node}]")
          if node.should_extend?
            Djinn.log_debug("extending time for node [#{node}]")
            node.extend_time
          elsif node.should_destroy?
            Djinn.log_debug("time is up for node [#{node}] - destroying it")
            @nodes.delete(node)
            @neptune_nodes.delete(node)
            infrastructure = @creds["infrastructure"]
            HelperFunctions.terminate_vms([node], infrastructure)
            FileUtils.rm_f("/etc/appscale/status-#{node.private_ip}.json")
          end
        }
      else
        Djinn.log_debug("No need to heartbeat, we aren't the shadow")
      end

      # TODO: consider only calling this if new apps are found
      start_appengine
      sleep(20)
    end
  end

  def get_online_users_list(secret)
    return BAD_SECRET_MSG unless valid_secret?(secret)
    online_users = []

    login_node = get_login
    ip = login_node.public_ip
    key = login_node.ssh_key
    raw_list = `ssh -i #{key} -o StrictHostkeyChecking=no root@#{ip} 'ejabberdctl connected-users'`
    raw_list.split("\n").each { |userdata|
      online_users << userdata.split("/")[0]
    }

    return online_users
  end

  def done_uploading(appname, location, secret)
    return BAD_SECRET_MSG unless valid_secret?(secret)

    if File.exists?(location)
      ZKInterface.add_app_entry(appname, my_node.serialize, location)
      result = "success"
    else
      result = "The #{appname} app was not found at #{location}."
    end

    Djinn.log_debug(result)
    return result
  end

  def is_app_running(appname, secret)
    return BAD_SECRET_MSG unless valid_secret?(secret)
    hosters = ZKInterface.get_app_hosters(appname)
    hosters_w_appengine = []
    hosters.each { |node|
      hosters_w_appengine << node if node.is_appengine?
    }

    app_running = !hosters_w_appengine.empty?
    Djinn.log_debug("Is app #{appname} running? #{app_running}")
    return app_running
  end

  def backup_appscale(backup_info, secret)
    return BAD_SECRET_MSG unless valid_secret?(secret)

    Djinn.log_debug("Got a backup request with info: #{backup_info.join(', ')}")

    # right now we only backup the db contents
    # TODO: what else should be backed up?

    uac = UserAppClient.new(@userappserver_private_ip, @@secret)
    all_apps = uac.get_all_apps()

    Djinn.log_debug("all apps are [#{all_apps}]")
    app_list = all_apps.split(":")
    app_list = app_list - [app_list[0]] # first item is a dummy value, ____
    app_list.each { |app|
      # call the bulkuploader here
    }

  end

  def add_role(new_role, secret)
    return BAD_SECRET_MSG unless valid_secret?(secret)
    my_node.add_roles(new_role)
    start_roles = new_role.split(":")
    start_roles.each { |role|
      send("start_#{role}".to_sym)
    }
    return "OK"
  end

  def remove_role(old_role, secret)
    return BAD_SECRET_MSG unless valid_secret?(secret)
    my_node.remove_roles(old_role)
    stop_roles = old_role.split(":")
    stop_roles.each { |role|
      send("stop_#{role}".to_sym)
    }
    return "OK"
  end

  private

  def self.log_debug(msg)
      # TODO: reduce the copypasta here
      puts "[#{Time.now}] #{msg}"
      STDOUT.flush # TODO: examine performance impact of this
      if @lock.nil?
        begin
          Syslog.open("appscale") { |s| s.debug(msg) }
        rescue RuntimeError
        end
      else
        @lock.synchronize {
          begin
            Syslog.open("appscale") { |s| s.debug(msg) }
          rescue RuntimeError
          end
        }
      end
  end

  def self.log_run(command)
    Djinn.log_debug(command)
    Djinn.log_debug(`#{command}`)
  end

  # kinda misnamed - it converts an array of strings
  # to an array of DjinnJobData
  def self.convert_location_array_to_class(nodes, keyname)
    if nodes.class != Array
      abort("Locations should be an array, not a #{nodes.class}")
    end

    Djinn.log_debug("keyname is of class #{keyname.class}")
    Djinn.log_debug("keyname is #{keyname}")

    array_of_nodes = []
    nodes.each { |node|
      converted = DjinnJobData.new(node, keyname)
      array_of_nodes << converted
      Djinn.log_debug("adding data " + converted.to_s)
    }

    return array_of_nodes
  end

  def self.convert_location_class_to_array(djinn_locations)
    if djinn_locations.class != Array
      raise Exception, "Locations should be an array"
    end

    djinn_loc_array = []
    djinn_locations.each { |location|
      djinn_loc_array << location.serialize
      Djinn.log_debug("serializing data " + location.serialize)
    }

    return djinn_loc_array
  end

  def initialize()
    @job = "none"
    @nodes = []
    @creds = {}
    @app_names = []
    @apps_loaded = []
    @kill_sig_received = false
    @my_index = nil
    @@secret = HelperFunctions.get_secret
    @done_loading = false
    @port = 8080
    @haproxy = 10000
    @userappserver_public_ip = "not-up-yet"
    @userappserver_private_ip = "not-up-yet"
    @state = "AppController just started"
    @total_boxes = 0
    @num_appengines = 3
    @restored = false
    @neptune_jobs = {}
    @neptune_nodes = []
    @lock = Monitor.new
    @api_status = {}
  end

  def get_login
    @nodes.each { |node|
      return node if node.is_login?
    }

    abort("No login nodes found.")
  end

  def get_shadow
    @nodes.each { |node|
      return node if node.is_shadow?
    }

    abort("No shadow nodes found.")
  end

  def get_db_master
    @nodes.each { |node|
      return node if node.is_db_master?
    }

    abort("No db master nodes found.")
  end

  def self.get_db_master_ip
    masters_file = File.expand_path("#{APPSCALE_HOME}/.appscale/masters")
    master_ip = HelperFunctions.read_file(masters_file)
    return master_ip
  end

  def self.get_db_slave_ips
    slaves_file = File.expand_path("#{APPSCALE_HOME}/.appscale/slaves")
    slave_ips = File.open(slaves_file).readlines.map { |f| f.chomp! }
    slave_ips = [] if slave_ips == [""]
    return slave_ips
  end

  def self.get_nearest_db_ip(is_mysql=false)
    db_ips = self.get_db_slave_ips
    # Unless this is mysql we include the master ip
    # Update, now mysql also has an API node
    db_ips << self.get_db_master_ip
    db_ips.compact!

    local_ip = HelperFunctions.local_ip
    Djinn.log_debug("db ips are [#{db_ips.join(', ')}]")
    if db_ips.include?(local_ip)
      # If there is a local database then use it
      local_ip
    else
      # Otherwise just select one randomly
      db_ips.sort_by { rand }[0]
    end
  end

  def valid_secret?(secret)
    @@secret = HelperFunctions.get_secret
    if secret != @@secret
      failed_match_msg = "Incoming secret [#{secret}] failed to match " + \
        " known secret [#{@@secret}]"
      Djinn.log_debug(failed_match_msg)
    end
    return secret == @@secret
  end

  def set_uaserver_ips()
    ip_addr = get_uaserver_ip()
    unless is_cloud?
      @userappserver_public_ip = ip_addr
      @userappserver_private_ip = ip_addr
      Djinn.log_debug("\n\nUAServer is at [#{@userappserver_public_ip}]\n\n")
      return
    end

    keyname = @creds["keyname"]
    infrastructure = @creds["infrastructure"]

    if is_hybrid_cloud?
      Djinn.log_debug("getting hybrid ips with creds #{@creds.inspect}")
      public_ips, private_ips = HelperFunctions.get_hybrid_ips(@creds)
    else
      Djinn.log_debug("getting cloud ips for #{infrastructure} with keyname #{keyname}")
      public_ips, private_ips = HelperFunctions.get_cloud_ips(infrastructure, keyname)
    end

    Djinn.log_debug("public ips are #{public_ips.join(', ')}")
    Djinn.log_debug("private ips are #{private_ips.join(', ')}")
    Djinn.log_debug("looking for #{ip_addr}")

    public_ips.each_index { |index|
      if public_ips[index] == ip_addr or private_ips[index] == ip_addr
        @userappserver_public_ip = public_ips[index]
        @userappserver_private_ip = private_ips[index]
        return
      end
    }

    unable_to_convert_msg = "[get uaserver ip] Couldn't find out whether #{ip_addr} was " +
      "a public or private IP address. Public IPs are " +
      "[#{public_ips.join(', ')}], private IPs are [#{private_ips.join(', ')}]"

    abort(unable_to_convert_msg)
  end

  def get_public_ip(private_ip)
    return private_ip unless is_cloud?

    keyname = @creds["keyname"]
    infrastructure = @creds["infrastructure"]

    if is_hybrid_cloud?
      Djinn.log_debug("getting hybrid ips with creds #{@creds.inspect}")
      public_ips, private_ips = HelperFunctions.get_hybrid_ips(@creds)
    else
      Djinn.log_debug("getting cloud ips for #{infrastructure} with keyname #{keyname}")
      public_ips, private_ips = HelperFunctions.get_cloud_ips(infrastructure, keyname)
    end

    Djinn.log_debug("public ips are #{public_ips.join(', ')}")
    Djinn.log_debug("private ips are #{private_ips.join(', ')}")
    Djinn.log_debug("looking for #{private_ip}")

    public_ips.each_index { |index|
      if private_ips[index] == private_ip or public_ips[index] == private_ip
        return public_ips[index]
      end
    }

    unable_to_convert_msg = "[get public ip] Couldn't convert private IP #{private_ip}" +
      " to a public address. Public IPs are [#{public_ips.join(', ')}]," +
      " private IPs are [#{private_ips.join(', ')}]"

    abort(unable_to_convert_msg)
  end

  def get_status(node)
    public_ip = node.private_ip
    ip = node.private_ip
    ssh_key = node.ssh_key
    acc = AppControllerClient.new(ip, @@secret)

    result = acc.get_status(ok_to_fail=true)
    Djinn.log_debug("#{ip}'s returned [#{result}] - class is #{result.class}")

    if !result
      node.failed_heartbeats += 1
      Djinn.log_debug("#{ip} returned false - is it not running?")
      Djinn.log_debug("#{ip} has failed to respond to #{node.failed_heartbeats} heartbeats in a row")
      return
    else
      Djinn.log_debug("#{ip} responded to the heartbeat - it is alive")
      node.failed_heartbeats = 0
    end

    status_file = "/etc/appscale/status-#{ip}.json"
    stats = acc.get_stats()
    json_state = JSON.dump(stats)
    HelperFunctions.write_file(status_file, json_state)

    unless my_node.is_login?
      login_ip = get_login.private_ip
      HelperFunctions.scp_file(status_file, status_file, login_ip, ssh_key)
    end

    # copy remote log over - handy for debugging
    local_log = "#{APPSCALE_HOME}/.appscale/logs/#{ip}.log"
    remote_log = "/tmp/*.log"

    FileUtils.mkdir_p("#{APPSCALE_HOME}/.appscale/logs/")
    Djinn.log_run("scp -o StrictHostkeyChecking=no -i #{ssh_key} #{ip}:#{remote_log} #{local_log}")
  end

  # TODO: add neptune file, which will have this function
  def run_neptune_in_cloud?(neptune_info)
    Djinn.log_debug("activecloud_info = #{neptune_info}")
    return true if is_cloud? && !neptune_info["nodes"].nil?
    return true if !is_cloud? && !neptune_info["nodes"].nil? && !neptune_info["machine"].nil?
    return false
  end

  def heartbeat(secret)
    return BAD_SECRET_MSG unless valid_secret?(secret)
    return true
  end

  def write_database_info()
    table = @creds["table"]
    replication = @creds["replication"]
    keyname = @creds["keyname"]

    tree = { :table => table, :replication => replication,
      :local_accesses => @global_local, :remote_accesses => @global_remote ,
      :keyname => keyname }
    db_info_path = "#{APPSCALE_HOME}/.appscale/database_info.yaml"
    File.open(db_info_path, "w") { |file| YAML.dump(tree, file) }

    num_of_nodes = @nodes.length
    HelperFunctions.write_file("#{APPSCALE_HOME}/.appscale/num_of_nodes", "#{num_of_nodes}\n")

    all_private_ips = []
    @nodes.each { |loc|
      all_private_ips << loc.private_ip
    }
    all_private_ips << "\n"
    HelperFunctions.write_file("#{APPSCALE_HOME}/.appscale/all_ips", all_private_ips.join("\n"))
    # Re-run the filewall script here since we just wrote the all_ips file
    `bash #{APPSCALE_HOME}/firewall.conf` if FIREWALL_IS_ON
  end

  def write_neptune_info()
    info = ""
    @neptune_jobs.each { |k, v|
      info << v.join("\n") + "\n"
    }

    HelperFunctions.write_file(NEPTUNE_INFO, info)
  end

  def load_neptune_info()
    unless File.exists?(NEPTUNE_INFO)
      Djinn.log_debug("no neptune data found - no need to restore")
      return
    end

    Djinn.log_debug("restoring neptune data!")
    jobs_info = (File.open(NEPTUNE_INFO) { |f| f.read }).chomp
    jobs = []
    jobs_info.split("\n").each { |job|
      info = job.split("::")
      name = info[0]
      num_nodes = Integer(info[1])

      begin
        start_time = Time._load(info[2])
        end_time = Time._load(info[3])
      rescue TypeError
        start_time = Time.now
        end_time = Time.now
      end

      this_job = NeptuneJobData.new(name, num_nodes, start_time, end_time)

      if @neptune_jobs[name].nil?
        @neptune_jobs[name] = [this_job]
      else
        @neptune_jobs[name] << this_job
      end
    }
  end

  def backup_appcontroller_state()
    Djinn.log_debug("oi! about to backup!")
    state = {'@@secret' => @@secret }

    instance_variables.each { |k|
      v = instance_variable_get(k)
      next if k == "@lock" # regenerate this later
      if k == "@nodes"
        v = Djinn.convert_location_class_to_array(@nodes)
      end

      state[k] = v
    }

    json_state = JSON.dump(state)
    HelperFunctions.write_file(STATE_FILE, json_state)

    Djinn.log_debug("backed up appcontroller state to #{STATE_FILE}")
  end

  def restore_appcontroller_state()
    Djinn.log_debug("oi! about to restore!")

    unless File.exists?(STATE_FILE)
      Djinn.log_debug("oi! no recovery data found - skipping recovery process")
      return false
    end

    # TODO: catch malformed json exceptions
    data = HelperFunctions.read_file(STATE_FILE)
    json_state = JSON.load(data)

    @@secret = json_state['@@secret']
    keyname = json_state['@creds']['keyname']

    json_state.each { |k, v|
      next if k == "@@secret"
      next if k == "@lock"
      if k == "@nodes"
        v = Djinn.convert_location_array_to_class(v, keyname)
      elsif k == "@neptune_nodes"
        new_v = []

        v.each { |data|
          new_v << NeptuneJobData.from_s(data)
        }

        v = new_v
      end

      instance_variable_set(k, v)
    }

    return true
  end

  def update_api_status()
    repo_url = "http://localhost:8079/health/all"

    begin
      response = Net::HTTP.get_response(URI.parse(repo_url))
      data = JSON.load(response.body)
    rescue Exception => e
      Djinn.log_debug("oi! saw exception #{e.class}")
      data = {}
    end

    Djinn.log_debug("oi! data received is #{data.inspect}")

    majorities = {}

    data.each { |k, v|
      @api_status[k] = [] if @api_status[k].nil?
      @api_status[k] << v
      @api_status[k] = HelperFunctions.shorten_to_n_items(10, @api_status[k])
      majorities[k] = HelperFunctions.find_majority_item(@api_status[k])
    }

    json_state = JSON.dump(majorities)
    HelperFunctions.write_file(HEALTH_FILE, json_state)
  end

  def wait_for_data()
    loop {
      break if got_all_data
      if @kill_sig_received
        msg = "Received kill signal, aborting startup"
        Djinn.log_debug(msg)
        abort(msg)
      else
        Djinn.log_debug("Waiting for data from the load balancer or cmdline tools")
        sleep(5)
      end
    }
  end

  def parse_creds
    got_data_msg = "Got data from another node! DLoc = " + \
      "#{@nodes.join(', ')}, #{@creds.inspect}, AppsToLoad = " + \
      "#{@app_names.join(', ')}"
    Djinn.log_debug(got_data_msg)

    if @creds["appengine"]
      @num_appengines = Integer(@creds["appengine"])
    end

    Djinn.log_debug("keypath is #{@creds['keypath']}, keyname is #{@creds['keyname']}")

    if @creds["keypath"] != ""
      my_key_dir = "/etc/appscale/keys/#{my_node.cloud}"
      my_key_loc = "#{my_key_dir}/#{@creds['keypath']}"
      Djinn.log_debug("Creating directory #{my_key_dir} for my ssh key #{my_key_loc}")
      FileUtils.mkdir_p(my_key_dir)
      `cp /etc/appscale/ssh.key #{my_key_loc}`
    end

    if is_cloud?
      # for euca
      ENV['EC2_ACCESS_KEY'] = @creds["ec2_access_key"]
      ENV['EC2_SECRET_KEY'] = @creds["ec2_secret_key"]
      ENV['EC2_URL'] = @creds["ec2_url"]

      # for ec2
      cloud_keys_dir = File.expand_path("/etc/appscale/keys/cloud1")
      ENV['EC2_PRIVATE_KEY'] = "#{cloud_keys_dir}/mykey.pem"
      ENV['EC2_CERT'] = "#{cloud_keys_dir}/mycert.pem"
    end

    write_database_info
    load_neptune_info
    write_neptune_info

    return
  end

  def got_all_data()
    return false if @nodes == []
    return false if @creds == {}
    return false if @app_names == []
    return true
  end

  def convert_fqdns_to_ips()
    return unless is_cloud?

    if @creds["hostname"] =~ /#{FQDN_REGEX}/
      begin
        @creds["hostname"] = HelperFunctions.convert_fqdn_to_ip(@creds["hostname"])
      rescue Exception => e
        Djinn.log_debug("rescue! failed to convert main hostname #{@creds['hostname']} to public, might want to look into this?")
      end
    end

    @nodes.each { |node|
      #pub = location.public_ip
      #if pub =~ /#{FQDN_REGEX}/
      #  location.public_ip = HelperFunctions.convert_fqdn_to_ip(pub)
      #end

      pri = node.private_ip
      if pri =~ /#{FQDN_REGEX}/
        begin
          node.private_ip = HelperFunctions.convert_fqdn_to_ip(pri)
        rescue Exception => e
          node.private_ip = location.public_ip
        end
      end
    }
  end

  def find_me_in_locations()
    @my_index = nil
    Djinn.log_debug("Searching for node index for #{HelperFunctions.local_ip}")
    @nodes.each_index { |index|
      Djinn.log_debug("Am I #{@nodes[index].private_ip}?")
      if @nodes[index].private_ip == HelperFunctions.local_ip
        Djinn.log_debug("Yes!")
        @my_index = index
        break
      end
      Djinn.log_debug("no...")
    }
    if @my_index.nil?
      Djinn.log_debug("I am lost, could not find my node")
    end
  end

  def valid_format_for_credentials(possible_credentials)
    return false if possible_credentials.class != Hash

    required_fields = ["table", "hostname", "ips"]
    required_fields.each { |field|
      return false if !possible_credentials[field]
    }

    return true
  end

  def sanitize_credentials()
    newcreds = {}
    @creds.each { |key, val|
      newkey = key.gsub(/[^\w\d_@-]/, "") unless key.nil?
      newval = val.gsub(/[^\w\d\.:\/_-]/, "") unless val.nil?
      newcreds[newkey] = newval
    }
    return newcreds
  end

  def change_job(secret)
    return BAD_SECRET_MSG unless valid_secret?(secret)

    my_data = my_node
    jobs_to_run = my_data.jobs

    if @creds['ips']
      @total_boxes = @creds['ips'].length + 1
    elsif @creds['min_images']
      @total_boxes = Integer(@creds['min_images'])
    end

    Djinn.log_debug("pre-loop: #{@nodes.join('\n')}")
    if @nodes.size == 1 and @total_boxes > 1
      spawn_and_setup_appengine
      loop {
        Djinn.log_debug("looping: #{@nodes.join('\n')}")
        @everyone_else_is_done = true
        @nodes.each_index { |index|
          unless index == @my_index
            ip = @nodes[index].private_ip
            acc = AppControllerClient.new(ip, @@secret)
            result = acc.done()
            Djinn.log_debug("#{ip} returned #{result} (#{result.class})")
            @everyone_else_is_done = false unless result
          end
        }
        break if @everyone_else_is_done
        Djinn.log_debug("Waiting on other nodes to come online")
        sleep(5)
      }
    end

    initialize_server
    # start_load_balancer

    memcache_ips = []
    @nodes.each { |node|
      memcache_ips << node.public_ip if node.is_memcache?
    }

    Djinn.log_debug("memcache servers will be at #{memcache_ips.join(', ')}")

    memcache_file = "/etc/appscale/memcache_ips"
    memcache_contents = memcache_ips.join("\n")
    HelperFunctions.write_file(memcache_file, memcache_contents)

    setup_config_files
    set_uaserver_ips
    write_hypersoap

    # ejabberd uses uaserver for authentication
    # so start it after we find out the uaserver's ip

    start_ejabberd if my_node.is_login?

    @done_loading = true

    # start zookeeper
    if my_node.is_zookeeper?
      configure_zookeeper(@nodes, @my_index)
      init = !(@creds.include?("keep_zookeeper_data"))
      start_zookeeper(init)
    end

    ZKInterface.init(my_node, @nodes)

    commands = {
      "load_balancer" => "start_load_balancer",
      "memcache" => "start_memcached",
      "db_master" => "start_db_master",
      "db_slave" => "start_db_slave"
    }

    jobs_to_run.each do |job|
      if commands.include?(job)
        Djinn.log_debug("About to run [#{commands[job]}]")
        send(commands[job].to_sym)
      end
    end

    # create initial tables
    if (my_node.is_db_master? || (defined?(is_priming_needed?) && is_priming_needed?(my_node))) && !restore_from_db?
      table = @creds['table']
      prime_script = "#{APPSCALE_HOME}/AppDB/#{table}/prime_#{table}.py"
      retries = 10
      retval = 0
      while retries > 0
        replication = @creds["replication"]
        Djinn.log_debug(`MASTER_IP="localhost" LOCAL_DB_IP="localhost" python2.6 #{prime_script} #{replication}; echo $? > /tmp/retval`)
        retval = `cat /tmp/retval`.to_i
        break if retval == 0
        Djinn.log_debug("Fail to create initial table. Retry #{retries} times.")
        sleep(5)
        retries -= 1
      end
      if retval != 0
        Djinn.log_debug("Fail to create initial table. Could not startup AppScale.")
        exit(1)
        # TODO: terminate djinn
      end
    end

    # start soap server and pb server
    if has_soap_server?(my_node)
      @state = "Starting up SOAP Server and PBServer"
      start_pbserver
      start_soap_server
      HelperFunctions.sleep_until_port_is_open(HelperFunctions.local_ip, UA_SERVER_PORT)
    end

   start_blobstore_server if my_node.is_appengine?

   # for neptune jobs, start a place where they can save output to
   # also, since repo does health checks on the app engine apis, start it up there too

   repo_ip = get_shadow.public_ip
   repo_ip = my_node.public_ip if my_node.is_appengine?
   Repo.init(repo_ip, @@secret)

   if my_node.is_shadow? or my_node.is_appengine?
     Repo.start(get_login.public_ip, @userappserver_private_ip)
   end

   # appengine is started elsewhere
  end

  def start_blobstore_server
    db_local_ip = @userappserver_private_ip
    BlobServer.start(db_local_ip, PbServer.listen_port)
    BlobServer.is_running(db_local_ip)
  end

  def start_soap_server
    db_master_ip = nil
    @nodes.each { |node|
      db_master_ip = node.private_ip if node.is_db_master?
    }
    abort("db master ip was nil") if db_master_ip.nil?

    db_local_ip = @userappserver_private_ip

    table = @creds['table']

    env_vars = {}

    env_vars['APPSCALE_HOME'] = ENV['APPSCALE_HOME']
    env_vars['MASTER_IP'] = db_master_ip
    env_vars['LOCAL_DB_IP'] = db_local_ip

    if table == "simpledb"
      env_vars['SIMPLEDB_ACCESS_KEY'] = @creds['SIMPLEDB_ACCESS_KEY']
      env_vars['SIMPLEDB_SECRET_KEY'] = @creds['SIMPLEDB_SECRET_KEY']
    end

    start_cmd = ["/usr/bin/python2.6 #{APPSCALE_HOME}/AppDB/soap_server.py",
            "-t #{table} -s #{HelperFunctions.get_secret}"].join(' ')
    stop_cmd = "pkill -9 soap_server"
    port = [4343]

    GodInterface.start(:uaserver, start_cmd, stop_cmd, port, env_vars)
  end

  def start_pbserver
    db_master_ip = nil
    my_ip = my_node.public_ip
    @nodes.each { |node|
      db_master_ip = node.private_ip if node.is_db_master?
    }
    abort("db master ip was nil") if db_master_ip.nil?
    table = @creds['table']
    zoo_connection = get_zk_connection_string(@nodes)
    PbServer.start(db_master_ip, @userappserver_private_ip, my_ip, table, zoo_connection)
    HAProxy.create_pbserver_config(my_ip, PbServer.proxy_port, table)
    Nginx.create_pbserver_config(my_ip, PbServer.proxy_port)
    Nginx.restart
    # TODO check the return value
    PbServer.is_running(my_ip)
  end

  def stop_blob_server
    BlobServer.stop
    Djinn.log_run("pkill -f blobstore_server")
  end

  def stop_soap_server
    GodInterface.stop(:uaserver)
    #Kernel.system "start-stop-daemon --stop --pidfile /var/appscale/appscale-soapserver.pid"
  end

  def stop_pbserver
    PbServer.stop(@creds['table'])
  end

  def is_hybrid_cloud?
    if @creds["infrastructure"].nil?
      false
    else
      @creds["infrastructure"] == "hybrid"
    end
  end

  def is_cloud?
    !@creds["infrastructure"].nil?
  end

  def restore_from_db?
    @creds['restore_from_tar'] || @creds['restore_from_ebs']
  end

  def spawn_and_setup_appengine()
    # should also make sure the tools are on the vm and the envvars are set

    table = @creds['table']

    nodes = HelperFunctions.deserialize_info_from_tools(@creds["ips"])
    appengine_info = spawn_appengine(nodes)

    @state = "Copying over needed files and starting the AppController on the other VMs"

    keyname = @creds["keyname"]
    appengine_info = Djinn.convert_location_array_to_class(appengine_info, keyname)
    @nodes.concat(appengine_info)

    creds = @creds.to_a.flatten
    Djinn.log_debug("Djinn locations: #{@nodes.join(', ')}")
    Djinn.log_debug("DB Credentials: #{@creds.inspect}")
    Djinn.log_debug("Apps to load: #{@app_names.join(', ')}")

    Djinn.log_debug("Appengine info: #{appengine_info}")

    threads = []
    appengine_info.each { |slave|
      threads << Thread.new {
        initialize_node(slave)
      }
    }

    threads.each { |t| t.join }
  end

  def spawn_appengine(nodes)
    appengine_info = []
    if nodes.length > 0
      if is_hybrid_cloud?
        num_of_vms_needed = nodes.length
        @state = "Spawning up hybrid virtual machines"
        appengine_info = HelperFunctions.spawn_hybrid_vms(@creds, nodes)
      elsif is_cloud?
        num_of_vms_needed = nodes.length
        machine = @creds["machine"]
        ENV['EC2_URL'] = @creds["ec2_url"]
        instance_type = @creds["instance_type"]
        keyname = @creds["keyname"]
        infrastructure = @creds["infrastructure"]

        @state = "Spawning up #{num_of_vms_needed} virtual machines"
        roles = nodes.values

        # since there's only one cloud, call it cloud1 to tell us
        # to use the first ssh key (the only key)
        HelperFunctions.set_creds_in_env(@creds, "1")
        appengine_info = HelperFunctions.spawn_vms(num_of_vms_needed, roles,
          machine, instance_type, keyname, infrastructure, "cloud1")
      else
        nodes.each_pair do |ip,roles|
          # for xen the public and private ips are the same
          # and we call it cloud1 since the first key (only key)
          # is the key to use

          info = "#{ip}:#{ip}:#{roles}:i-SGOOBARZ:cloud1"
          appengine_info << info
          Djinn.log_debug("Received appengine info: #{info}")
        end
      end
    end

    return appengine_info
  end

  def initialize_node(node)
    copy_encryption_keys(node)
    validate_image(node)
    restore_db_state_if_needed(node)
    rsync_files(node)
    start_appcontroller(node)
  end

  def validate_image(node)
    ip = node.public_ip
    key = node.ssh_key
    HelperFunctions.ensure_image_is_appscale(ip, key)
    HelperFunctions.ensure_db_is_supported(ip, @creds["table"], key)
  end

  def restore_db_state_if_needed(dest_node)
    return unless dest_node.is_db_master?
    return unless @creds["restore_from_tar"]

    ip = dest_node.private_ip
    ssh_key = dest_node.ssh_key
    Djinn.log_debug("Restoring DB, copying data to DB master at #{ip}")
    db_tar_loc = @creds["restore_from_tar"]
    HelperFunctions.scp_file(db_tar_loc, db_tar_loc, ip, ssh_key)
  end

  def copy_encryption_keys(dest_node)
    ip = dest_node.private_ip
    ssh_key = dest_node.ssh_key

    HelperFunctions.sleep_until_port_is_open(ip, SSH_PORT)
    sleep(3)

    if @creds["infrastructure"] == "ec2" or @creds["infrastructure"] == "hybrid" or @creds["infrastructure"] == "euca"
      options = "-o StrictHostkeyChecking=no -o NumberOfPasswordPrompts=0"
      enable_root_login = "sudo cp /home/ubuntu/.ssh/authorized_keys /root/.ssh/"
      Djinn.log_run("ssh -i #{ssh_key} #{options} 2>&1 ubuntu@#{ip} '#{enable_root_login}'")
    end

    secret_key_loc = "/etc/appscale/secret.key"
    cert_loc = "/etc/appscale/certs/mycert.pem"
    key_loc = "/etc/appscale/certs/mykey.pem"

    HelperFunctions.scp_file(secret_key_loc, secret_key_loc, ip, ssh_key)
    HelperFunctions.scp_file(cert_loc, cert_loc, ip, ssh_key)
    HelperFunctions.scp_file(key_loc, key_loc, ip, ssh_key)

    # TODO: should be able to merge these together
    if is_hybrid_cloud?
      cloud_num = 1
      loop {
        cloud_type = @creds["CLOUD#{cloud_num}_TYPE"]
        break if cloud_type.nil? or cloud_type == ""
        cloud_keys_dir = File.expand_path("/etc/appscale/keys/cloud#{cloud_num}")
        make_dir = "mkdir -p #{cloud_keys_dir}"

        keyname = @creds["keyname"]
        cloud_ssh_key = "#{cloud_keys_dir}/#{keyname}.key"
        cloud_private_key = "#{cloud_keys_dir}/mykey.pem"
        cloud_cert = "#{cloud_keys_dir}/mycert.pem"

        HelperFunctions.run_remote_command(ip, make_dir, ssh_key, NO_OUTPUT)
        HelperFunctions.scp_file(cloud_ssh_key, cloud_ssh_key, ip, ssh_key)
        HelperFunctions.scp_file(cloud_private_key, cloud_private_key, ip, ssh_key)
        HelperFunctions.scp_file(cloud_cert, cloud_cert, ip, ssh_key)
        cloud_num += 1
      }
    else
      cloud_keys_dir = File.expand_path("/etc/appscale/keys/cloud1")
      make_dir = "mkdir -p #{cloud_keys_dir}"

      cloud_private_key = "#{cloud_keys_dir}/mykey.pem"
      cloud_cert = "#{cloud_keys_dir}/mycert.pem"

      HelperFunctions.run_remote_command(ip, make_dir, ssh_key, NO_OUTPUT)
      HelperFunctions.scp_file(ssh_key, ssh_key, ip, ssh_key)
      HelperFunctions.scp_file(cloud_private_key, cloud_private_key, ip, ssh_key)
      HelperFunctions.scp_file(cloud_cert, cloud_cert, ip, ssh_key)
    end
  end

  def rsync_files(dest_node)
    controller = "#{APPSCALE_HOME}/AppController"
    server = "#{APPSCALE_HOME}/AppServer"
    loadbalancer = "#{APPSCALE_HOME}/AppLoadBalancer"
    appdb = "#{APPSCALE_HOME}/AppDB"
    neptune = "#{APPSCALE_HOME}/Neptune"
    loki = "#{APPSCALE_HOME}/Loki"

    ssh_key = dest_node.ssh_key
    ip = dest_node.private_ip

    Djinn.log_run("rsync -e 'ssh -i #{ssh_key}' -arv #{controller}/* root@#{ip}:#{controller}")
    Djinn.log_run("rsync -e 'ssh -i #{ssh_key}' -arv --filter '- *.pyc' #{server}/* root@#{ip}:#{server}")
    Djinn.log_run("rsync -e 'ssh -i #{ssh_key}' -arv #{loadbalancer}/* root@#{ip}:#{loadbalancer}")
    Djinn.log_run("rsync -e 'ssh -i #{ssh_key}' -arv --exclude='logs/*' --exclude='hadoop-*' --exclude='hbase/hbase-*' --exclude='voldemort/voldemort/*' --exclude='cassandra/cassandra/*' #{appdb}/* root@#{ip}:#{appdb}")
    Djinn.log_run("rsync -e 'ssh -i #{ssh_key}' -arv #{neptune}/* root@#{ip}:#{neptune}")
    Djinn.log_run("rsync -e 'ssh -i #{ssh_key}' -arv #{loki}/* root@#{ip}:#{loki}")
  end

  def setup_config_files()
    @state = "Setting up database configuration files"

    master_ip = []
    slave_ips = []

    # load datastore helper
    # TODO: this should be the class or module
    table = @creds['table']
    # require db_file
    begin
      require "#{APPSCALE_HOME}/AppDB/#{table}/#{table}_helper"
    rescue Exception => e
      backtrace = e.backtrace.join("\n")
      bad_datastore_msg = "Unable to find #{table} helper." + \
        " Please verify datastore type: #{e}\n#{backtrace}"
      Djinn.log_debug(bad_datastore_msg)
      abort(bad_datastore_msg)
    end
    FileUtils.mkdir_p("#{APPSCALE_HOME}/AppDB/logs")

    @nodes.each { |node|
      master_ip = node.private_ip if node.jobs.include?("db_master")
      slave_ips << node.private_ip if node.jobs.include?("db_slave")
    }

    Djinn.log_debug("Master is at #{master_ip}, slaves are at #{slave_ips.join(', ')}")

    my_public = my_node.public_ip
    HelperFunctions.write_file("#{APPSCALE_HOME}/.appscale/my_public_ip", "#{my_public}\n")

    my_private = my_node.private_ip
    HelperFunctions.write_file("#{APPSCALE_HOME}/.appscale/my_private_ip", "#{my_private}\n")

    head_node_ip = get_public_ip(@creds['hostname'])
    HelperFunctions.write_file("#{APPSCALE_HOME}/.appscale/head_node_ip", "#{head_node_ip}\n")

    login_ip = get_login.public_ip
    HelperFunctions.write_file("#{APPSCALE_HOME}/.appscale/login_ip", "#{login_ip}\n")

    masters_file = "#{APPSCALE_HOME}/.appscale/masters"
    HelperFunctions.write_file(masters_file, "#{master_ip}\n")

    if @total_boxes == 1
      slave_ips = [ @creds['hostname'] ]
    end

    slave_ips_newlined = slave_ips.join("\n")
    HelperFunctions.write_file("#{APPSCALE_HOME}/.appscale/slaves", "#{slave_ips_newlined}\n")

    # n = @creds["replication"]

    # setup_hadoop_config(template_loc, hadoop_hbase_loc, master_ip, slave_ips, n)

    # Invoke datastore helper function
    setup_db_config_files(master_ip, slave_ips, @creds)

    all_nodes = ""
    @nodes.each_with_index { |node, index|
      all_nodes << "#{node.private_ip} appscale-image#{index}\n"
    }

    etc_hosts = "/etc/hosts"
    my_hostname = "appscale-image#{@my_index}"
    etc_hostname = "/etc/hostname"

    new_etc_hosts = <<HOSTS
127.0.0.1 localhost.localdomain localhost
127.0.1.1 localhost
::1     ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
ff02::3 ip6-allhosts
#{all_nodes}
HOSTS

    File.open(etc_hosts, "w+") { |file| file.write(new_etc_hosts) }
    File.open(etc_hostname, "w+") { |file| file.write(my_hostname) }

    # on ubuntu jaunty, we can change the hostname by running hostname.sh
    # on karmic, this file doesn't exist - run /bin/hostname instead
    # TODO: does the fix for karmic hold for lucid?

    jaunty_hostname_file = "/etc/init.d/hostname.sh"

    if File.exists?(jaunty_hostname_file)
      `/etc/init.d/hostname.sh`
    else
      `/bin/hostname #{my_hostname}`
    end

    # use iptables to lock down outside traffic
    # nodes can talk to each other on any port
    # but only the outside world on certain ports
    #`iptables --flush`
    `bash #{APPSCALE_HOME}/firewall.conf` if FIREWALL_IS_ON
  end

  def write_hypersoap()
    HelperFunctions.write_file("#{APPSCALE_HOME}/.appscale/hypersoap", @userappserver_private_ip)
  end

  def my_node()
    if @my_index.nil?
      find_me_in_locations
    #  Djinn.log_debug("my index is nil - is nodes nil? #{@nodes.nil?}")
    #  return nil
    end
    if @my_index == nil
      Djinn.log_debug("My index is nil - is nodes nil? #{@nodes.nil?}")
      if @nodes.nil?
        Djinn.log_debug("My nodes is null also, timing error? race condition?")
      else
        Djinn.log_debug("Setting it to 0 position, even though it was not found")
        # pray its in the 0 position
        return @nodes[0]
      end
    end

    @nodes[@my_index]
  end

  # Perform any necessary initialization steps before we begin starting up services
  def initialize_server
    my_public_ip = my_node.public_ip
    head_node_ip = get_public_ip(@creds['hostname'])

    HAProxy.initialize_config
    Nginx.initialize_config
    Collectd.initialize_config(my_public_ip, head_node_ip)
    Monitoring.reset
  end

  def start_appcontroller(node)
    ip = node.private_ip
    ssh_key = node.ssh_key

    remote_home = HelperFunctions.get_remote_appscale_home(ip, ssh_key)
    env = {'APPSCALE_HOME' => remote_home}

    start = "ruby #{remote_home}/AppController/djinnServer.rb"
    stop = "ruby #{remote_home}/AppController/terminate.rb"

    # remove any possible appcontroller state that may not have been
    # properly removed in non-cloud runs
    remove_state = "rm -rf /etc/appscale/appcontroller-state.json"
    HelperFunctions.run_remote_command(ip, remove_state, ssh_key, NO_OUTPUT)

    GodInterface.start_god(ip, ssh_key)
    sleep(1)

    begin
      GodInterface.start(:controller, start, stop, DJINN_SERVER_PORT, env, ip, ssh_key)
      HelperFunctions.sleep_until_port_is_open(ip, DJINN_SERVER_PORT, USE_SSL)
    rescue Exception => except
      backtrace = except.backtrace.join("\n")
      remote_start_msg = "[remote_start] Unforeseen exception when " + \
        "talking to #{ip}: #{except}\nBacktrace: #{backtrace}"
      Djinn.log_debug(remote_start_msg)
      retry
    end

    Djinn.log_debug("Sending data to #{ip}")
    acc = AppControllerClient.new(ip, @@secret)

    loc_array = Djinn.convert_location_class_to_array(@nodes)
    credentials = @creds.to_a.flatten

    result = acc.set_parameters(loc_array, credentials, @app_names)
    Djinn.log_debug("#{ip} responded with #{result}")
  end

  def is_running?(name)
    `ps ax | grep #{name} | grep -v grep` != ""
  end

  def start_memcached()
    @state = "Starting up memcached"
    Djinn.log_debug("Starting up memcached")
    start_cmd = "/usr/bin/memcached -d -m 32 -p 11211 -u root"
    stop_cmd = "pkill memcached"
    GodInterface.start(:memcached, start_cmd, stop_cmd, [11211])
  end

  def stop_memcached()
    GodInterface.stop(:memcached)
  end

  def start_ejabberd()
    @state = "Starting up XMPP server"
    my_public = my_node.public_ip
    Ejabberd.stop
    Djinn.log_run("rm -f /var/lib/ejabberd/*")
    Ejabberd.write_auth_script(my_public, @@secret)
    Ejabberd.write_config_file(my_public)
    Ejabberd.start
  end

  def stop_ejabberd()
    Ejabberd.stop
  end

  def start_load_balancer()
    @state = "Starting up Load Balancer"
    Djinn.log_debug("Starting up Load Balancer")

    my_ip = my_node.public_ip
    HAProxy.create_app_load_balancer_config(my_ip, LoadBalancer.proxy_port)
    Nginx.create_app_load_balancer_config(my_ip, LoadBalancer.proxy_port)
    LoadBalancer.start
    Nginx.restart
    Collectd.restart

    head_node_ip = get_public_ip(@creds['hostname'])
    if my_ip == head_node_ip
      # Only start monitoring on the head node
      HAProxy.create_app_monitoring_config(my_ip, Monitoring.proxy_port)
      Nginx.create_app_monitoring_config(my_ip, Monitoring.proxy_port)
      Nginx.restart
      Monitoring.start
    end

    LoadBalancer.server_ports.each do |port|
      HelperFunctions.sleep_until_port_is_open("localhost", port)
      begin
        Net::HTTP.get_response("localhost:#{port}", '/')
      rescue SocketError
      end
    end
  end

  # TODO: this function should use hadoop_helper
  def setup_hadoop_config_org(source_dir, dest_dir, master_ip, slave_ips, n)
    ["source_dir", "dest_dir", "master_ip"].each { |param_name|
      param = eval(param_name)
      abort("#{param_name} wasn't a String. It was a/an #{param.class}") if param.class != String
    }

    source_dir = File.expand_path(source_dir)
    dest_dir = File.expand_path(dest_dir)

    abort("Source dir [#{source_dir}] didn't exist") unless File.directory?(source_dir)
    abort("Dest dir [#{dest_dir}] didn't exist") unless File.directory?(dest_dir)

    files_to_config = `ls #{source_dir}`.split
    files_to_config.each{ |filename|
      full_path_to_read = source_dir + File::Separator + filename
      full_path_to_write = dest_dir + File::Separator + filename
      File.open(full_path_to_read) { |source_file|
        contents = source_file.read
        contents.gsub!(/APPSCALE-MASTER/, master_ip)
        contents.gsub!(/APPSCALE-SLAVES/, slave_ips.join("\n"))
        contents.gsub!(/REPLICATION/, n)

        HelperFunctions.write_file(full_path_to_write, contents)
      }
    }
  end

  # TODO: this function should use hadoop_helper
  def start_hadoop_org()
    i = my_node
    return unless i.is_shadow? # change this later to db_master
    hadoop_home = File.expand_path("#{APPSCALE_HOME}/AppDB/hadoop-0.20.2/")
    Djinn.log_run("#{hadoop_home}/bin/hadoop namenode -format 2>&1")
    Djinn.log_run("#{hadoop_home}/bin/start-dfs.sh 2>&1")
    Djinn.log_run("python #{hadoop_home}/../wait_on_hadoop.py 2>&1")
    Djinn.log_run("#{hadoop_home}/bin/start-mapred.sh 2>&1")
  end

  def stop_load_balancer()
    Djinn.log_debug("Shutting down Load Balancer")
    LoadBalancer.stop
  end

  def start_appengine()
    @state = "Preparing to run AppEngine apps if needed"
    Djinn.log_debug("starting appengine - pbserver is at [#{@userappserver_private_ip}]")

    uac = UserAppClient.new(@userappserver_private_ip, @@secret)

    if @restored == false #and restore_from_db?
      Djinn.log_debug("need to restore")
      all_apps = uac.get_all_apps()
      Djinn.log_debug("all apps are [#{all_apps}]")
      app_list = all_apps.split(":")
      app_list = app_list - [app_list[0]] # first item is a dummy value, ____
      app_list.each { |app|
        app_is_enabled = uac.does_app_exist?(app)
        Djinn.log_debug("is app #{app} enabled? #{app_is_enabled}")
        if app_is_enabled == "true"
          @app_names = @app_names + [app]
        end
      }

      @app_names.uniq!
      Djinn.log_debug("decided to restore these apps: [#{@app_names.join(', ')}]")
      @restored = true
    else
      Djinn.log_debug("don't need to restore")
    end

    apps_to_load = @app_names - @apps_loaded - ["none"]
    apps_to_load.each { |app|
      app_data = uac.get_app_data(app)
      Djinn.log_debug("Get app data for #{app} said [#{app_data}]")

      loop {
        app_version = app_data.scan(/version:(\d+)/).flatten.to_s
        Djinn.log_debug("Waiting for app data to have instance info for app named #{app}: #{app_data}")
        Djinn.log_debug("The app's version is #{app_version}, and its class is #{app_version.class}")

        if app_data[0..4] != "Error"
          app_version = "0" if app_version == ""
          app_version = Integer(app_version)
          break if app_version >= 0
        end

        app_data = uac.get_app_data(app)
        sleep(5)
      }

      my_public = my_node.public_ip
      app_version = app_data.scan(/version:(\d+)/).flatten.to_s
      app_language = app_data.scan(/language:(\w+)/).flatten.to_s

      # TODO: merge these
      shadow = get_shadow
      shadow_ip = shadow.private_ip
      ssh_key = shadow.ssh_key
      app_dir = "/var/apps/#{app}/app"
      app_path = "#{app_dir}/#{app}.tar.gz"
      FileUtils.mkdir_p(app_dir)

      copy_app_to_local(app)
      HelperFunctions.setup_app(app)

      if my_node.is_shadow?
        CronHelper.update_cron(my_public, app_language, app)
        start_xmpp_for_app(app, app_language)
      end

      if my_node.is_appengine?
        app_number = @port - 8080
        start_port = 20000
        static_handlers = HelperFunctions.parse_static_data(app)
        proxy_port = HAProxy.app_listen_port(app_number)
        login_ip = get_login.public_ip
        success = Nginx.write_app_config(app, app_number, my_public, proxy_port, static_handlers, login_ip)
        if not success
          Djinn.log_debug("ERROR: Failure to create valid nginx config file for application #{app}.")
          next
        end
        HAProxy.write_app_config(app, app_number, @num_appengines, my_public)
        Collectd.write_app_config(app)

        # send a warmup request to the app to get it loaded - can shave a
        # number of seconds off the initial request if it's java or go
        # go provides a default warmup route
        # TODO: if the user specifies a warmup route, call it instead of /
        warmup_url = "/"

        @num_appengines.times { |index|
          app_true_port = start_port + app_number * @num_appengines + index
          Djinn.log_debug("Starting #{app_language} app #{app} on #{HelperFunctions.local_ip}:#{app_true_port}")
          xmpp_ip = get_login.public_ip
          pid = HelperFunctions.run_app(app, app_true_port, @userappserver_private_ip, my_public, app_version, app_language, @port, xmpp_ip)
          if pid == -1
            Djinn.log_debug("ERROR: Unable to start application #{app}.")
            next
          end
          pid_file_name = "#{APPSCALE_HOME}/.appscale/#{app}-#{app_true_port}.pid"
          HelperFunctions.write_file(pid_file_name, pid)

          location = "http://#{my_public}:#{app_true_port}#{warmup_url}"
          wget_cmd = "wget --tries=1000 --no-check-certificate #{location} -q -O /dev/null"
          Djinn.log_run(wget_cmd)
        }

        Nginx.reload
        HAProxy.reload
        Collectd.restart

        loop {
          sleep(5)
          add_instance_info = uac.add_instance(app, my_public, @port)
          Djinn.log_debug("Add instance returned #{add_instance_info}")
          break if add_instance_info == "true"
        }

        nginx = @port
        haproxy = @haproxy
        login_ip = get_login.public_ip

        Thread.new {
          haproxy_location = "http://#{my_public}:#{haproxy}#{warmup_url}"
          nginx_location = "http://#{my_public}:#{nginx}#{warmup_url}"

          wget_haproxy = "wget --tries=1000 --no-check-certificate #{haproxy_location} -q -O /dev/null"
          wget_nginx = "wget --tries=1000 --no-check-certificate #{nginx_location} -q -O /dev/null"

          Djinn.log_run(wget_haproxy)
          Djinn.log_run(wget_nginx)
        }

        @port += 1
        @haproxy += 1

        # now doing this at the real end so that the tools will
        # wait for the app to actually be running before returning
        done_uploading(app, app_path, @@secret)
      end

      Monitoring.restart if my_node.is_shadow?

      if @app_names.include?("none")
        @apps_loaded = @apps_loaded - ["none"]
        @app_names = @app_names - ["none"]
      end

      @apps_loaded << app
    }

    Djinn.log_debug("#{apps_to_load.size} apps loaded")
  end

  def stop_appengine()
    Djinn.log_debug("Shutting down AppEngine")

    uac = UserAppClient.new(@userappserver_private_ip, @@secret)
    all_apps = uac.get_all_apps()
    my_public = my_node.public_ip

    Djinn.log_debug("all apps are [#{all_apps}]")
    app_list = all_apps.split(":")
    app_list = app_list - [app_list[0]] # first item is a dummy value, ____
    app_list.each { |app|
      app_is_enabled = uac.does_app_exist?(app)
      Djinn.log_debug("[stop appengine] is app #{app} enabled? #{app_is_enabled}")
      if app_is_enabled == "true"
        app_data = uac.get_app_data(app)
        Djinn.log_debug("[stop appengine] app data for #{app} is [#{app_data}]")
        hosts = app_data.scan(/\nhosts:([\d\.|:]+)\n/).flatten.to_s.split(":")
        ports = app_data.scan(/\nports: ([\d|:]+)\n/).flatten.to_s.split(":")
        # TODO : make sure that len hosts = len ports
        hosts.each_index { |i|
          if hosts[i] == my_public
            Djinn.log_debug("[stop appengine] deleting instance for app #{app} at #{hosts[i]}:#{ports[i]}")
            uac.delete_instance(app, hosts[i], ports[i])
          end
        }
        Djinn.log_debug("finished deleting instances for app #{app}")
        #Djinn.log_run("rm -fv /etc/nginx/#{app}.conf")
        Nginx.reload
      else
        Djinn.log_debug("app #{app} wasnt enabled, skipping it")
      end
    }

    @app_names = []
    @apps_loaded = []
    @restored = false

    Djinn.log_run("pkill -f dev_appserver")
    Djinn.log_run("pkill -f DevAppServerMain")
  end

  # Returns true on success, false otherwise
  def copy_app_to_local(appname)
    app_dir = "/var/apps/#{appname}/app"
    app_path = "#{app_dir}/#{appname}.tar.gz"

    if File.exists?(app_path)
      Djinn.log_debug("I already have a copy of app #{appname} - won't grab it remotely")
      return true
    else
      Djinn.log_debug("I don't have a copy of app #{appname} - will grab it remotely")
    end

    nodes_with_app = []
    loop {
      nodes_with_app = ZKInterface.get_app_hosters(appname)
      break unless nodes_with_app.empty?
      Djinn.log_debug("No nodes currently have a copy of app #{appname}, waiting...")
      sleep(5)
    }

    # Try 3 times on each node known to have this application
    nodes_with_app.each { |node|
      ssh_key = node.ssh_key
      ip = node.public_ip
      tries = 3
      loop {
        Djinn.log_debug("Trying #{ip}:#{app_path} for the application")
        Djinn.log_run("scp -o StrictHostkeyChecking=no -i #{ssh_key} #{ip}:#{app_path} #{app_path}")
        if File.exist?("#{app_path}") == true
          Djinn.log_debug("Got a copy of #{appname} from #{ip}")
          return true
        end
        Djinn.log_debug("ERROR: Unable to get the application from #{ip}:#{app_path}! scp failed.")
        if tries > 0
          Djinn.log_debug("Trying again in 5 seconds")
          tries = tries - 1
          sleep(5)
        else
          Djinn.log_debug("Giving up on node #{ip} for the application")
          break
        end
      }
    }
    Djinn.log_debug("Unable to get the application from any node")
    return false
  end

  def start_xmpp_for_app(app, app_language)
    # create xmpp account for the app
    # for app named baz, this translates to baz@login_ip

    login_ip = get_login.public_ip
    login_uac = UserAppClient.new(login_ip, @@secret)
    xmpp_user = "#{app}@#{login_ip}"
    xmpp_pass = HelperFunctions.encrypt_password(xmpp_user, @@secret)
    login_uac.commit_new_user(xmpp_user, xmpp_pass, "app")

    Djinn.log_debug("Created user [#{xmpp_user}] with password [#{@@secret}] and hashed password [#{xmpp_pass}]")

    if Ejabberd.does_app_need_receive?(app, app_language)
      start_cmd = "python #{APPSCALE_HOME}/AppController/xmpp_receiver.py #{app} #{login_ip} #{@@secret}"
      stop_cmd = "ps ax | grep '#{start_cmd}' | grep -v grep | awk '{print $1}' | xargs -d '\n' kill -9"
      GodInterface.start(app, start_cmd, stop_cmd, 9999)
      Djinn.log_debug("app #{app} does need xmpp receive functionality")
    else
      Djinn.log_debug("app #{app} does not need xmpp receive functionality")
    end
  end
end