text:
/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package Forms.Promotion;

import Classes.Promotion;
import Forms.Login;
import Forms.MainView;
import java.sql.SQLException;
import javax.swing.JOptionPane;
import javax.swing.table.DefaultTableModel;

/**
 *
 * @author jourdanrodrigues
 */
public final class PromotionList extends javax.swing.JFrame {
    String userName;
    int isManager, employeeId;

    /**
     * Creates new form PromotionList
     */
    public PromotionList() {
        initComponents();
    }

    public PromotionList(String userName, int isManager, int employeeId){
        this.userName = userName;
        this.isManager = isManager;
        this.employeeId = employeeId;

        initComponents();

        listPromotions();

        UserNameLabel.setText(userName + ".");
    }

    public void listPromotions(){
        try {
            Promotion.getPromotionList((DefaultTableModel) PromotionsList.getModel());
        } catch (SQLException ex) {
            JOptionPane.showMessageDialog(null, "Ocorreu o seguinte erro:\n" + ex.getMessage());
        }
    }

    /**
     * This method is called from within the constructor to initialize the form.
     * WARNING: Do NOT modify this code. The content of this method is always
     * regenerated by the Form Editor.
     */
    @SuppressWarnings("unchecked")
    // <editor-fold defaultstate="collapsed" desc="Generated Code">//GEN-BEGIN:initComponents
    private void initComponents() {

        LogoutButton = new javax.swing.JButton();
        UserNameLabel = new javax.swing.JLabel();
        PromotionRegisterButton = new javax.swing.JButton();
        BackButton = new javax.swing.JButton();
        jScrollPane1 = new javax.swing.JScrollPane();
        PromotionsList = new javax.swing.JTable();
        TopLabel = new javax.swing.JLabel();

        setDefaultCloseOperation(javax.swing.WindowConstants.EXIT_ON_CLOSE);

        LogoutButton.setFont(new java.awt.Font("Ubuntu", 0, 11)); // NOI18N
        LogoutButton.setText("Logout");
        LogoutButton.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                LogoutButtonActionPerformed(evt);
            }
        });

        UserNameLabel.setFont(new java.awt.Font("Ubuntu", 0, 11)); // NOI18N
        UserNameLabel.setText("Não conseguimos obter seu nome.");

        PromotionRegisterButton.setText("CADASTRAR PROMOÇÃO");
        PromotionRegisterButton.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                PromotionRegisterButtonActionPerformed(evt);
            }
        });

        BackButton.setText("Voltar");
        BackButton.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                BackButtonActionPerformed(evt);
            }
        });

        PromotionsList.setModel(new javax.swing.table.DefaultTableModel(
            new Object [][] {
                {null, null, null, null},
                {null, null, null, null},
                {null, null, null, null},
                {null, null, null, null}
            },
            new String [] {
                "ID", "DESC. (%)", "DATA FINAL", "VEÍCULO"
            }
        ) {
            Class[] types = new Class [] {
                java.lang.String.class, java.lang.String.class, java.lang.String.class, java.lang.String.class
            };
            boolean[] canEdit = new boolean [] {
                false, false, false, false
            };

            public Class getColumnClass(int columnIndex) {
                return types [columnIndex];
            }

            public boolean isCellEditable(int rowIndex, int columnIndex) {
                return canEdit [columnIndex];
            }
        });
        jScrollPane1.setViewportView(PromotionsList);
        if (PromotionsList.getColumnModel().getColumnCount() > 0) {
            PromotionsList.getColumnModel().getColumn(0).setPreferredWidth(15);
        }

        TopLabel.setFont(new java.awt.Font("Tahoma", 1, 18)); // NOI18N
        TopLabel.setHorizontalAlignment(javax.swing.SwingConstants.CENTER);
        TopLabel.setText("Lista de Promoções");

        javax.swing.GroupLayout layout = new javax.swing.GroupLayout(getContentPane());
        getContentPane().setLayout(layout);
        layout.setHorizontalGroup(
            layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
            .addGroup(layout.createSequentialGroup()
                .addContainerGap(60, Short.MAX_VALUE)
                .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING, false)
                    .addComponent(PromotionRegisterButton, javax.swing.GroupLayout.Alignment.LEADING, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
                    .addComponent(TopLabel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
                    .addComponent(jScrollPane1, javax.swing.GroupLayout.DEFAULT_SIZE, 420, Short.MAX_VALUE))
                .addGap(0, 60, Short.MAX_VALUE))
            .addGroup(layout.createSequentialGroup()
                .addComponent(BackButton)
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
                .addComponent(UserNameLabel)
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                .addComponent(LogoutButton))
        );
        layout.setVerticalGroup(
            layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
            .addGroup(layout.createSequentialGroup()
                .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
                    .addComponent(LogoutButton)
                    .addComponent(UserNameLabel)
                    .addComponent(BackButton))
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, 24, Short.MAX_VALUE)
                .addComponent(TopLabel)
                .addGap(18, 18, 18)
                .addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, 99, javax.swing.GroupLayout.PREFERRED_SIZE)
                .addGap(18, 18, 18)
                .addComponent(PromotionRegisterButton, javax.swing.GroupLayout.PREFERRED_SIZE, 50, javax.swing.GroupLayout.PREFERRED_SIZE)
                .addContainerGap(24, Short.MAX_VALUE))
        );

        pack();
    }// </editor-fold>//GEN-END:initComponents

    private void LogoutButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_LogoutButtonActionPerformed
        new Login().setVisible(true);
        dispose();
    }//GEN-LAST:event_LogoutButtonActionPerformed

    private void PromotionRegisterButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_PromotionRegisterButtonActionPerformed
        new PromotionRegister(this.userName, this.isManager, this.employeeId, "fromPromotionList").setVisible(true);
        dispose();
    }//GEN-LAST:event_PromotionRegisterButtonActionPerformed

    private void BackButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_BackButtonActionPerformed
        new MainView(this.userName, this.isManager, this.employeeId).setVisible(true);
        dispose();
    }//GEN-LAST:event_BackButtonActionPerformed

    /**
     * @param args the command line arguments
     */
    public static void main(String args[]) {
        /* Set the Nimbus look and feel */
        //<editor-fold defaultstate="collapsed" desc=" Look and feel setting code (optional) ">
        /* If Nimbus (introduced in Java SE 6) is not available, stay with the default look and feel.
         * For details see http://download.oracle.com/javase/tutorial/uiswing/lookandfeel/plaf.html
         */
        try {
            for (javax.swing.UIManager.LookAndFeelInfo info : javax.swing.UIManager.getInstalledLookAndFeels()) {
                if ("Nimbus".equals(info.getName())) {
                    javax.swing.UIManager.setLookAndFeel(info.getClassName());
                    break;
                }
            }
        } catch (ClassNotFoundException ex) {
            java.util.logging.Logger.getLogger(PromotionList.class.getName()).log(java.util.logging.Level.SEVERE, null, ex);
        } catch (InstantiationException ex) {
            java.util.logging.Logger.getLogger(PromotionList.class.getName()).log(java.util.logging.Level.SEVERE, null, ex);
        } catch (IllegalAccessException ex) {
            java.util.logging.Logger.getLogger(PromotionList.class.getName()).log(java.util.logging.Level.SEVERE, null, ex);
        } catch (javax.swing.UnsupportedLookAndFeelException ex) {
            java.util.logging.Logger.getLogger(PromotionList.class.getName()).log(java.util.logging.Level.SEVERE, null, ex);
        }
        //</editor-fold>

        /* Create and display the form */
        java.awt.EventQueue.invokeLater(new Runnable() {
            public void run() {
                new PromotionList().setVisible(true);
            }
        });
    }

    // Variables declaration - do not modify//GEN-BEGIN:variables
    private javax.swing.JButton BackButton;
    private javax.swing.JButton LogoutButton;
    private javax.swing.JButton PromotionRegisterButton;
    private javax.swing.JTable PromotionsList;
    private javax.swing.JLabel TopLabel;
    private javax.swing.JLabel UserNameLabel;
    private javax.swing.JScrollPane jScrollPane1;
    // End of variables declaration//GEN-END:variables
}


text:
title: "Class 10: Local Models"
author: "Taylor Arnold"
output: html_notebook
---

{% highlight r %}
library(dplyr)
library(ggplot2)
library(readr)
library(viridis)
library(methods)
{% endhighlight %}

## Income data

For the first half of today, we are going to look at the dataset
you used on your last lab.

{% highlight r %}
acs <- read_csv("https://statsmaths.github.io/ml_data/tract_median_income.csv")
{% endhighlight %}

Looking at the map, we see that income is highly local and highest
near large cities (such as NYC, Chicago, LA, and Houston).

{% highlight r %}
library(smodels)
temp <- filter(acs, train_id == "train", !(state %in% c("AK", "HI")))
qplot(lon, lat, data = temp, color = bin(median_income, 6),
      size = I(0.5), alpha = I(0.5)) +
  theme_minimal() +
  coord_map()
{% endhighlight %}

While we have seen ways of building models that account for
linearites and interactions (GAMs and basis expansion), today's
focus will be on using inherantly local models to learn the
response variable here.

### Tuning k-nearest neighbors

The knn model, as we saw last time, requires us to use
a model matrix. We will construct one here using only
longitude and latitude:

{% highlight r %}
X <- as.matrix(select(acs, lon, lat))
y <- acs$median_income

X_train <- X[acs$train_id == "train",]
X_valid <- X[acs$train_id == "valid",]
y_train <- y[acs$train_id == "train"]
y_valid <- y[acs$train_id == "valid"]
{% endhighlight %}

While we usually do not need a specific validation X matrix
and response vector y, it will be useful here today. We will
fit the k-nearest neighbors algorithm for 25 values of the
hyperparameter k, specifically, the integers 1 through 25.
I will save the RMSE of the validation set for each value of
k.

{% highlight r %}
library(FNN)
rmse <- rep(NA, 25)
for (k in seq_along(rmse)) {
  y_valid_pred <- knn.reg(train = X_train, y = y_train,
                          test = X_valid, k = k)$pred
  rmse[k] <- sqrt( mean((y_valid_pred - y_valid)^2) )
}
{% endhighlight %}

With these values, we can plot the RMSE as a function of
the parameter k:

{% highlight r %}
qplot(seq_along(rmse), rmse) +
  geom_line() +
  theme_minimal()
{% endhighlight %}

The optimal value seems to be at k equal to 5, with a drastic
degradation for k equal to 1 or 2.

One thing that may be counterintuitive about k-nearest neighbors
is the relationship between k and the model complexity. Here is
a plot of the fitted values for k equal to 3000 over the Northeastern
US:

{% highlight r %}
id <- (X_valid[,1] > -80) & (X_valid[,2] > 35)
pred_id <- knn.reg(train = X_train, y = y_train,
                   test = X_valid[id,], k = 3000)$pred

df <- data_frame(lon = X_valid[id,1], lat = X_valid[id,2],
                 pred = pred_id)

library(ggmap)
qmplot(lon, lat, data = df, color = bin(pred, 4)) +
  viridis::scale_color_viridis(discrete = TRUE) +
  ggtitle("k = 3000")
{% endhighlight %}

{% highlight text %}
## Warning: `panel.margin` is deprecated. Please use `panel.spacing` property
## instead
{% endhighlight %}

There are large regions that all have very similar predicted
values. In contrast, here are the predicted value from k equal to
3.

{% highlight r %}
id <- (X_valid[,1] > -80) & (X_valid[,2] > 35)
pred_id <- knn.reg(train = X_train, y = y_train,
                   test = X_valid[id,], k = 3)$pred

df <- data_frame(lon = X_valid[id,1], lat = X_valid[id,2],
                 pred = pred_id)

qmplot(lon, lat, data = df, color = bin(pred, 4)) +
  viridis::scale_color_viridis(discrete = TRUE) +
  ggtitle("k = 5")
{% endhighlight %}

{% highlight text %}
## Warning: `panel.margin` is deprecated. Please use `panel.spacing` property
## instead
{% endhighlight %}

Here, the values change rapidly and there are many nearby
points with drastically different values. So, a high value
of k is a less complex model and a low value is a more
complex model. If this seems confusing, think about k being
the size of the entire training set (every point will then
be equal to a constant: the training set mean). Also perhaps
confusing is that models with a larger k take longer to fit;
this should make sense as there are more points to average,
but goes against our general idea that more complex models
are more difficult to fit.

## Clustering with K-means

An alternative to k-nearest neighbors is to split the dataset
into fixed neighborhoods of nearby points and use these as
indicator variables in a model. To do this, we need to cluster
the input data. There are many algorithms available for clustering;
one of the most widely known and implemented is k-means clustering.
This is achieved by way of the `kmeans` function and requires only
that we set the number of centers. Here is a plot of kmeans applied
to the `acs` data:

{% highlight r %}
set.seed(1)
acs$cluster_20 <- kmeans(X, centers = 20)$cluster
qmplot(lon, lat, data = acs, color = factor(cluster_20))
{% endhighlight %}

{% highlight text %}
## Warning: `panel.margin` is deprecated. Please use `panel.spacing` property
## instead
{% endhighlight %}

Notice that I have set the random seed as the output of the algorithm
is stochastic. Here, longitude and latitude are on similar scales, but
generally you should scale the data matrix X prior to clustering.

Let's now build 20, 100, and 200 member clusters in the dataset:

{% highlight r %}
set.seed(1)
acs$cluster_20 <- kmeans(X, centers = 20)$cluster
acs$cluster_100 <- kmeans(X, centers = 100)$cluster
acs$cluster_200 <- kmeans(X, centers = 200)$cluster
{% endhighlight %}

We can use these in a `glmnet` model to determine which clusters
should have their own offsets. I am going to use the function
`model.Matrix` (note the captial M) from the **MatrixModels**
package for efficency purposes. We will discuss the details of
the function once we get to textual data:

{% highlight r %}
library(MatrixModels)
X <- model.Matrix(~ factor(cluster_20) + factor(cluster_100) +
                    factor(cluster_200) + -1 , data = acs, sparse = TRUE)
y <- acs$median_income
X_train <- X[acs$train_id == "train",]
y_train <- y[acs$train_id == "train"]
{% endhighlight %}

And we will fit an elastic net model to the data:

{% highlight r %}
library(glmnet)
{% endhighlight %}

{% highlight text %}
## Warning: package 'glmnet' was built under R version 3.4.2
{% endhighlight %}

{% highlight r %}
model <- cv.glmnet(X_train, y_train, alpha = 0.2)
acs$median_income_pred <- predict(model, newx = X)
{% endhighlight %}

Notice that the output resembles k-nearest neighbors:

{% highlight r %}
acs_ne <- filter(acs, lon > -80, lat > 35, lon < -30)
qmplot(lon, lat, data = acs_ne, color = bin(median_income_pred, 4)) +
  viridis::scale_color_viridis(discrete = TRUE) +
  ggtitle("glmnet")
{% endhighlight %}

{% highlight text %}
## Warning: `panel.margin` is deprecated. Please use `panel.spacing` property
## instead
{% endhighlight %}

There are important differences with the k-nearest neighbors
method, however. The regions to group together have been determined
adaptively. Notice that upstate New York and Maine have moslty be
pushed into the same values. The Metro NYC area, in constrast, has
many more regions.

## Decision Trees

We will now look at a new dataset of housing prices in Ames,
Iowa. Unlike our other housing datasets though, the samples
here are individual houses.

{% highlight r %}
ames <- read_csv("https://statsmaths.github.com/ml_data/ames.csv")
{% endhighlight %}

One way to build a predictive algorithm is to describe a decision
tree. The best way to understand what this is is to see an example.
Here, we use the `tree` library (pre-installed in R) to predict
sales price:

{% highlight r %}
library(tree)
model <- tree(saleprice ~ neighborhood + overall_qual + overall_cond + year_built,
                      data = ames, subset = train_id == "train",
              control = tree.control(nobs = nrow(ames), mindev = 0.005))
{% endhighlight %}

{% highlight text %}
## Warning in tree(saleprice ~ neighborhood + overall_qual + overall_cond + :
## NAs introduced by coercion
{% endhighlight %}

This relatively old library needs a special plotting notation (so
don't worry if this looks strange)

{% highlight r %}
par(mar = c(0,0,0,0))
plot(model, type = "uniform")
text(model, cex = 0.7, col = "purple")
{% endhighlight %}

In order to do prediction, we start at the top of the tree and
look at each logical statement. If True, move to the left and
if False move to right. At the bottom (the *terminal nodes* or *leaves*),
there are predicted values. The tree was built by greedily picking
variables to spit the dataset up by until some stopping criterion
was reached; this is usually a fixed depth, a fixed proportion of
the data, or a fixed decrease in the RMSE of accuracy rate.

## Random Forests

Decision trees give very noisy predictions due to their use of
greedy logic and because points on the boundary of a decision
cut-off are forced into a fixed bucket. By noisy, I mean that a
slightly different training set would yield significantly
different predictions for at least some of the test points.
This may seem like a design flaw, but we can easily turn it into
a design feature!

The idea of a random forest is to add some randomness into the
decision tree algorithm, fit a large number of trees using this
random variation, and produce predictions by averaging together
the predictions from all these individual trees (its a *forest*
because there are a lot of trees; get it?). The random logic
applies only the building of the trees; once created, each tree
is exactly the same as in the case above. The randomness comes
from two sources:

- for each tree, select only a subset of the training data to
train with
- for each split, select only a subset of the available variables
to split on

The exact values for these two random features can be set as
hyperparameters. We can fit random forests using the `randomForest`
function from the package with the same name as follows:

{% highlight r %}
library(randomForest)
set.seed(1)
model <- randomForest(saleprice ~ overall_qual + year_built,
                      data = ames, subset = train_id == "train",
                      ntree = 20, maxnodes = 3, mtry = 1)
{% endhighlight %}

Here I selected 20 randomly generated trees, each having at most
3 terminal nodes and only allowing one variable to be used at
each split. These are very low settings, used only for illustrating
the algorithm here. We can get predictions from each individual
tree by setting `predict.all` to `TRUE`:

{% highlight r %}
obj <- predict(model, newdata = ames, predict.all = TRUE)$individual
{% endhighlight %}

Here is the prediction for just the third tree:

{% highlight r %}
ames$price_pred <- obj[,4]
qplot(overall_qual, year_built, data = ames, color = price_pred) +
  viridis::scale_color_viridis()
{% endhighlight %}

Can you figure out roughly what the tree looks like? It first
splits on overal quality being less than 7.5, and then splits
the lower quality houses by year built around 1982. The individual
prediction is not very smooth or complex.

Taking all of the twenty trees together, the average model
looks quite a bit different:

{% highlight r %}
ames$price_pred <- predict(model, newdata = ames)
qplot(overall_qual, year_built, data = ames, color = price_pred) +
  viridis::scale_color_viridis()
{% endhighlight %}

Helpfully, the **randomForest** also provides the function
`importance` that measures how important each variable is
to the model.

{% highlight r %}
importance(model)
{% endhighlight %}

{% highlight text %}
##              IncNodePurity
## overall_qual  3.119819e+12
## year_built    1.886067e+12
{% endhighlight %}

This is a measurement of how often the variable was used
in the model and how much it decreased the RMSE each time it
was used to split the dataset.

## Gradient Boosted Trees

Gradient boosted trees offer a slightly different approach
to random forests for making use of the noisy nature of
decision trees. Like random forests, they construct a
number of trees, each using only a subset of the training
data. They do not restrict the variables available for
each node to split on. Most importantly, gradient boosted
trees are fit in a sequence, with each tree trying to predict
the residuals left over by the other trees.

More exactly, if the fitted values from the t-th tree
are given by:

$$ \widehat{Y_i^t} $$
Then we train the k-th tree on the values Z given by:

$$ Z_i = Y_i - \eta \cdot \sum_{t = 1}^{k - 1} \widehat{Y_i^t} $$

The parameter eta is the learning rate. If set to one, this
is exactly fitting on the residuals of the prior trees.
Setting to less than one stop the trees from overfitting from
the first few trees. Here, we prepare a larger set of variables
from the `ames` dataset:

{% highlight r %}
X <- model.matrix(~ . -1 , data = ames[,-c(1:3)])
y <- ames$saleprice

y_train <- y[ames$train_id == "train"]
y_valid <- y[ames$train_id == "valid"]
X_train <- X[ames$train_id == "train",]
X_valid <- X[ames$train_id == "valid",]
{% endhighlight %}

We will use the **xgboost** package to fit gradient
boosted trees. I will set the eta parameter to 0.02.

{% highlight r %}
library(xgboost)
model <- xgboost(data = X_train, label = y_train,
                 max_depth = 2, eta = 0.01, nthread = 2,
                 nrounds = 10, objective = "reg:linear",
                 verbose = 1)
{% endhighlight %}

{% highlight text %}
## [1]	train-rmse:200613.875000
## [2]	train-rmse:198727.640625
## [3]	train-rmse:196860.953125
## [4]	train-rmse:195013.859375
## [5]	train-rmse:193185.953125
## [6]	train-rmse:191377.140625
## [7]	train-rmse:189587.203125
## [8]	train-rmse:187815.984375
## [9]	train-rmse:186063.203125
## [10]	train-rmse:184328.828125
{% endhighlight %}

And we can do prediction on the dataset:

{% highlight r %}
y_valid_pred <- predict(model, newdata = X_valid)
sqrt(mean((y_valid - y_valid_pred)^2))
{% endhighlight %}

{% highlight text %}
## [1] 184162.9
{% endhighlight %}

Alternatively, we can use the function `xgb.DMatrix` to
combine the data matrix and labels:

{% highlight r %}
data_train <- xgb.DMatrix(data = X_train, label = y_train)
data_valid <- xgb.DMatrix(data = X_valid, label = y_valid)
{% endhighlight %}

And use a more advanced calling method for **xgboost**:

{% highlight r %}
watchlist <- list(train=data_train, valid=data_valid)

model <- xgb.train(data = data_train,
                 max_depth = 3, eta = 1, nthread = 2,
                 nrounds = 100, objective = "reg:linear",
                 watchlist = watchlist)
{% endhighlight %}

{% highlight text %}
## [1]	train-rmse:39387.718750	valid-rmse:43630.218750
## [2]	train-rmse:33187.484375	valid-rmse:39454.351562
## [3]	train-rmse:30578.279297	valid-rmse:38123.277344
## [4]	train-rmse:28023.640625	valid-rmse:35523.105469
## [5]	train-rmse:26166.941406	valid-rmse:34078.468750
## [6]	train-rmse:24995.691406	valid-rmse:33140.027344
## [7]	train-rmse:24229.658203	valid-rmse:32568.550781
## [8]	train-rmse:22881.064453	valid-rmse:31502.035156
## [9]	train-rmse:21674.976562	valid-rmse:31560.398438
## [10]	train-rmse:21044.863281	valid-rmse:30859.064453
## [11]	train-rmse:20491.050781	valid-rmse:30980.572266
## [12]	train-rmse:20045.650391	valid-rmse:31187.617188
## [13]	train-rmse:19656.458984	valid-rmse:30825.869141
## [14]	train-rmse:19116.199219	valid-rmse:30785.873047
## [15]	train-rmse:18829.880859	valid-rmse:30708.177734
## [16]	train-rmse:18453.912109	valid-rmse:31036.326172
## [17]	train-rmse:17860.994141	valid-rmse:30655.796875
## [18]	train-rmse:17397.365234	valid-rmse:30416.314453
## [19]	train-rmse:17067.007812	valid-rmse:30946.679688
## [20]	train-rmse:16704.880859	valid-rmse:31027.126953
## [21]	train-rmse:16278.335938	valid-rmse:31071.878906
## [22]	train-rmse:16038.116211	valid-rmse:31153.265625
## [23]	train-rmse:15765.764648	valid-rmse:30998.392578
## [24]	train-rmse:15456.485352	valid-rmse:31223.222656
## [25]	train-rmse:15273.040039	valid-rmse:31096.722656
## [26]	train-rmse:14958.646484	valid-rmse:31197.484375
## [27]	train-rmse:14811.100586	valid-rmse:31233.912109
## [28]	train-rmse:14576.267578	valid-rmse:31070.244141
## [29]	train-rmse:14414.208984	valid-rmse:31219.134766
## [30]	train-rmse:14274.974609	valid-rmse:31211.501953
## [31]	train-rmse:13968.666992	valid-rmse:31381.671875
## [32]	train-rmse:13699.797852	valid-rmse:31438.687500
## [33]	train-rmse:13550.646484	valid-rmse:31520.996094
## [34]	train-rmse:13337.493164	valid-rmse:31807.056641
## [35]	train-rmse:13176.716797	valid-rmse:31770.550781
## [36]	train-rmse:13036.213867	valid-rmse:31813.195312
## [37]	train-rmse:12772.610352	valid-rmse:31904.640625
## [38]	train-rmse:12569.961914	valid-rmse:31994.439453
## [39]	train-rmse:12410.292969	valid-rmse:31991.849609
## [40]	train-rmse:12311.220703	valid-rmse:31941.285156
## [41]	train-rmse:12095.907227	valid-rmse:31696.376953
## [42]	train-rmse:11897.456055	valid-rmse:31808.634766
## [43]	train-rmse:11658.248047	valid-rmse:31799.466797
## [44]	train-rmse:11449.622070	valid-rmse:31775.031250
## [45]	train-rmse:11338.799805	valid-rmse:31746.113281
## [46]	train-rmse:11242.211914	valid-rmse:31920.767578
## [47]	train-rmse:11067.416016	valid-rmse:32018.767578
## [48]	train-rmse:10891.774414	valid-rmse:32050.312500
## [49]	train-rmse:10721.001953	valid-rmse:32075.419922
## [50]	train-rmse:10560.923828	valid-rmse:32084.414062
## [51]	train-rmse:10402.637695	valid-rmse:31847.986328
## [52]	train-rmse:10356.886719	valid-rmse:31851.632812
## [53]	train-rmse:10234.088867	valid-rmse:31902.597656
## [54]	train-rmse:10094.040039	valid-rmse:31966.564453
## [55]	train-rmse:9937.343750	valid-rmse:31960.406250
## [56]	train-rmse:9794.728516	valid-rmse:32056.812500
## [57]	train-rmse:9684.227539	valid-rmse:32176.064453
## [58]	train-rmse:9586.635742	valid-rmse:32149.871094
## [59]	train-rmse:9476.002930	valid-rmse:32195.085938
## [60]	train-rmse:9271.000000	valid-rmse:32174.087891
## [61]	train-rmse:9147.355469	valid-rmse:32320.279297
## [62]	train-rmse:9008.699219	valid-rmse:32299.322266
## [63]	train-rmse:8864.962891	valid-rmse:32288.832031
## [64]	train-rmse:8830.485352	valid-rmse:32258.412109
## [65]	train-rmse:8786.670898	valid-rmse:32259.580078
## [66]	train-rmse:8683.625977	valid-rmse:32298.400391
## [67]	train-rmse:8613.983398	valid-rmse:32299.162109
## [68]	train-rmse:8570.147461	valid-rmse:32173.515625
## [69]	train-rmse:8483.368164	valid-rmse:32090.998047
## [70]	train-rmse:8394.336914	valid-rmse:32118.982422
## [71]	train-rmse:8276.410156	valid-rmse:32177.640625
## [72]	train-rmse:8151.367676	valid-rmse:32210.968750
## [73]	train-rmse:8084.541016	valid-rmse:32225.222656
## [74]	train-rmse:8038.354492	valid-rmse:32280.789062
## [75]	train-rmse:7937.657715	valid-rmse:32341.556641
## [76]	train-rmse:7865.796875	valid-rmse:32508.480469
## [77]	train-rmse:7805.263672	valid-rmse:32546.832031
## [78]	train-rmse:7761.944336	valid-rmse:32599.972656
## [79]	train-rmse:7705.764648	valid-rmse:32585.083984
## [80]	train-rmse:7624.173828	valid-rmse:32597.328125
## [81]	train-rmse:7568.135254	valid-rmse:32645.367188
## [82]	train-rmse:7505.100098	valid-rmse:32713.955078
## [83]	train-rmse:7419.775879	valid-rmse:32800.683594
## [84]	train-rmse:7322.812012	valid-rmse:32858.601562
## [85]	train-rmse:7231.112793	valid-rmse:32824.699219
## [86]	train-rmse:7183.702637	valid-rmse:32830.074219
## [87]	train-rmse:7103.186523	valid-rmse:32778.093750
## [88]	train-rmse:7016.894531	valid-rmse:32807.566406
## [89]	train-rmse:6915.380371	valid-rmse:32755.068359
## [90]	train-rmse:6838.368652	valid-rmse:32792.367188
## [91]	train-rmse:6778.382324	valid-rmse:32758.763672
## [92]	train-rmse:6700.780762	valid-rmse:32731.150391
## [93]	train-rmse:6604.482910	valid-rmse:32684.847656
## [94]	train-rmse:6545.086426	valid-rmse:32620.916016
## [95]	train-rmse:6522.374512	valid-rmse:32671.328125
## [96]	train-rmse:6433.093750	valid-rmse:32659.060547
## [97]	train-rmse:6382.375488	valid-rmse:32710.066406
## [98]	train-rmse:6335.753906	valid-rmse:32692.509766
## [99]	train-rmse:6244.332031	valid-rmse:32710.580078
## [100]	train-rmse:6163.050781	valid-rmse:32633.583984
{% endhighlight %}

The algorithm is the same, but there are more options available
with `xgb.train`. As with random forests, there is a way of
looking at variable importance. I don't like the default output
view, so here is some code to make it look nicer:

{% highlight r %}
importance_matrix <- xgb.importance(model = model)
importance_matrix[,1] <- colnames(X)[as.numeric(importance_matrix[[1]]) + 1]
importance_matrix
{% endhighlight %}

{% highlight text %}
##             Feature         Gain        Cover   Frequency
##   1:   overall_qual 5.830815e-01 3.878433e-02 0.024024024
##   2:     price_pred 1.368774e-01 2.544728e-02 0.018018018
##   3:    gr_liv_area 6.700802e-02 7.306831e-02 0.063063063
##   4:   `1st_flr_sf` 4.027978e-02 2.863572e-02 0.033033033
##   5:   bsmtfin_sf_1 3.837214e-02 6.649631e-02 0.051051051
##  ---
## 134: ms_subclass080 1.482758e-05 3.333742e-03 0.001501502
## 135: ms_subclass085 8.917817e-06 1.037902e-05 0.001501502
## 136: ms_subclass120 5.179132e-06 6.227413e-06 0.001501502
## 137: ms_subclass050 3.606932e-06 1.037902e-05 0.001501502
## 138: ms_subclass190 2.644195e-06 8.303217e-06 0.001501502
{% endhighlight %}

## Thoughts on local models

We've covered a lot today. Here are some take aways:

- k-nearest neighbors are great for smoothing predictions
or creating meta-features
- k is inversely related to the complexity of the model
- using clusters simulated KNN but allows for covariates
and adaptation to the data
- don't use the `tree` function for actual predictions; I
only used it to illustrate decision trees
- random forests are easy to use and difficult to overfit
with
- gradient boosted trees are incredibly powerful (often
the give the most predictive models in large ML competitions)
- you need to tune eta and number of trees in GBT to get
a good model

The last point should be the object of study for the next
lab. We will discuss the details more next week when going
over the lab.

redpajama_set_name: RedPajamaGithub


text:
// Copyright 2012 Cloudera Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.cloudera.impala.catalog;

import com.cloudera.impala.thrift.TColumnType;
import com.cloudera.impala.thrift.TStructField;
import com.cloudera.impala.thrift.TTypeNode;

/**
 * TODO: Support comments for struct fields. The Metastore does not properly store
 * comments of struct fields. We set comment_ to null to avoid compatibility issues.
 */
public class StructField {
  protected final String name_;
  protected Type type_;
  protected String comment_;

  public StructField(String name, Type type, String comment) {
    name_ = name;
    type_ = type;
    comment_ = comment;
  }

  public String getComment() { return comment_; }
  public String getName() { return name_; }
  public Type getType() { return type_; }

  public String toSql() {
    StringBuilder sb = new StringBuilder(name_);
    if (type_ != null) sb.append(":" + type_.toSql());
    if (comment_ != null) sb.append(String.format(" COMMENT '%s'", comment_));
    return sb.toString();
  }

  public void toThrift(TColumnType container, TTypeNode node) {
    TStructField field = new TStructField();
    field.setName(name_);
    if (comment_ != null) field.setComment(comment_);
    node.struct_fields.add(field);
    type_.toThrift(container);
  }
}


text:
using System;
using System.Diagnostics;
using static System.StringComparison;

namespace Generaid
{
    [DebuggerDisplay("{FullName}")]
    internal sealed class CmpNode
    {
        private static readonly StringComparer
            IgnoreCase = StringComparer.OrdinalIgnoreCase;

        public string DependentUpon { get; }
        public string FullName { get; }

        public CmpNode(string fullName, string dependentUpon = null)
        {
            FullName = fullName;
            DependentUpon = dependentUpon ?? "";
        }

        private bool Equals(CmpNode other)
        {
            return string.Equals(DependentUpon, other.DependentUpon, OrdinalIgnoreCase)
                && string.Equals(FullName, other.FullName, OrdinalIgnoreCase);
        }

        public override bool Equals(object obj)
        {
            if (ReferenceEquals(null, obj)) return false;
            if (ReferenceEquals(this, obj)) return true;
            var other = obj as CmpNode;
            return other != null && Equals(other);
        }

        public override int GetHashCode()
        {
            unchecked
            {
                return (
                    IgnoreCase.GetHashCode(DependentUpon)*397) ^
                    IgnoreCase.GetHashCode(FullName);
            }
        }

        public static bool operator ==(CmpNode left, CmpNode right) => Equals(left, right);
        public static bool operator !=(CmpNode left, CmpNode right) => !Equals(left, right);
    }
}


text:
#ifndef InputType_h
#define InputType_h

#include "FeatureObserver.h"
#include "HTMLTextFormControlElement.h"
#include "StepRange.h"
#include <wtf/FastMalloc.h>
#include <wtf/Forward.h>
#include <wtf/Noncopyable.h>
#include <wtf/RefPtr.h>

namespace WebCore {

class BeforeTextInsertedEvent;
class Chrome;
class Color;
class DateComponents;
class DragData;
class Event;
class FileList;
class FormDataList;
class HTMLElement;
class HTMLFormElement;
class HTMLInputElement;
class Icon;
class KeyboardEvent;
class MouseEvent;
class Node;
class RenderArena;
class RenderObject;
class RenderStyle;
class TouchEvent;

typedef int ExceptionCode;

struct ClickHandlingState {
    WTF_MAKE_FAST_ALLOCATED;

public:
    bool checked;
    bool indeterminate;
    RefPtr<HTMLInputElement> checkedRadioButton;
};

// An InputType object represents the type-specific part of an HTMLInputElement.
// Do not expose instances of InputType and classes derived from it to classes
// other than HTMLInputElement.
class InputType {
    WTF_MAKE_NONCOPYABLE(InputType);
    WTF_MAKE_FAST_ALLOCATED;

public:
    static OwnPtr<InputType> create(HTMLInputElement&, const AtomicString&);
    static OwnPtr<InputType> createText(HTMLInputElement&);
    virtual ~InputType();

    static bool themeSupportsDataListUI(InputType*);

    virtual const AtomicString& formControlType() const = 0;
    virtual bool canChangeFromAnotherType() const;

    // Type query functions

    // Any time we are using one of these functions it's best to refactor
    // to add a virtual function to allow the input type object to do the
    // work instead, or at least make a query function that asks a higher
    // level question. These functions make the HTMLInputElement class
    // inflexible because it's harder to add new input types if there is
    // scattered code with special cases for various types.

#if ENABLE(INPUT_TYPE_COLOR)
    virtual bool isColorControl() const;
#endif
    virtual bool isCheckbox() const;
    virtual bool isDateField() const;
    virtual bool isDateTimeField() const;
    virtual bool isDateTimeLocalField() const;
    virtual bool isEmailField() const;
    virtual bool isFileUpload() const;
    virtual bool isHiddenType() const;
    virtual bool isImageButton() const;
    virtual bool supportLabels() const;
    virtual bool isMonthField() const;
    virtual bool isNumberField() const;
    virtual bool isPasswordField() const;
    virtual bool isRadioButton() const;
    virtual bool isRangeControl() const;
    virtual bool isSearchField() const;
    virtual bool isSubmitButton() const;
    virtual bool isTelephoneField() const;
    virtual bool isTextButton() const;
    virtual bool isTextField() const;
    virtual bool isTextType() const;
    virtual bool isTimeField() const;
    virtual bool isURLField() const;
    virtual bool isWeekField() const;

    // Form value functions

    virtual bool shouldSaveAndRestoreFormControlState() const;
    virtual FormControlState saveFormControlState() const;
    virtual void restoreFormControlState(const FormControlState&);
    virtual bool isFormDataAppendable() const;
    virtual bool appendFormData(FormDataList&, bool multipart) const;

    // DOM property functions

    virtual bool getTypeSpecificValue(String&); // Checked first, before internal storage or the value attribute.
    virtual String fallbackValue() const; // Checked last, if both internal storage and value attribute are missing.
    virtual String defaultValue() const; // Checked after even fallbackValue, only when the valueWithDefault function is called.
    virtual double valueAsDate() const;
    virtual void setValueAsDate(double, ExceptionCode&) const;
    virtual double valueAsDouble() const;
    virtual void setValueAsDouble(double, TextFieldEventBehavior, ExceptionCode&) const;
    virtual void setValueAsDecimal(const Decimal&, TextFieldEventBehavior, ExceptionCode&) const;

    // Validation functions
    virtual String validationMessage() const;
    virtual bool supportsValidation() const;
    virtual bool typeMismatchFor(const String&) const;
    // Type check for the current input value. We do nothing for some types
    // though typeMismatchFor() does something for them because of value
    // sanitization.
    virtual bool typeMismatch() const;
    virtual bool supportsRequired() const;
    virtual bool valueMissing(const String&) const;
    virtual bool hasBadInput() const;
    virtual bool patternMismatch(const String&) const;
    bool rangeUnderflow(const String&) const;
    bool rangeOverflow(const String&) const;
    bool isInRange(const String&) const;
    bool isOutOfRange(const String&) const;
    virtual Decimal defaultValueForStepUp() const;
    double minimum() const;
    double maximum() const;
    virtual bool sizeShouldIncludeDecoration(int defaultSize, int& preferredSize) const;
    virtual float decorationWidth() const;
    bool stepMismatch(const String&) const;
    virtual bool getAllowedValueStep(Decimal*) const;
    virtual StepRange createStepRange(AnyStepHandling) const;
    virtual void stepUp(int, ExceptionCode&);
    virtual void stepUpFromRenderer(int);
    virtual String badInputText() const;
    virtual String typeMismatchText() const;
    virtual String valueMissingText() const;
    virtual bool canSetStringValue() const;
    virtual String localizeValue(const String&) const;
    virtual String visibleValue() const;
    // Returing the null string means "use the default value."
    // This function must be called only by HTMLInputElement::sanitizeValue().
    virtual String sanitizeValue(const String&) const;

    // Event handlers

    virtual void handleClickEvent(MouseEvent*);
    virtual void handleMouseDownEvent(MouseEvent*);
    virtual OwnPtr<ClickHandlingState> willDispatchClick();
    virtual void didDispatchClick(Event*, const ClickHandlingState&);
    virtual void handleDOMActivateEvent(Event*);
    virtual void handleKeydownEvent(KeyboardEvent*);
    virtual void handleKeypressEvent(KeyboardEvent*);
    virtual void handleKeyupEvent(KeyboardEvent*);
    virtual void handleBeforeTextInsertedEvent(BeforeTextInsertedEvent*);
#if ENABLE(TOUCH_EVENTS)
    virtual void handleTouchEvent(TouchEvent*);
#endif
    virtual void forwardEvent(Event*);
    // Helpers for event handlers.
    virtual bool shouldSubmitImplicitly(Event*);
    virtual PassRefPtr<HTMLFormElement> formForSubmission() const;
    virtual bool hasCustomFocusLogic() const;
    virtual bool isKeyboardFocusable(KeyboardEvent*) const;
    virtual bool isMouseFocusable() const;
    virtual bool shouldUseInputMethod() const;
    virtual void handleFocusEvent(Node* oldFocusedNode, FocusDirection);
    virtual void handleBlurEvent();
    virtual void accessKeyAction(bool sendMouseEvents);
    virtual bool canBeSuccessfulSubmitButton();
    virtual void subtreeHasChanged();
#if ENABLE(TOUCH_EVENTS)
    virtual bool hasTouchEventHandler() const;
#endif

    virtual void blur();

    // Shadow tree handling

    virtual void createShadowSubtree();
    virtual void destroyShadowSubtree();

    virtual HTMLElement* containerElement() const { return 0; }
    virtual HTMLElement* innerBlockElement() const { return 0; }
    virtual HTMLElement* innerTextElement() const { return 0; }
    virtual HTMLElement* innerSpinButtonElement() const { return 0; }
    virtual HTMLElement* resultsButtonElement() const { return 0; }
    virtual HTMLElement* cancelButtonElement() const { return 0; }
#if ENABLE(INPUT_SPEECH)
    virtual HTMLElement* speechButtonElement() const { return 0; }
#endif
    virtual HTMLElement* sliderThumbElement() const { return 0; }
    virtual HTMLElement* sliderTrackElement() const { return 0; }
    virtual HTMLElement* placeholderElement() const;

    // Miscellaneous functions

    virtual bool rendererIsNeeded();
    virtual RenderElement* createRenderer(RenderArena&, RenderStyle&) const;
    virtual void addSearchResult();
    virtual void attach();
    virtual void detach();
    virtual void minOrMaxAttributeChanged();
    virtual void stepAttributeChanged();
    virtual void altAttributeChanged();
    virtual void srcAttributeChanged();
    virtual bool shouldRespectAlignAttribute();
    virtual FileList* files();
    virtual void setFiles(PassRefPtr<FileList>);
    // Should return true if the given DragData has more than one dropped files.
    virtual bool receiveDroppedFiles(const DragData*);
#if ENABLE(FILE_SYSTEM)
    virtual String droppedFileSystemId();
#endif
    virtual Icon* icon() const;
    // Should return true if the corresponding renderer for a type can display a suggested value.
    virtual bool canSetSuggestedValue();
    virtual bool shouldSendChangeEventAfterCheckedChanged();
    virtual bool canSetValue(const String&);
    virtual bool storesValueSeparateFromAttribute();
    virtual void setValue(const String&, bool valueChanged, TextFieldEventBehavior);
    virtual bool shouldResetOnDocumentActivation();
    virtual bool shouldRespectListAttribute();
    virtual bool shouldRespectSpeechAttribute();
    virtual bool isEnumeratable();
    virtual bool isCheckable();
    virtual bool isSteppable() const;
    virtual bool shouldRespectHeightAndWidthAttributes();
    virtual bool supportsPlaceholder() const;
    virtual bool supportsReadOnly() const;
    virtual void updateInnerTextValue();
    virtual void updatePlaceholderText();
    virtual void attributeChanged();
    virtual void multipleAttributeChanged();
    virtual void disabledAttributeChanged();
    virtual void readonlyAttributeChanged();
    virtual void requiredAttributeChanged();
    virtual void valueAttributeChanged();
    virtual String defaultToolTip() const;
#if ENABLE(DATALIST_ELEMENT)
    virtual void listAttributeTargetChanged();
    virtual Decimal findClosestTickMarkValue(const Decimal&);
#endif
    virtual void updateClearButtonVisibility();

    // Parses the specified string for the type, and return
    // the Decimal value for the parsing result if the parsing
    // succeeds; Returns defaultValue otherwise. This function can
    // return NaN or Infinity only if defaultValue is NaN or Infinity.
    virtual Decimal parseToNumber(const String&, const Decimal& defaultValue) const;

    // Parses the specified string for this InputType, and returns true if it
    // is successfully parsed. An instance pointed by the DateComponents*
    // parameter will have parsed values and be modified even if the parsing
    // fails. The DateComponents* parameter may be 0.
    virtual bool parseToDateComponents(const String&, DateComponents*) const;

    // Create a string representation of the specified Decimal value for the
    // input type. If NaN or Infinity is specified, this returns an empty
    // string. This should not be called for types without valueAsNumber.
    virtual String serialize(const Decimal&) const;

    virtual bool supportsIndeterminateAppearance() const;

    virtual bool supportsSelectionAPI() const;

    // Gets width and height of the input element if the type of the
    // element is image. It returns 0 if the element is not image type.
    virtual unsigned height() const;
    virtual unsigned width() const;

    void dispatchSimulatedClickIfActive(KeyboardEvent*) const;

protected:
    explicit InputType(HTMLInputElement& element) : m_element(element) { }
    HTMLInputElement& element() const { return m_element; }
    Chrome* chrome() const;
    Decimal parseToNumberOrNaN(const String&) const;
    void observeFeatureIfVisible(FeatureObserver::Feature) const;

private:
    // Helper for stepUp()/stepDown(). Adds step value * count to the current value.
    void applyStep(int count, AnyStepHandling, TextFieldEventBehavior, ExceptionCode&);

    // Raw pointer because the HTMLInputElement object owns this InputType object.
    HTMLInputElement& m_element;
};

} // namespace WebCore
#endif


text:
<?xml version="1.0" encoding="utf-8"?>
<migration name="Add fuel consumption indicator">
  <item-creation item="indicators#fuel_consumption" datatype="measure" unit="liter_per_hour" sampling_temporal_mode="instant"/>
  <item-change  item="product_natures#car" variable_indicators="net_surface_area, motor_power, geolocation, fuel_consumption"/>
  <item-change  item="product_natures#tractor" variable_indicators="net_surface_area, motor_power, geolocation, fuel_consumption"/>
  <item-change  item="product_natures#truck" variable_indicators="net_surface_area, motor_power, geolocation, fuel_consumption"/>
</migration>


text:
package i5.las2peer.services.iStarMLModelService;

import i5.las2peer.api.Service;

import i5.las2peer.p2p.ArtifactNotFoundException;
import i5.las2peer.p2p.StorageException;

import i5.las2peer.persistency.Envelope;
import i5.las2peer.security.Agent;
import i5.las2peer.security.UserAgent;
import i5.las2peer.services.iStarMLModelService.data.*;
import i5.las2peer.services.iStarMLModelService.mail.*;

import i5.las2peer.restMapper.RESTMapper;
import i5.las2peer.restMapper.annotations.*;

import java.io.File;
import java.io.FileInputStream;
//import java.io.FileOutputStream;
//import java.io.PrintStream;
import java.io.StringReader;

//import java.io.PrintWriter;
import java.io.StringWriter;

import java.security.MessageDigest;
import java.util.HashMap;
import java.util.Timer;
import java.util.TimerTask;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;

import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathFactory;

import org.w3c.dom.Document;
import org.w3c.dom.Element;

import org.xml.sax.InputSource;
import org.apache.commons.codec.binary.Base64;
/**
 * Service responsible for model management of iStarML models
 * @author Alexander
 *
 */
@Path("IStarMLModelService")
public class IStarMLModelService extends Service
{
	private static final String ENVELOPE_NAME = "_IStarMLModelService_eXistWrite";
	private static final String SERVICE_CONFIG = "./config/serviceConfig.conf";
	private static final String DEPENDEE_TYPE = "dependee";
	private static final String DEPENDER_TYPE = "depender";
	private static final String IELEMENT_NODE_TYPE = "ielement";
	private static final String ACTOR_NODE_TYPE = "actor";
	static final String OK = "ok";

	protected static String existURI =  "xmldb:exist://localhost:8080/exist/xmlrpc";//URI to the Database

	private XPath _xPath =  XPathFactory.newInstance().newXPath();

	private Document _doc;
    private DocumentBuilder _dBuilder;
	private MailData _userCreatedMail;
	private String _admin;
	private String _adminPass;
    private boolean _useHTTPS=false;

    private HashMap<Long,ModelBuffer> bufferHack=new HashMap<>();
    private static Timer timer=new Timer();
	/**
	 * Reads a local file from a given path
	 * @param path
	 * @return file contents as a String
	 * @throws Exception
	 */
	private String getFile(String path)throws Exception
	{
		File file = new File(path);
		FileInputStream fis = new FileInputStream(file);
		byte[] data = new byte[(int)file.length()];
		fis.read(data);
		fis.close();
        return new String(data, "UTF-8");
	}
	/*
	private void setFile(String path, String text) throws Exception
	{
		PrintStream out = null;
		try {
		    out = new PrintStream(new FileOutputStream(path));
		    out.print(text);
		}
		finally {
		    if (out != null) out.close();
		}
	}*/
	/**
	 * Reads the service config file
	 * @throws Exception
	 */
	private void getConfig() throws Exception
	{
		try {
			String contents=getFile(SERVICE_CONFIG);
			_doc= _dBuilder.parse(new InputSource(new StringReader(contents)));
			_doc.getDocumentElement().normalize();

			existURI =(String) _xPath.compile("//uris/existuri/@value").evaluate(_doc, XPathConstants.STRING);
			_admin = (String) _xPath.compile("//dblogin/admin/@value").evaluate(_doc, XPathConstants.STRING);
			_adminPass = (String) _xPath.compile("//dblogin/adminPass/@value").evaluate(_doc, XPathConstants.STRING);
            boolean _sendMail = ((String) _xPath.compile("//mail/@value").evaluate(_doc, XPathConstants.STRING)).toLowerCase().trim().equals("true");
			_useHTTPS=((String) _xPath.compile("//dblogin/https/@value").evaluate(_doc, XPathConstants.STRING)).toLowerCase().trim().equals("true");
			_userCreatedMail=null;

			if(_sendMail)
			{
				String to=(String) _xPath.compile("//mail/to/@value").evaluate(_doc, XPathConstants.STRING);
				String host=(String) _xPath.compile("//mail/host/@value").evaluate(_doc, XPathConstants.STRING);
				int port=Integer.valueOf((String) _xPath.compile("//mail/port/@value").evaluate(_doc, XPathConstants.STRING));
				String user=(String) _xPath.compile("//mail/user/@value").evaluate(_doc, XPathConstants.STRING);
				String pass=(String) _xPath.compile("//mail/pass/@value").evaluate(_doc, XPathConstants.STRING);
				String subject=(String) _xPath.compile("//mail/subject/@value").evaluate(_doc, XPathConstants.STRING);
				String text=(String) _xPath.compile("//mail/text/@value").evaluate(_doc, XPathConstants.STRING);

				_userCreatedMail = new MailData(to, host, port, user, pass, subject, text);
			}

		} catch (Exception e) {
			System.out.println("Could not read config file "+ SERVICE_CONFIG);
			e.printStackTrace();
		}

	}
	/**
	 * Retrieves the data stored in the users service envelope
	 * @return
	 */
	private ModelBuffer getEnvelopeData()
	{
		try {
			Agent agent=this.getContext().getMainAgent();
			Long userID=agent.getId();

			if(!bufferHack.containsKey(userID))
            {
                bufferHack.put(userID,new ModelBuffer("","",""));
            }
            else
                return bufferHack.get(userID);

            /*String envelopeName=Long.toString(userID)+ENVELOPE_NAME;
			Envelope envi;
			try
			{
				envi=Envelope.fetchClassIdEnvelope(ModelBuffer.class, envelopeName);
			}
			catch(StorageException | ArtifactNotFoundException se)//Envelope not existing?
			{

				return null;
			}
			if(envi!=null)
			{
				envi.open(agent);//open
				ModelBuffer mb=envi.getContent(ModelBuffer.class);//read
				envi.close();//close
				return mb;
			}*/

		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return null;
	}
	/**
	 * Updates the current envelope contents or creates a new envelope, if none exists
	 * @param update new contents
	 */
	private void updateEnvelopeData(ModelBuffer update)
	{
		try
		{

			Agent agent=this.getContext().getMainAgent();
			Long userID=agent.getId();

            if(!bufferHack.containsKey(userID))
                bufferHack.put(userID,update);
            else
            {
                bufferHack.put(userID, update);
            }

            cleanUp();
			/*String envelopeName=Long.toString(userID)+ENVELOPE_NAME;
			Envelope envi;
			try
			{
				envi=Envelope.fetchClassIdEnvelope(ModelBuffer.class, envelopeName);
			}
			catch(StorageException | ArtifactNotFoundException se)//Envelope not existing?
			{
				//create new
				envi = Envelope.createClassIdEnvelope(update,envelopeName,this.getContext().getMainAgent());
				envi.open(agent);
				envi.setOverWriteBlindly(true);
				envi.store();
				envi.close();

			}
			if(envi!=null)
			{
				envi.open(agent);
				ModelBuffer mb =envi.getContent(ModelBuffer.class);	//change the contents
				mb.setCollection(update.getCollection());
				mb.setResource(update.getResource());
				mb.setContent(update.getContent());
				envi.store();//re serialize it

				envi.close();

			}*/

		} catch (Exception e) {

			e.printStackTrace();
		}
	}
	/**
	 * Commits the buffered contents to the database
	 * @param buffer
	 */
	private void commitModelBuffer(ModelBuffer buffer)
	{
		try
		{
			if(!buffer.getContent().trim().isEmpty())//only if content is provided, empty "" is illegal anyway
			{
				StringTuple st =getDBLogin();
				Storage storage= new Storage(existURI,_useHTTPS, st.getS1(), st.getS2());
				storage.loadCollection(buffer.getCollection());
				storage.setResource(buffer.getResource(), buffer.getContent());
				storage.closeCollection();
				updateEnvelopeData(new ModelBuffer("", "", ""));//remove, so no further repeated 'ghost' commits
			}
		}
		catch(Exception e)
		{
			updateEnvelopeData(new ModelBuffer("", "", ""));
			e.printStackTrace();
		}
	}
	/**
	 * cleanUp is called from the web connector before user logout.
	 * Commits all buffered changes
	 */
	public void cleanUp()
	{
		ModelBuffer buffer=getEnvelopeData();
		if(buffer!=null)
		{
			commitModelBuffer(buffer);

		}

	}
	/**
	 * Retrieves the contents of a resource from the DB
	 * @param collection path
	 * @param resource resource name
	 * @return XML content as String
	 * @throws Exception
	 */
	private String getResourceContentFromDB(String collection, String resource)throws Exception
	{
		StringTuple st =getDBLogin();
		Storage storage= new Storage(existURI,_useHTTPS, st.getS1(), st.getS2());
		storage.loadCollection(collection);
		return storage.getResource(resource);

	}
	/**
	 * Loads the {@link IStarMLEditor} and the contents from the proper source (buffer or DB)
	 * @param collection
	 * @param resource
	 * @return {@link IStarMLEditor} with the already loaded XML
	 * @throws Exception
	 */
	private IStarMLEditor getEditor(String collection,String resource) throws Exception
	{
		IStarMLEditor editor=new IStarMLEditor();
		ModelBuffer mb = getEnvelopeData();
		String resContent;
		boolean validate;

		if(mb==null||mb.getContent().isEmpty()||mb.getResource().isEmpty())//if buffer is not existent or empty, load from DB
		{
			resContent=getResourceContentFromDB(collection,resource);
			validate=true;//validate from database
		}
		else
		{

			if(!(mb.getCollection().equals(collection)&&mb.getResource().equals(resource)))//some other resource specified in the buffer?
			{
				commitModelBuffer(mb); //commit and get from DB
				resContent=getResourceContentFromDB(collection,resource);
				validate=true;//validate from database
			}
			else
			{
				resContent=mb.getContent();//get from envelope
				validate=false;//don't validate from envelope

			}
		}
		//System.out.println(validate);
		//load and check
		if(editor.loadXML(resContent, validate).startsWith("Error:"))
		{
			updateEnvelopeData(new ModelBuffer("", "", ""));//if error (invalid XML), remove buffer

			/*resContent=getResourceContentFromDB(collection,resource);
			if(editor.loadXML(resContent).startsWith("Error:"))
			{
				StringTuple st =getDBLogin();
				Storage storage= new Storage(existURI, st.getS1(), st.getS2());
				storage.loadCollection(collection);
				ResourceVersion[] rv=storage.getVersions(resource);
				int highestrev=0;
				for(int i=0;i<rv.length;i++)
				{
					int rev=rv[i].getRevision();
					if(rev>highestrev)
						highestrev=rev;
				}
				if(highestrev>0)//restore last probably correct
				{
					resContent=storage.getVersion(resource, highestrev);
					editor.loadXML(resContent);
				}
			}*/
		}
		return editor;
	}
	/**
	 * Constructor, reads config
	 * @throws Exception
	 */
	public IStarMLModelService() throws Exception
	{

        DocumentBuilderFactory _dbFactory = DocumentBuilderFactory.newInstance();
		_dBuilder = _dbFactory.newDocumentBuilder();
		getConfig();
/*
        timer.scheduleAtFixedRate(
                new TimerTask()
                {
                    public void run()
                    {
                       cleanUp();
                    }
                },
                0,      // run first occurrence immediately
                5*1000);
*/

	}
    public String getRESTMapping()
    {
        String result="";
        try
        {
            result= RESTMapper.getMethodsAsXML(this.getClass());
        }
        catch (Exception e)
        {
            e.printStackTrace();
        }
        return result;
    }

	/**
	 * Converts the given path from the URI to a valid Collection path (col1.col12.col123-> col1/col12/col123)
	 * @param collection URI collection path to convert
	 * @return valid collection path
	 */
	private String getCollectionPath(String collection) {
		return collection.replace('.', '/');
	}
	/**
	 * Creates a XML as a String to return multiple data values to the service caller
	 * @param data shallow tree of data elements
	 * @return XML
	 * @throws Exception
	 */
	private String createXMLResponse(XMLResponseElement data) throws Exception
	{
		return createXMLResponse(new XMLResponseElement[]{data});
	}
	/**
	 * Creates a XML as a String to return multiple data values to the service caller
	 * @param data shallow trees of data elements
	 * @return XML
	 * @throws Exception
	 */
	private String createXMLResponse(XMLResponseElement[] data) throws Exception
	{
		_doc=_dBuilder.newDocument();
		Element root=_doc.createElement("ModelResponse");//wrap everything into a root node
		_doc.appendChild(root);

		for (XMLResponseElement xmlResponseElement : data) //for each group
		{
			Element elem=_doc.createElement(xmlResponseElement.getName());
			for (XMLResponseElementChild dataElem : xmlResponseElement.getChildren()) //for each element of a group
			{
				Element subElem=_doc.createElement(dataElem.getName()); //create an element and set attributes
				for (StringTuple attribute : dataElem.getAttributes())
				{
					subElem.setAttribute(attribute.getS1(), attribute.getS2());
				}
				elem.appendChild(subElem);
			}
			root.appendChild(elem);
		}
		return prettyPrintXML(_doc);
	}
	/**
	 * Creates automatic user name and password from las2peer login data (private key is hashed)
	 * @return name and password for DB login
	 * @throws Exception
	 */

	private StringTuple getDBLogin() throws Exception
	{
		Long userID=this.getContext().getMainAgent().getId();
		String name=((UserAgent)(this.getContext().getLocalNode().getAgent(userID))).getLoginName();
		String key=Base64.encodeBase64String(this.getContext().getMainAgent().getPrivateKey().getEncoded());

		MessageDigest md = MessageDigest.getInstance("SHA-256");
		md.update(key.getBytes("UTF-8"));
		byte[] digest = md.digest();
		String pass=Base64.encodeBase64String(digest);

		name=name.replace(" ", "_");
		return new StringTuple(name,pass);
	}

	/**
	 * Creates a new account for the database, if not already existing and informs someone via mail (if set in the config file)
	 * @param message optional personal message to the mail receiver
	 * @return OK
	 * @throws Exception
	 */
	@POST
	@Path("setting/register/DB")
	public String createDBAccount(@DefaultValue("") @ContentParam() String message) throws Exception
	{
		try
        {
            StringTuple st =getDBLogin();
            Storage storage= new Storage(existURI,_useHTTPS, st.getS1(), st.getS2());

            _userCreatedMail.setSubject("User registered for DB: "+st.getS1());
            _userCreatedMail.setText("User message: \n\n"+message);

            storage.createUser(_admin, _adminPass, st.getS1(), st.getS2(), _userCreatedMail);

            return OK;
        }
        catch(Exception e)
        {
            return e.getMessage();
        }

	}

	/**
	 * Checks, if a DB account already exists for the current user
	 * @return true, if user has already a DB account
	 * @throws Exception
	 */
	@GET
	@Path("setting/register/DB")
	public boolean existsDBAccount() throws Exception
	{
		StringTuple st =getDBLogin();
		try
		{
			Storage storage= new Storage(existURI,_useHTTPS, st.getS1(), st.getS2());
			storage.loadCollection("");
			storage.closeCollection();
			return true;
		}
		catch(Exception e) //lazy check (for an xquery a login is needed)
		{
			return false;
		}

	}
	/**
	 * Simply checks, if valid login data is provided for the service
	 * @return
	 * @throws Exception
	 */
	@GET
	@Path("")
	public String loginCheck() throws Exception
	{

		return OK;
	}
	/**
	 * Lists the contents (sub-Collections and resources) of a collection, allows search
	 * @param collection root collection to search/list
	 * @param searchQuery optional search query for a resource or node element
	 * @param searchType type of search: model or node type (actor, role, goal etc)
	 * @return list of contents (or matches of the search)
	 * @throws Exception
	 */
	@GET
	@Path("{collection}")
	public String listCollectionContents(@PathParam("collection") String collection,
			@QueryParam(name="search",defaultValue = "") String searchQuery, @QueryParam(name="searchType", defaultValue = "model") String searchType) throws Exception
	{
		cleanUp();
		collection = getCollectionPath(collection);
		StringTuple st =getDBLogin();
		Storage storage= new Storage(existURI, _useHTTPS,st.getS1(), st.getS2());
		storage.loadCollection(collection);
		String result="";
		if(!searchQuery.trim().isEmpty())//user wants to search
		{

			String[] found=storage.searchStorage(collection,searchQuery,searchType);
			XMLResponseElement resp = new XMLResponseElement("SearchResult"); //construct return XML structure
            for(String aFound : found)
            {

                String name = aFound;
                String path = aFound;
                int lastSlash = name.lastIndexOf('/');
                if(lastSlash > 0)
                {
                    name = name.substring(lastSlash + 1); //separate path and resource name
                    path = path.substring(0, lastSlash);
                }

                XMLResponseElementChild respChild = new XMLResponseElementChild("Resource");
                respChild.addAttribute("name", name);
                respChild.addAttribute("path", path);
                respChild.addAttribute("lastModified", storage.getLastModifiedDate(path, name).toString());
                resp.addChild(respChild);
            }
			result=createXMLResponse(resp);

		}
		else//just output collection contents
		{

			CollectionData[] cols=storage.listSubCollections(collection);
			ResourceData[] res=storage.listResources(collection);
			XMLResponseElement respCol = new XMLResponseElement("Collections");
            for(CollectionData col : cols)
            {
                XMLResponseElementChild respChild = new XMLResponseElementChild("Collection");
                respChild.addAttribute("name", col.getName());
                respChild.addAttribute("owner", col.getOwner());
                respCol.addChild(respChild);
            }

			XMLResponseElement respRes = new XMLResponseElement("Resources");
            for(ResourceData re : res)
            {
                XMLResponseElementChild respChild = new XMLResponseElementChild("Resource");
                respChild.addAttribute("name", re.getName());
                respChild.addAttribute("lastModified", re.getLastModified().toString());
                respRes.addChild(respChild);
            }

			result=createXMLResponse(new XMLResponseElement[]{respCol,respRes});
		}
		storage.closeCollection();
		return result;
	}
	/**
	 * Retrieves the contents of a resource (XML)
	 * @param collection
	 * @param resource
	 * @return XML content of the given resource
	 * @throws Exception
	 */
	@GET
	@Path("{collection}/{resource}")
	public String getResourceContents(@PathParam("collection") String collection,
			@PathParam("resource") String resource) throws Exception
	{
		cleanUp();
		collection = getCollectionPath(collection);
		StringTuple st =getDBLogin();
		Storage storage= new Storage(existURI, _useHTTPS,st.getS1(), st.getS2());
		storage.loadCollection(collection);
		String result;
		result=storage.getResource(resource);
		storage.closeCollection();

		return result;
	}
	/**
	 * Copies a resource
	 * @param collection source collection
	 * @param resource source resource
	 * @param targetCollection target collection
	 * @param targetResource target resource
	 * @return OK
	 * @throws Exception
	 */
	@PUT
	@Path("{collection}/{resource}/copy/{targetCollection}/{targetResource}")
	public String copyResource(@PathParam("collection") String collection,
			@PathParam("resource") String resource,
			@PathParam("targetCollection") String targetCollection,
			@PathParam("targetResource") String targetResource
			) throws Exception
	{

		String res=getResourceContents(collection, resource);
		setResourceContent(targetCollection,targetResource,res);

		return OK;
	}
	/**
	 * Copies a specific resource version
	 * @param collection source collection
	 * @param resource source resource
	 * @param version version of source resource, use 0 for base version.
	 * @param targetCollection target collection
	 * @param targetResource target resource
	 * @return OK
	 * @throws Exception
	 */
	@PUT
	@Path("{collection}/{resource}/versions/{version}/copy/{targetCollection}/{targetResource}")
	public String copyResource(@PathParam("collection") String collection,
			@PathParam("resource") String resource,
			@PathParam("version") int version,
			@PathParam("targetCollection") String targetCollection,
			@PathParam("targetResource") String targetResource
			) throws Exception
	{
		String res=getResourceContents(collection, resource,version);
		setResourceContent(targetCollection,targetResource,res);

		return OK;
	}
	/**
	 * Lists all previous versions of a resource
	 * @param collection
	 * @param resource
	 * @return list with previous versions (name, revision, date, user)
	 * @throws Exception
	 */
	@GET
	@Path("{collection}/{resource}/versions")
	public String getResourceVersions(@PathParam("collection") String collection,
			@PathParam("resource") String resource) throws Exception
	{
		cleanUp();
		collection = getCollectionPath(collection);
		StringTuple st =getDBLogin();
		Storage storage= new Storage(existURI,_useHTTPS, st.getS1(), st.getS2());
		storage.loadCollection(collection);
		String result;
		ResourceVersion[] versionData=storage.getVersions(resource);

		XMLResponseElement respRes = new XMLResponseElement("Versions");
        for(ResourceVersion aVersionData : versionData)
        {
            XMLResponseElementChild respChild = new XMLResponseElementChild("Version");
            respChild.addAttribute("name", resource);
            respChild.addAttribute("revision", Integer.toString(aVersionData.getRevision()));
            respChild.addAttribute("date", aVersionData.getDate().toString());
            respChild.addAttribute("user", aVersionData.getUser());
            respRes.addChild(respChild);
        }
		result=createXMLResponse(respRes);

		storage.closeCollection();
		return result;
	}
	/**
	 * Gets the specific version of a resource
	 * @param collection
	 * @param resource
	 * @param version use 0 for base version
	 * @return XML content of the specific version
	 * @throws Exception
	 */
	@GET
	@Path("{collection}/{resource}/versions/{version}")
	public String getResourceContents(@PathParam("collection") String collection,
			@PathParam("resource") String resource,@PathParam("version") int version) throws Exception
	{
		cleanUp();
		collection = getCollectionPath(collection);
		StringTuple st =getDBLogin();
		Storage storage= new Storage(existURI,_useHTTPS, st.getS1(), st.getS2());
		storage.loadCollection(collection);
		String result;
		result=storage.getVersion(resource,version);
		storage.closeCollection();
		return result;
	}
	/**
	 * Creates a new collection
	 * @param collection path to the new collection
	 * @return OK
	 * @throws Exception
	 */
	@PUT
	@Path("{collection}")
	public String createCollection(@PathParam("collection") String collection) throws Exception
	{
		cleanUp();
		collection = getCollectionPath(collection);
		StringTuple st =getDBLogin();

        Storage storage= new Storage(existURI,_useHTTPS, st.getS1(), st.getS2());

		storage.createCollection(collection);

		String result=OK;
		storage.closeCollection();

		return result;
	}
	/**
	 * Creates a new resource. Already existing resources are not overwritten.
	 * @param collection
	 * @param resource
	 * @return OK
	 * @throws Exception
	 */
	@PUT
	@Path("{collection}/{resource}")
	public String createResource(@PathParam("collection") String collection,
			@PathParam("resource") String resource) throws Exception
	{
		cleanUp();
		collection = getCollectionPath(collection);
		StringTuple st =getDBLogin();
		Storage storage= new Storage(existURI,_useHTTPS, st.getS1(), st.getS2());
		storage.loadCollection(collection);
		storage.createResource(resource, "", false);
		String result=OK;
		storage.closeCollection();

		return result;
	}
	/**
	 * Deletes an existing collection
	 * @param collection
	 * @return OK
	 * @throws Exception
	 */
	@DELETE
	@Path("{collection}")
	public String deleteCollection(@PathParam("collection") String collection) throws Exception
	{
		cleanUp();
		collection = getCollectionPath(collection);
		StringTuple st =getDBLogin();
		Storage storage= new Storage(existURI,_useHTTPS, st.getS1(), st.getS2());
		storage.deleteCollection(collection);
		String result=OK;
		storage.closeCollection();

		return result;
	}
	/**
	 * Deletes an existing resource
	 * @param collection
	 * @param resource
	 * @return
	 * @throws Exception
	 */
	@DELETE
	@Path("{collection}/{resource}")
	public String deleteResource(@PathParam("collection") String collection,
			@PathParam("resource") String resource) throws Exception
	{
		cleanUp();
		collection = getCollectionPath(collection);
		StringTuple st =getDBLogin();
		Storage storage= new Storage(existURI,_useHTTPS, st.getS1(), st.getS2());
		storage.loadCollection(collection);
		storage.deleteResource(resource);
		String result=OK;
		storage.closeCollection();

		return result;
	}
	/**
	 * Creates a new resource with the given content.
	 * Existing resources are overwritten.
	 * @param collection
	 * @param resource
	 * @param content
	 * @return OK
	 * @throws Exception
	 */
	@POST
	@Path("{collection}/{resource}")
	public String setResourceContent(@PathParam("collection") String collection,
			@PathParam("resource") String resource, @ContentParam() String content ) throws Exception
	{
		cleanUp();
		collection = getCollectionPath(collection);
		StringTuple st =getDBLogin();
		Storage storage= new Storage(existURI,_useHTTPS, st.getS1(), st.getS2());
		storage.loadCollection(collection);
		storage.createResource(resource, content,true);
		String result=OK;
		storage.closeCollection();

		return result;
	}

	/**
	 * Creates a new node (actor/ielement) in an existing resource
	 * @param collection
	 * @param resource
	 * @param nodeType type of the node (actor/ielement)
	 * @param nodeID id of the node to create
	 * @param name name of the node to create
	 * @param type type of the node (agent, goal, role etc.)
	 * @param comment optional comment
	 * @return OK
	 * @throws Exception
	 */
	private String createNode(String collection,
			String resource, String nodeType, String nodeID,
			String name, String type,  String comment) throws Exception
	{
		collection = getCollectionPath(collection);

		IStarMLEditor editor=getEditor(collection,resource);

        switch(nodeType)
        {
            case ACTOR_NODE_TYPE:
                editor.createActor(nodeID, name, type, comment);
                break;
            case IELEMENT_NODE_TYPE:
                editor.createIElement(nodeID, name, type, comment);
                break;
            default:
                throw new Exception("Not supported node type: " + nodeType);
        }
		updateEnvelopeData(new ModelBuffer(editor.toString(),collection,resource));

		//storage.setResource(resource, editor.toString());

        //storage.closeCollection();

		return OK;
	}
	/**
	 * Creates a new actor node in a resource
	 * @param collection
	 * @param resource
	 * @param nodeID id of the actor
	 * @param name name of the actor
	 * @param type type of the actor
	 * @param comment optional comment
	 * @return OK
	 * @throws Exception
	 */
	@PUT
	@Path("{collection}/{resource}/actor/{nodeID}")
	public String createActor(@PathParam("collection") String collection,
			@PathParam("resource") String resource, @PathParam("nodeID") String nodeID,
			@QueryParam(name="name",defaultValue = "Actor") String name, @QueryParam(name="type",defaultValue = "actor") String type, @QueryParam(name="comment", defaultValue = "") String comment) throws Exception
	{
		return createNode(collection,resource,ACTOR_NODE_TYPE, nodeID, name,type,comment);
	}
	/**
	 * Creates a new ielement node in a resource
	 * @param collection
	 * @param resource
	 * @param nodeID id of the ielement
	 * @param name name of the ielement
	 * @param type type of the ielement
	 * @param comment optional comment
	 * @return OK
	 * @throws Exception
	 */
	@PUT
	@Path("{collection}/{resource}/ielement/{nodeID}")
	public String createIElement(@PathParam("collection") String collection,
			@PathParam("resource") String resource, @PathParam("nodeID") String nodeID,
			@QueryParam(name="name",defaultValue = "Goal") String name, @QueryParam(name="type",defaultValue = "goal") String type, @QueryParam(name="comment",defaultValue = "") String comment) throws Exception
	{
		return createNode(collection,resource,IELEMENT_NODE_TYPE, nodeID, name,type,comment);
	}
	/**
	 * Creates a new node in a resource without a specified id. The id is generated internally and returned to the caller.
	 * @param collection
	 * @param resource
	 * @param nodeType actor/ielement
	 * @param name name of the node
	 * @param type type of the node
	 * @param comment optional comment
	 * @return id used for the node
	 * @throws Exception
	 */
	private String createNode(String collection,
			String resource, String nodeType,
			String name, String type,  String comment) throws Exception
	{
		collection = getCollectionPath(collection);
		IStarMLEditor editor=getEditor(collection,resource);
		String id=editor.generateSomeFreeID();
		createNode(collection, resource, nodeType, id, name, type, comment);

		return id;
	}
	/**
	 * Creates a new actor node in a resource without a specific id
	 * @param collection
	 * @param resource
	 * @param name name of the actor
	 * @param type type of the actor
	 * @param comment optional comment
	 * @return id used for the actor
	 * @throws Exception
	 */
	@POST
	@Path("{collection}/{resource}/actor")
	public String createActor(@PathParam("collection") String collection,
			@PathParam("resource") String resource,
			@QueryParam(name="name",defaultValue = "Actor") String name, @QueryParam(name="type",defaultValue = "actor") String type, @QueryParam(name="comment",defaultValue = "") String comment) throws Exception
	{
		return createNode(collection, resource, ACTOR_NODE_TYPE, name, type, comment);
	}
	/**
	 * Creates a new ielement node in a resource without a specified id
	 * @param collection
	 * @param resource
	 * @param name name of the ielement
	 * @param type type of the ielement
	 * @param comment optional comment
	 * @return id used for the ielement
	 * @throws Exception
	 */
	@POST
	@Path("{collection}/{resource}/ielement")
	public String createIElement(@PathParam("collection") String collection,
			@PathParam("resource") String resource,
			@QueryParam(name="name",defaultValue = "Goal") String name, @QueryParam(name="type",defaultValue = "goal") String type, @QueryParam(name="comment", defaultValue = "") String comment) throws Exception
	{
		return createNode(collection, resource, IELEMENT_NODE_TYPE, name, type, comment);
	}
	/**
	 * Deletes a node in a resource
	 * @param collection
	 * @param resource
	 * @param nodeType actor/ielement
	 * @param nodeID id of the node to delete
	 * @return OK
	 * @throws Exception
	 */
	private String deleteNode(String collection,
			String resource, String nodeType,
			String nodeID) throws Exception
	{
		collection = getCollectionPath(collection);
		IStarMLEditor editor=getEditor(collection,resource);
        switch(nodeType)
        {
            case ACTOR_NODE_TYPE:
                editor.deleteActor(nodeID);
                break;
            case IELEMENT_NODE_TYPE:
                editor.deleteIElement(nodeID);
                break;
            default:
                throw new Exception("Not supported node type: " + nodeType);
        }

		updateEnvelopeData(new ModelBuffer(editor.toString(),collection,resource));
        return OK;
	}
	/**
	 * Deletes an actor in a resource
	 * @param collection
	 * @param resource
	 * @param nodeID id of the actor to delete
	 * @return OK
	 * @throws Exception
	 */
	@DELETE
	@Path("{collection}/{resource}/actor/{nodeID}")
	public String deleteActor(@PathParam("collection") String collection,
			@PathParam("resource") String resource, @PathParam("nodeID") String nodeID) throws Exception
	{
		return deleteNode(collection, resource, ACTOR_NODE_TYPE, nodeID);
	}
	/**
	 * Deletes an ielement of a resource
	 * @param collection
	 * @param resource
	 * @param nodeID id of the ielement to delete
	 * @return OK
	 * @throws Exception
	 */
	@DELETE
	@Path("{collection}/{resource}/ielement/{nodeID}")
	public String deleteIElement(@PathParam("collection") String collection,
			@PathParam("resource") String resource,@PathParam("nodeID") String nodeID) throws Exception
	{
		return deleteNode(collection, resource, IELEMENT_NODE_TYPE, nodeID);
	}
	/**
	 * Creates a new actorLink in a resource
	 * @param collection
	 * @param resource
	 * @param actorID id of the actor the link starts from
	 * @param targetID id of the target actor
	 * @param type type of the actorLink
	 * @return OK
	 * @throws Exception
	 */
	@PUT
	@Path("{collection}/{resource}/actor/{actorID}/actorLink/{targetID}")
	public String createActorLink(@PathParam("collection") String collection,
			@PathParam("resource") String resource, @PathParam("actorID") String actorID, @PathParam("targetID") String targetID,
			@QueryParam(name="type", defaultValue = "is_a") String type) throws Exception
	{

		collection = getCollectionPath(collection);
		IStarMLEditor editor=getEditor(collection,resource);

		editor.createActorLink(actorID, type, targetID);

		updateEnvelopeData(new ModelBuffer(editor.toString(),collection,resource));
        return OK;
	}
	/**
	 * Deletes an actorLink in a resource
	 * @param collection
	 * @param resource
	  * @param actorID id of the actor the link starts from
	 * @param targetID id of the target actor
	 * @return OK
	 * @throws Exception
	 */
	@DELETE
	@Path("{collection}/{resource}/actor/{actorID}/actorLink/{targetID}")
	public String deleteActorLink(@PathParam("collection") String collection,
			@PathParam("resource") String resource, @PathParam("actorID") String actorID, @PathParam("targetID") String targetID) throws Exception
	{
		collection = getCollectionPath(collection);
		IStarMLEditor editor=getEditor(collection,resource);
		String result=OK;

		editor.deleteActorLink(actorID, targetID);

		updateEnvelopeData(new ModelBuffer(editor.toString(),collection,resource));

		return result;
	}
	/**
	 * Creates a new dependency link in a resource
	 * @param collection
	 * @param resource
	 * @param ielementID id of the source ielement
	 * @param dependencyType depender/dependee
	 * @param targetID id of the target actor
	 * @return OK
	 * @throws Exception
	 */
	@PUT
	@Path("{collection}/{resource}/ielement/{ielementID}/{dependencyType}/{targetID}")
	public String createDependency(@PathParam("collection") String collection,
			@PathParam("resource") String resource, @PathParam("ielementID") String ielementID, @PathParam("dependencyType") String dependencyType,
			@PathParam("targetID") String targetID) throws Exception
	{
		collection = getCollectionPath(collection);
		IStarMLEditor editor=getEditor(collection,resource);
		String result=OK;

		if(dependencyType.equals(DEPENDER_TYPE))
		{
			editor.createDepender(ielementID, targetID);
		}
		else if(dependencyType.equals(DEPENDEE_TYPE))
		{
			editor.createDependee(ielementID, targetID);
		}
		else
			throw new Exception("Not supported dependency type: "+dependencyType);

		updateEnvelopeData(new ModelBuffer(editor.toString(),collection,resource));

		return result;
	}
	/**
	 * Deletes a Dependency link for a resource
	 * @param collection
	 * @param resource
	 * @param ielementID id of the source ielement
	 * @param dependencyType depender/dependee
	 * @param targetID id of the target actor
	 * @return OK
	 * @throws Exception
	 */
	@DELETE
	@Path("{collection}/{resource}/ielement/{ielementID}/{dependencyType}/{targetID}")
	public String deleteDependency(@PathParam("collection") String collection,
			@PathParam("resource") String resource, @PathParam("ielementID") String ielementID, @PathParam("dependencyType") String dependencyType,
			@PathParam("targetID") String targetID) throws Exception
	{
		collection = getCollectionPath(collection);
		IStarMLEditor editor=getEditor(collection,resource);
		String result=OK;

        switch(dependencyType)
        {
            case DEPENDER_TYPE:
                editor.deleteDepender(ielementID, targetID);
                break;
            case DEPENDEE_TYPE:
                editor.deleteDependee(ielementID, targetID);
                break;
            default:
                throw new Exception("Not supported dependency type: " + dependencyType);
        }

		updateEnvelopeData(new ModelBuffer(editor.toString(),collection,resource));

		return result;
	}

	/**
	 * creates a readable text from a given XML document
	 * @param document
	 * @return readable XML
	 * @throws Exception
	 */
	private String prettyPrintXML(Document document) throws Exception
	{
		Transformer t = TransformerFactory.newInstance().newTransformer();
		StreamResult out = new StreamResult(new StringWriter());
		t.setOutputProperty(OutputKeys.INDENT, "yes"); //pretty printing
		t.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2");
		t.setOutputProperty("{http://xml.apache.org/xalan}indent-amount", "2");
		t.transform(new DOMSource(document),out);
		return out.getWriter().toString();
	}

}


text:
import puremvc.interfaces
import puremvc.patterns.mediator

from ApplicationConstants import Notification

from pyjamas.Window import alert

class DialogMediator(puremvc.patterns.mediator.Mediator, puremvc.interfaces.IMediator):

    NAME = 'DialogMediator'

    def __init__(self, viewComponent):
        super(DialogMediator, self).__init__(DialogMediator.NAME, viewComponent)
        self.viewComponent.mediator = self

    def listNotificationInterests(self):
        return [
        Notification.SHOW_DIALOG,
        Notification.HELLO,
        ]

    def handleNotification(self, note):
        try:
            noteName = note.getName()
            if noteName == Notification.SHOW_DIALOG:
                alert(note.getBody())
            elif noteName == Notification.HELLO:
                alert("Hello there")
        except:
            raise