<file_sep>/*
 * Copyright 2015 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

//var KafkaUtils = function() {}
var JavaKakfaUtils = Java.type("org.apache.spark.streaming.kafka.KafkaUtils")
var KafkaUtils = {}

KafkaUtils.createStream = function(ssc, zkQuorum, group, topic) {
    var integer = new java.lang.Integer(1);
    var m = new java.util.HashMap();
    m.put(topic, integer);
    return new DStream(JavaKakfaUtils.createStream(ssc.getJavaObject(),
                                                   zkQuorum,
                                                   group,
                                                   m),
                       ssc);
};
<file_sep><!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="utf-8">
    <title>JSDoc: Source: sql/DataFrame.js</title>
    
    <script src="scripts/prettify/prettify.js"> </script>
    <script src="scripts/prettify/lang-css.js"> </script>
    <!--[if lt IE 9]>
      <script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
    <![endif]-->
    <link type="text/css" rel="stylesheet" href="styles/prettify-tomorrow.css">
    <link type="text/css" rel="stylesheet" href="styles/jsdoc-default.css">
</head>

<body>

<div id="main">
    
    <h1 class="page-title">Source: sql/DataFrame.js</h1>
    
    


    
    <section>
        <article>
            <pre class="prettyprint source"><code>/*
 * Copyright 2015 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/** 
 * @constructor
 * @classdesc A distributed collection of data organized into named columns. A DataFrame is equivalent to a relational table in Spark SQL.
 * @example
 * var people = sqlContext.read.parquet("...") 
 * @example  
 * // Once created, it can be manipulated using the various domain-specific-language (DSL) functions defined in: 
 * // DataFrame (this class), Column, and functions.
 * // To select a column from the data frame:
 * var ageCol = people("age") 
 */
var DataFrame = function(jvmDataFrame) {
	JavaWrapper.call(this, jvmDataFrame);

	  // Initialize our Row-specific properties
	this.logger = Logger.getLogger("sql.DataFrame_js");
};

DataFrame.prototype = Object.create(JavaWrapper.prototype); 

//Set the "constructor" property to refer to DataFrame
DataFrame.prototype.constructor = DataFrame;

/**
 * aggregates on the entire DataFrame without groups.
 * @example
 * // df.agg(...) is a shorthand for df.groupBy().agg(...)
 * var map = {};
 * map["age"] = "max";
 * map["salary"] = "avg";
 * df.agg(map)
 * df.groupBy().agg(map)
 * @param {hashMap} - hashMap&lt;String,String> exprs
 * @returns {DataFrame}
 */
DataFrame.prototype.agg = function(hashMap) {

    return new DataFrame(this.getJavaObject().agg(hashMap));

};
/**
 * Returns a new DataFrame with an alias set.
 * @param {string} alias
 * @returns {DataFrame}
 */
DataFrame.prototype.as = function(alias) {
    return new DataFrame(this.getJavaObject().as(alias));
};
/**
 * Selects column based on the column name and return it as a Column. 
 * Note that the column name can also reference to a nested column like a.b.
 * @param {string} colName
 * @returns {Column}
 */
DataFrame.prototype.apply = function(colName) {
    return new Column(this.getJavaObject().apply(colName));
};
/**
 * Persist this DataFrame with the default storage level (`MEMORY_ONLY`).
 * @returns {DataFrame}
 */
DataFrame.prototype.cache = function() {
    return new DataFrame(this.getJavaObject().cache());
};
/**
 * Returns a new DataFrame that has exactly numPartitions partitions. 
 * Similar to coalesce defined on an RDD, this operation results in a narrow dependency, 
 * e.g. if you go from 1000 partitions to 100 partitions, there will not be a shuffle, 
 * instead each of the 100 new partitions will claim 10 of the current partitions.
 * @param {integer} numPartitions
 * @returns {DataFrame}
 */
DataFrame.prototype.coalesce = function(numPartitions) {
    return new DataFrame(this.getJavaObject().coalesce());
};
/**
 * Selects column based on the column name and return it as a Column.
 * @param {string} name
 * @returns {Column}
 */
DataFrame.prototype.col = function(name) {
    return new Column(this.getJavaObject().col(name));
};
/**
 * Returns an array that contains all of Rows in this DataFrame.
 * @returns {Row[]}
 */
DataFrame.prototype.collect = function() {
    var jRows = this.getJavaObject().collect();
    var rows = [];
    for (var i = 0; i &lt; jRows.length; i++) {
    	rows.push(new Row(jRows[i]));
    }
    return rows;
};
/**
 * Returns all column names as an array.
 * @returns {string[]}
 */
DataFrame.prototype.columns = function() {
	var x = this.getJavaObject().columns();
	var s = [];
	for (var i = 0; i &lt; x.length; i++) {
		s.push(x[i]);
	}
    return s; 
};
/**
 * Returns the number of rows in the DataFrame.
 * @returns {integer}
 */
DataFrame.prototype.count = function() {
    return this.getJavaObject().count();
};
/**
 * Create a multi-dimensional cube for the current DataFrame using the specified columns, so we can run aggregation on them.
 * @param {string| Column} cols...
 * @example 
 * var df = peopleDataFrame.cube("age", "expense");
 * @returns {GroupedData}
 */
DataFrame.prototype.cube = function() {

	var args = Array.prototype.slice.call(arguments);
    if(typeof args[0] !== 'object') 
    	args = args.map(function(v) {
    		return this.col(v);
        }.bind(this));
    
    var jCols = args.map(function(v) {
        return Utils.unwrapObject(v);
    });
    
    return new GroupedData(this.getJavaObject().cube(jCols));
};
/**
 * Computes statistics for numeric columns, including count, mean, stddev, min, and max. 
 * If no columns are given, this function computes statistics for all numerical columns.
 * This function is meant for exploratory data analysis, as we make no guarantee about the backward 
 * compatibility of the schema of the resulting DataFrame. If you want to programmatically compute 
 * summary statistics, use the agg function instead.
 * @param {string} cols.... 
 * @example
 * var df = peopleDataFrame.describe("age", "expense");
 * @returns {DataFrame}
 */
DataFrame.prototype.describe = function() {
	var args = Array.prototype.slice.call(arguments);
    return new DataFrame(this.getJavaObject().describe(args));
};
/**
 * Returns a new DataFrame that contains only the unique rows from this DataFrame. This is an alias for dropDuplicates.
 */
DataFrame.prototype.distinct = function() {
    return this.dropDuplicates();
};
/**
 * Returns a new DataFrame with a column dropped.
 * @param {string | Column} col 
 * @returns {DataFrame}
 */
DataFrame.prototype.drop = function(col) {
    return new DataFrame(this.getJavaObject().drop(Utils.unwrapObject(col)));
};
/**
 * Returns a new DataFrame that contains only the unique rows from this DataFrame, if colNames then considering only the subset of columns.
 * @param {string[]} colNames
 * @returns {DataFrame}
 */
DataFrame.prototype.dropDuplicates = function(colNames) {
	if (!colNames) {
		return new DataFrame(this.getJavaObject().dropDuplicates());
	} else {
		return new DataFrame(this.getJavaObject().dropDuplicates(colNames));
	}
    
};
/**
 * Returns all column names and their data types as an array of arrays. ex. [["name","StringType"],["age","IntegerType"],["expense","IntegerType"]]
 * @returns {Array} Array of Array[2] 
 */
DataFrame.prototype.dtypes = function() {
	var d = this.getJavaObject().dtypes();
	var arrayOfTuple2 = [];
	for (var i = 0; i &lt; d.length; i++) {
		var tuple2 = Utils.javaToJs(d[i]); // convert Tuple2 to array[o1, o2]
		arrayOfTuple2.push(tuple2);
	}
	
	return arrayOfTuple2;
    
};
/**
 * Returns a new DataFrame containing rows in this frame but not in another frame. This is equivalent to EXCEPT in SQL.
 * @param {DataFrame} otherDataFrame to compare to this DataFrame
 * @returns {DataFrame}
 */
DataFrame.prototype.except = function(otherDataFrame) {
	return new DataFrame(this.getJavaObject().except(Utils.unwrapObject(otherDataFrame)));
};
/**
 * Prints the plans (logical and physical) to the console for debugging purposes.
 * @parma {boolean} if false prints the physical plans only.
 */
DataFrame.prototype.explain = function(extended) {
	var b = (extended) ? true : false;
	this.getJavaObject().explain(b);
};
/**
 * Filters rows using the given SQL expression string or Filters rows using the given Column..
 * @param {string | Column} 
 * @returns {DataFrame}
 */
DataFrame.prototype.filter = function(arg) {
    if(typeof arg === 'object') 
        return this.filterWithColumn(arguments[0]);
    else 
        return this.filterWithString(arguments[0]);
};
/**
 * Filters rows using the given Column
 * @param {Column} col
 * @returns {DataFrame}
 * @private
 */
DataFrame.prototype.filterWithColumn = function(col) {
    return new DataFrame(this.getJavaObject().filter(Utils.unwrapObject(col)));
};
/**
 * Filters rows using the given SQL expression string
 * @param {string} columnExpr
 * @returns {DataFrame}
 * @private
 */
DataFrame.prototype.filterWithString = function(columnExpr) {
    return new DataFrame(this.getJavaObject().filter(columnExpr));
};
/**
 * Returns the first row. Alias for head().
 * returns {Row}
 */
DataFrame.prototype.first = function() {
    return this.head();
};
/**
 * Returns a new RDD by first applying a function to all rows of this DataFrame, and then flattening the results.
 * @param {function} func
 * @returns {RDD}
 */
DataFrame.prototype.flatMap = function(func) {
 	return this.toRDD().flatMap(func);
};
/**
 * Applies a function to all elements of this DataFrame.
 * @example
 * rdd3.foreach(function(record) {
 *    var connection = createNewConnection()
 *    connection.send(record);	
 *    connection.close()
 * });
 * @param {function} Function with one parameter
 * @returns {void}
 */
DataFrame.prototype.foreach = function(func) {
	
	return this.toRDD().foreach(func);
};
/**
 * Applies a function to each partition of this DataFrame.
 * @example
 * df.foreachPartition(function(partitionOfRecords) {
 *    var connection = createNewConnection()
 *    partitionOfRecords.forEach(function(record){
 *       connection.send(record);	
 *    });
 *    connection.close()
 * });
 * @param {function} Function with one Array parameter
 * @returns {void}
 */
DataFrame.prototype.foreachPartition = function(func) {

	return this.toRDD().foreachPartition(func);
};

/**
 * Groups the DataFrame using the specified columns, so we can run aggregation on them
 * @param {string[] | Column[]} - Array of Column objects of column name strings
 * @returns {GroupedData}
 */
DataFrame.prototype.groupBy = function() {
    var args = Array.prototype.slice.call(arguments);
 
    if(typeof args[0] === 'object') 
        return this.groupByWithColumns(args);
    else 
        return this.groupByWithStrings(args);
};
/**
 * Groups the DataFrame using the specified columns, so we can run aggregation on them
 * @param {Columns[]}
 * @returns {GroupedData}
 * @private
 */
DataFrame.prototype.groupByWithColumns = function(args) {
    var jCols = args.map(function(v) {
        return Utils.unwrapObject(v);
    });

    var jGroupedData = this.getJavaObject().groupBy(jCols);
    var gd = new GroupedData(jGroupedData);

    return gd;
};
/**
 * Groups the DataFrame using the specified columns, so we can run aggregation on them
 * @param {string[]} columnNames
 * @returns {GroupedData}
 * @private
 */
DataFrame.prototype.groupByWithStrings = function(args) {
	var jCols = args.map(function(v) {
		return this.col(v);
    }.bind(this));
	
	return this.groupByWithColumns(jCols);

};
/**
 * Returns the first row.
 * @returns {Row}
 */
DataFrame.prototype.head = function() {
    return new Row(this.getJavaObject().head());
};
/**
 * Returns a best-effort snapshot of the files that compose this DataFrame. This method simply asks each constituent 
 * BaseRelation for its respective files and takes the union of all results. Depending on the source relations, 
 * this may not find all input files. Duplicates are removed.
 * @returns {string[]} files
 */
DataFrame.prototype.inputFiles = function() {
    var files=  this.getJavaObject().inputFiles();
    var retFiles = [];
    for (var i = 0; i &lt; files.length; i++ ) {
    	retFiles.push(files[i]);
    }
    return retFiles;
};
/**
 * Returns a new DataFrame containing rows only in both this frame and another frame. This is equivalent to INTERSECT in SQL
 * @param {DataFrame} other
 * @returns {DataFrame}
 */
DataFrame.prototype.intersect = function(other) {
    return new DataFrame(this.getJavaObject().intersect(Utils.unwrapObject(other)));
};
/**
 * Returns true if the collect and take methods can be run locally (without any Spark executors).
 * @returns {boolean}
 */
DataFrame.prototype.isLocal = function() {
	return this.getJavaObject().isLocal();
};
/**
 * Cartesian join with another DataFrame. Note that cartesian joins are very expensive without an extra filter that can be pushed down.
 * @param {DataFrame} Right side of the join operation.
 * @param {string | string[] | Column} columnNamesOrJoinExpr Optional: If string or array of strings column names, inner equi-join with another DataFrame using the given columns. 
 * Different from other join functions, the join columns will only appear once in the output, i.e. similar to SQL's JOIN USING syntax.
 * If Column object, joinExprs inner join with another DataFrame, using the given join expression.
 * @param {string} joinType Optional, only valid if using Column joinExprs. 
 * @returns {DataFrame} 
 * @example
 * var joinedDf = df1.join(df2);
 * // or
 * var joinedDf = df1.join(df2,"age");
 * // or
 * var joinedDf = df1.join(df2, ["age", "DOB"]);
 * // or Column joinExpr
 * var joinedDf = df1.join(df2, df1.col("name").equalTo(df2.col("name")));
 * // or Column joinExpr
 * var joinedDf = df1.join(df2, df1.col("name").equalTo(df2.col("name")), "outer");
 */
DataFrame.prototype.join = function(right, usingColumns, joinType) {
	var result;
	if (usingColumns) {
		if (Array.isArray(usingColumns)) {
			var scalaSeq = org.eclairjs.nashorn.Utils.toScalaSeq(usingColumns);
			result = this.getJavaObject().join(Utils.unwrapObject(right), scalaSeq);
		} else if (usingColumns instanceof Column) {
			var jType = !joinType ? "inner" : joinType;
			result = this.getJavaObject().join(Utils.unwrapObject(right), Utils.unwrapObject(usingColumns), jType);
		} else {
			result = this.getJavaObject().join(Utils.unwrapObject(right), usingColumns);
		}
		
	} else {
		result = this.getJavaObject().join(Utils.unwrapObject(right));
	}
	return new DataFrame(result);
	
};
/**
 * Returns a new DataFrame by taking the first n rows. The difference between this function and head is that head 
 * returns an array while limit returns a new DataFrame.
 * @param {integer} number
 * @returns {DataFrame}
 */
DataFrame.prototype.limit = function(number) {

	return new DataFrame(this.getJavaObject().limit(number));
};
/**
 * Returns a new RDD by applying a function to all rows of this DataFrame.
 * @param {function} func
 * @returns {RDD}
 */
DataFrame.prototype.map = function(func) {
 	return this.toRDD().map(func);
};
/**
 * Return a new RDD by applying a function to each partition of this DataFrame. 
 * Similar to map, but runs separately on each partition (block) of the DataFrame, so func must accept an Array.  
 * func should return a array rather than a single item.
 * @param {function}  
 * @returns {RDD}
 */
DataFrame.prototype.mapPartitions = function(func) {
 	return this.toRDD().mapPartitions(func);
};
/**
 * Returns a DataFrameNaFunctions for working with missing data.
 * @returns {DataFrameNaFunctions}
 
 */
DataFrame.prototype.na = function() {
 	return new DataFrameNaFunctions(this.getJavaObject().na());
};
/**
 * Returns a new DataFrame sorted by the specified columns, if columnName is used sorted in ascending order.
 * This is an alias of the sort function.
 * @param {string | Column} columnName,...columnName or sortExprs,... sortExprs
 * @returns {DataFrame}
 */
DataFrame.prototype.orderBy = function() {
 	return this.sort.apply(this, arguments);
};
/**
 * @param {StorageLevel} newLevel
 * @returns {DataFrame}
 */
DataFrame.prototype.persist = function(newLevel) {
	var arg = newLevel ? Utils.unwrapObject(newLevel) : null;
 	return new DataFrame(this.getJavaObject().persist(arg));
};
/**
 * Prints the schema to the console in a nice tree format.
 */
DataFrame.prototype.printSchema = function() {
 	this.getJavaObject().printSchema();
};
/**
 * @returns {SQLContextQueryExecution}
 */
DataFrame.prototype.queryExecution = function() {
 	return new SQLContext.QueryExecution(this.getJavaObject().queryExecution());
};
/**
 * Randomly splits this DataFrame with the provided weights.
 * @param {float[]} weights - weights for splits, will be normalized if they don't sum to 1.
 * @param {int} seed - Seed for sampling.
 * @returns {DataFrame[]}
 */
DataFrame.prototype.randomSplit = function(weights, seed) {
	var dfs = this.getJavaObject().randomSplit(weights, seed);
	var retDfs = [];
	for (var i = 0; i &lt; dfs.length; i++) {
		retDfs.push(new DataFrame(dfs[i]));
	}
 	return retDfs;
};
/**
 * Represents the content of the DataFrame as an RDD of Rows.
 * @returns {RDD}
 */
DataFrame.prototype.rdd = function() {
	return this.toRDD();
};
/**
 * Registers this DataFrame as a temporary table using the given name.
 * @param {string} tableName
 */
DataFrame.prototype.registerTempTable = function(tableName) {
    this.getJavaObject().registerTempTable(tableName);
};
/**
 * Returns a new DataFrame that has exactly numPartitions partitions.
 * @param {integer} numPartitions
 * @returns {DataFrame} 
 */
DataFrame.prototype.repartition = function(numPartitions) {
    return new DataFrame(this.getJavaObject().repartition(numPartitions));
};
/**
 * Create a multi-dimensional rollup for the current DataFrame using the specified columns,
 * so we can run aggregation on them. See GroupedData for all the available aggregate functions.
 * @param {string | Column} columnName, .....columnName or sortExprs,... sortExprs
 * @returns {GroupedData}
 * @example
 *  var result = peopleDataFrame.rollup("age", "networth").count();
 *  // or 
 *  var col = peopleDataFrame.col("age");
 *	var result = peopleDataFrame.rollup(col).count();
 */
DataFrame.prototype.rollup = function() {
	
	var columns = [];
	for (var i = 0; i &lt; arguments.length; i++) {
		var o = arguments[i];
		if (typeof o === 'string' || o instanceof String) {
			o = this.col(o);	
		} 
		columns.push(Utils.unwrapObject(o));
	}

 	return new GroupedData(this.getJavaObject().rollup(columns));
};
/**
 * Returns a new DataFrame by sampling a fraction of rows, using a random seed.
 * @param {boolean} withReplacement
 * @param {float} fraction
 * @param {integer} seed Optional
 * @returns {DataFrame}
 */
DataFrame.prototype.sample = function(withReplacement, fraction, seed) {
	return new DataFrame(this.getJavaObject().sample(withReplacement, fraction, seed));
};
/**
 * Returns the schema of this DataFrame.
 * @returns {StructType}
 */
DataFrame.prototype.schema = function() {
	return new StructType(this.getJavaObject().schema());
};
/**
 * Selects a set of column based expressions.
 * @param {Column[] | string[]}
 * @returns  {DataFrame}
 */
DataFrame.prototype.select = function() {
    var args = Array.prototype.slice.call(arguments);

    if(typeof args[0] === 'object') 
        return this.selectWithColumns(args);
    else 
        return this.selectWithStrings(args);
};
/**
 * Selects a set of SQL expressions. This is a variant of select that accepts SQL expressions.
 * @param {string} exprs,...exprs 
 * @returns {DataFrame}
 * @example
 * var result = peopleDataFrame.selectExpr("name", "age > 19");
 */
DataFrame.prototype.selectExpr = function() {
	var args = Array.prototype.slice.call(arguments);
	return new DataFrame(this.getJavaObject().selectExpr(args));
};

/**
 * Selects a set of column based expressions.
 * @param {Column[]}
 * @returns {DataFrame}
 * @private
 */
DataFrame.prototype.selectWithColumns = function(args) {
	var jCols = args.map(function(v) {
		return Utils.unwrapObject(v);
	});
	var jdf = this.getJavaObject().select(jCols);
	var df = new DataFrame(jdf);
	return df;
};
/**
 * Selects a set of column based expressions.
 * @param {string[]}
 * @returns {DataFrame}
 * @private
 */
DataFrame.prototype.selectWithStrings = function(args) {
	var jCols = args.map(function(v) {
		return this.col(v);
    }.bind(this));
	
	return this.selectWithColumns(jCols);

};
/**
 * Displays the DataFrame rows in a tabular form.
 * @param {interger} numberOfRows Optional, defaults to 20.
 * @param {boolean] truncate Optional defaults to false, Whether truncate long strings. If true, strings more than 20 characters will be 
 * truncated and all cells will be aligned right
 */
DataFrame.prototype.show = function(numberOfRows, truncate) {
	var numRow = numberOfRows ? numberOfRows : 20;
	var trunk = truncate ? true : false;
    this.getJavaObject().show(numRow, trunk);
};
/**
 * Returns a new DataFrame sorted by the specified columns, if columnName is used sorted in ascending order.
 * @param {string | Column} columnName,...columnName or sortExprs,... sortExprs
 * @returns {DataFrame}
 * @example
 *  var result = peopleDataFrame.sort("age", "name");
 *  // or 
 *  var col = peopleDataFrame.col("age");
 *	var colExpr = col.desc();
 *	var result = peopleDataFrame.sort(colExpr);
 */
DataFrame.prototype.sort = function() {
	
	var sortExprs = [];
	for (var i = 0; i &lt; arguments.length; i++) {
		var o = arguments[i];
		if (typeof o === 'string' || o instanceof String) {
			o = this.col(o).asc();		
		} 
		sortExprs.push(Utils.unwrapObject(o));
	}

 	return new DataFrame(this.getJavaObject().sort(Utils.unwrapObject(sortExprs)));
};
/**
 * Returns SQLContext
 * @returns {SQLContext}
 */
DataFrame.prototype.sqlContext = function() {
	return new SQLContext(this.getJavaObject().sqlContext());
};
/**
 * Returns a DataFrameStatFunctions for working statistic functions support.
 * @example
 * var stat = peopleDataFrame.stat().cov("income", "networth");
 * 
 * @returns {DataFrameStatFunctions}
 */
DataFrame.prototype.stat = function() {
	return new DataFrameStatFunctions(this.getJavaObject().stat());
};
/**
 * Returns the first n rows in the DataFrame.
 * @param {integer} num
 * @returns {Row[]}
 */
DataFrame.prototype.take = function(num) {
    var rows = this.getJavaObject().take(num);
    var r = [];
    for (var i = 0; i &lt; rows.length; i++) {
    	r.push(new Row(rows[i]));
    }
    return r;
};
/**
 * Returns a new DataFrame with columns renamed. This can be quite convenient in conversion from a 
 * RDD of tuples into a DataFrame with meaningful names. For example:
 * @param {string} colNames,...colNames 
 * @return {DataFrame}
 * @example 
 * var result = nameAgeDF.toDF("newName", "newAge");
 */
DataFrame.prototype.toDF = function() {
	var args = Array.prototype.slice.call(arguments);
	return new DataFrame(this.getJavaObject().toDF(args));
};
/**
 * Returns the content of the DataFrame as a RDD of JSON strings.
 * @returns {RDD}
 */
DataFrame.prototype.toJSON = function() {
    return new RDD(this.getJavaObject().toJSON());
};
/**
 * Represents the content of the DataFrame as an RDD of Rows.
 * @returns {RDD}
 */
DataFrame.prototype.toRDD = function() {
    return new RDD(this.getJavaObject().javaRDD());
};
/**
 * Returns a new DataFrame containing union of rows in this frame and another frame. This is equivalent to UNION ALL in SQL.
 * @param {DataFrame} other
 * @returns {DataFrame}
 */
DataFrame.prototype.unionAll = function(other) {
    return new DataFrame(this.getJavaObject().unionAll(Utils.unwrapObject(other)));
};
/**
 * @param {boolean} blocking
 */
DataFrame.prototype.unpersist = function(blocking) {
    this.getJavaObject().unpersist(blocking);
};
/**
 * Filters rows using the given Column or SQL expression.
 * @param {Column | string} condition - .
 * @returns {DataFrame}
 */
DataFrame.prototype.where = function(condition) {
    return this.filter(condition);
};
/**
 * Returns a new DataFrame by adding a column or replacing the existing column that has the same name.
 * @param {string} name
 * @param {Column} col
 * @returns {DataFrame}
 * @example
 *  var col = peopleDataFrame.col("age");
 *  var df1 = peopleDataFrame.withColumn("newCol", col);
 */
DataFrame.prototype.withColumn = function(name, col) {
    return new DataFrame(this.getJavaObject().withColumn(name, Utils.unwrapObject(col)));
};
/**
 * Returns a new DataFrame with a column renamed. This is a no-op if schema doesn't contain existingName.
 * @param {string} existingName
 * @param {string} newName
 * @returns {DataFrame} 
 */
DataFrame.prototype.withColumnRenamed = function(existingName, newName) {
    return new DataFrame(this.getJavaObject().withColumnRenamed(existingName, newName));
};
/**
 * Interface for saving the content of the DataFrame out into external storage. 
 * @returns {DataFrameWriter}
 */
DataFrame.prototype.write = function() {
    return new DataFrameWriter(this.getJavaObject().write());
};
</code></pre>
        </article>
    </section>




</div>

<nav>
    <h2><a href="index.html">Index</a></h2><h3>Classes</h3><ul><li><a href="Accumulable.html">Accumulable</a></li><li><a href="AccumulableParam.html">AccumulableParam</a></li><li><a href="Accumulator.html">Accumulator</a></li><li><a href="ALS.html">ALS</a></li><li><a href="ArrayType.html">ArrayType</a></li><li><a href="AssociationRules.html">AssociationRules</a></li><li><a href="BinaryType.html">BinaryType</a></li><li><a href="BisectingKMeans.html">BisectingKMeans</a></li><li><a href="BisectingKMeansModel.html">BisectingKMeansModel</a></li><li><a href="BooleanType.html">BooleanType</a></li><li><a href="CalendarIntervalType.html">CalendarIntervalType</a></li><li><a href="Column.html">Column</a></li><li><a href="DataFrame.html">DataFrame</a></li><li><a href="DataFrameNaFunctions.html">DataFrameNaFunctions</a></li><li><a href="DataFrameReader.html">DataFrameReader</a></li><li><a href="DataFrameStatFunctions.html">DataFrameStatFunctions</a></li><li><a href="DataFrameWriter.html">DataFrameWriter</a></li><li><a href="DataType.html">DataType</a></li><li><a href="DataTypes.html">DataTypes</a></li><li><a href="DateType.html">DateType</a></li><li><a href="DenseVector.html">DenseVector</a></li><li><a href="DoubleType.html">DoubleType</a></li><li><a href="DStream.html">DStream</a></li><li><a href="Duration.html">Duration</a></li><li><a href="FloatAccumulatorParam.html">FloatAccumulatorParam</a></li><li><a href="FloatType.html">FloatType</a></li><li><a href="FPGrowth.html">FPGrowth</a></li><li><a href="FPGrowthModel.html">FPGrowthModel</a></li><li><a href="FreqItemset.html">FreqItemset</a></li><li><a href="functions.html">functions</a></li><li><a href="FutureAction.html">FutureAction</a></li><li><a href="GroupedData.html">GroupedData</a></li><li><a href="HashPartitioner.html">HashPartitioner</a></li><li><a href="IntAccumulatorParam.html">IntAccumulatorParam</a></li><li><a href="IntegerType.html">IntegerType</a></li><li><a href="LabeledPoint.html">LabeledPoint</a></li><li><a href="LinearRegressionModel.html">LinearRegressionModel</a></li><li><a href="LinearRegressionWithSGD.html">LinearRegressionWithSGD</a></li><li><a href="List.html">List</a></li><li><a href="MapType.html">MapType</a></li><li><a href="MatrixFactorizationModel.html">MatrixFactorizationModel</a></li><li><a href="Metadata.html">Metadata</a></li><li><a href="MLWord2Vec.html">MLWord2Vec</a></li><li><a href="MLWord2VecModel.html">MLWord2VecModel</a></li><li><a href="NullType.html">NullType</a></li><li><a href="NumericType.html">NumericType</a></li><li><a href="PartialResult.html">PartialResult</a></li><li><a href="Partitioner.html">Partitioner</a></li><li><a href="RangePartitioner.html">RangePartitioner</a></li><li><a href="Rating.html">Rating</a></li><li><a href="RDD.html">RDD</a></li><li><a href="Row.html">Row</a></li><li><a href="RowFactory.html">RowFactory</a></li><li><a href="Rule.html">Rule</a></li><li><a href="SparkConf.html">SparkConf</a></li><li><a href="SparkContext.html">SparkContext</a></li><li><a href="SparkFiles.html">SparkFiles</a></li><li><a href="SparkStatusTracker.html">SparkStatusTracker</a></li><li><a href="SparseVector.html">SparseVector</a></li><li><a href="SQLContext.html">SQLContext</a></li><li><a href="SQLContext.QueryExecution.html">QueryExecution</a></li><li><a href="SQLContext.SparkPlanner.html">SparkPlanner</a></li><li><a href="SQLContext.SQLSession.html">SQLSession</a></li><li><a href="SqlDate.html">SqlDate</a></li><li><a href="SqlTimestamp.html">SqlTimestamp</a></li><li><a href="StorageLevel.html">StorageLevel</a></li><li><a href="StreamingContext.html">StreamingContext</a></li><li><a href="StringType.html">StringType</a></li><li><a href="StructField.html">StructField</a></li><li><a href="StructType.html">StructType</a></li><li><a href="Time.html">Time</a></li><li><a href="TimestampType.html">TimestampType</a></li><li><a href="Vectors.html">Vectors</a></li><li><a href="VectorUDT.html">VectorUDT</a></li><li><a href="Word2Vec.html">Word2Vec</a></li><li><a href="Word2VecModel.html">Word2VecModel</a></li></ul><h3>Global</h3><ul><li><a href="global.html#Vector">Vector</a></li></ul>
</nav>

<br clear="both">

<footer>
    Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 3.2.3-dev</a> on Thu Feb 18 2016 17:00:52 GMT-0500 (EST)
</footer>

<script> prettyPrint(); </script>
<script src="scripts/linenumber.js"> </script>
</body>
</html>
<file_sep>/*                                                                         
 * Copyright 2015 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


/**
 * :: Experimental ::
 * Word2Vec trains a model of `Map(String, Vector)`, i.e. transforms a word into a code for further
 * natural language processing or machine learning process.
 * @classdesc
 * @param {string}
 *  @class
 */


MLWord2Vec = function (obj) {
    this.logger = Logger.getLogger("Word2Vec_js");
    var jvmObject;
    if (obj instanceof org.apache.spark.ml.feature.Word2Vec) {
        jvmObject = obj;
    } else if ((typeof obj) == 'string') {
        jvmObject = new org.apache.spark.ml.feature.Word2Vec(obj);
    } else {
        jvmObject = new org.apache.spark.ml.feature.Word2Vec();
    }

    JavaWrapper.call(this, jvmObject);

};

MLWord2Vec.prototype = Object.create(JavaWrapper.prototype);

MLWord2Vec.prototype.constructor = MLWord2Vec;


/**
 * @param {string}
 * @returns {MLWord2Vec}
 */
MLWord2Vec.prototype.setInputCol = function (value) {
    return new MLWord2Vec(this.getJavaObject().setInputCol(value));
}


/**
 * @param {string}
 * @returns {MLWord2Vec}
 */
MLWord2Vec.prototype.setOutputCol = function (value) {
    return new MLWord2Vec(this.getJavaObject().setOutputCol(value));
}


/**
 * @param {integer}
 * @returns {MLWord2Vec}
 */
MLWord2Vec.prototype.setVectorSize = function (value) {
    return new MLWord2Vec(this.getJavaObject().setVectorSize(value));
}


/**
 * @param {integer} value
 * @returns {MLWord2Vec}
 */
MLWord2Vec.prototype.setWindowSize = function (value) {
    return  new MLWord2Vec(this.getJavaObject().setWindowSize(value));
}


/**
 * @param {float} value
 * @returns {MLWord2Vec}
 */
MLWord2Vec.prototype.setStepSize = function (value) {
    return new MLWord2Vec(this.getJavaObject().setStepSize(value));
}


/**
 * @param {integer} value
 * @returns {MLWord2Vec}
 */
MLWord2Vec.prototype.setNumPartitions = function (value) {
    return  new MLWord2Vec(this.getJavaObject().setNumPartitions(value));
}


/**
 * @param {integer} value
 * @returns {MLWord2Vec}
 */
MLWord2Vec.prototype.setMaxIter = function (value) {
    return  new MLWord2Vec(this.getJavaObject().setMaxIter(value));
}


/**
 * @param {integer} value
 * @returns {MLWord2Vec}
 */
MLWord2Vec.prototype.setSeed = function (value) {
    return  new MLWord2Vec(this.getJavaObject().setSeed(value));
}


/**
 * @param {integer} value
 * @returns {MLWord2Vec}
 */
MLWord2Vec.prototype.setMinCount = function (value) {
    return new MLWord2Vec(this.getJavaObject().setMinCount(value));
}


/**
 * @param {DataFrame} dataset
 * @returns {MLWord2VecModel}
 */
MLWord2Vec.prototype.fit = function (dataset) {
    var dataset_uw = Utils.unwrapObject(dataset);
    return new MLWord2VecModel(this.getJavaObject().fit(dataset_uw));
}


/**
 * @param {StructType} schema
 * @returns {StructType}
 */
MLWord2Vec.prototype.transformSchema = function (schema) {
    var schema_uw = Utils.unwrapObject(schema);
    return  new StructType(this.getJavaObject().transformSchema(schema_uw));
}


/**
 * @param {ParamMap}
 * @returns {MLWord2Vec}
 * @private
 */
MLWord2Vec.prototype.copy = function (extra) {
    throw "not implemented by ElairJS";
//   var extra_uw = Utils.unwrapObject(extra);
//   return  this.getJavaObject().copy(extra_uw);
}


/**
 * :: Experimental ::
 * Model fitted by {@link MLWord2Vec}.
 * @classdesc
 * @constructor
 */


MLWord2VecModel = function (jvmObject) {

    this.logger = Logger.getLogger("MLWord2VecModel_js");
    JavaWrapper.call(this, jvmObject);

};

MLWord2VecModel.prototype = Object.create(JavaWrapper.prototype);

MLWord2VecModel.prototype.constructor = MLWord2VecModel;


/**
 * Find "num" number of words closest in similarity to the given word.
 * Returns a dataframe with the words and the cosine similarities between the
 * synonyms and the given word.
 * @param {string}
 * @param {number}
 * @returns {DataFrame}
 * @private
 *
 */
MLWord2VecModel.prototype.findSynonymswithnumber = function (word, num) {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().findSynonyms(word,num);
}


/**
 * Find "num" number of words closest to similarity to the given vector representation
 * of the word. Returns a dataframe with the words and the cosine similarities between the
 * synonyms and the given word vector.
 * @param {Vector}
 * @param {number}
 * @returns {DataFrame}
 * @private
 *
 */
MLWord2VecModel.prototype.findSynonymswithnumber = function (word, num) {
    throw "not implemented by ElairJS";
//   var word_uw = Utils.unwrapObject(word);
//   return  this.getJavaObject().findSynonyms(word_uw,num);
}


/**
 * @param {string}
 * @returns {string}
 *  @private
 *
 */
MLWord2VecModel.prototype.setInputCol = function (value) {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().setInputCol(value);
}


/**
 * @param {string}
 * @returns {string}
 *  @private
 *
 */
MLWord2VecModel.prototype.setOutputCol = function (value) {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().setOutputCol(value);
}


/**
 * Transform a sentence column to a vector column to represent the whole sentence. The transform
 * is performed by averaging all word vectors it contains.
 * @param {DataFrame}
 * @returns {DataFrame}
 *  @private
 *
 */
MLWord2VecModel.prototype.transform = function (dataset) {
    var dataset_uw = Utils.unwrapObject(dataset);
    return new DataFrame(this.getJavaObject().transform(dataset_uw));
}


/**
 * @param {StructType}
 * @returns {StructType}
 *  @private
 *
 */
MLWord2VecModel.prototype.transformSchema = function (schema) {
    throw "not implemented by ElairJS";
//   var schema_uw = Utils.unwrapObject(schema);
//   return  this.getJavaObject().transformSchema(schema_uw);
}


/**
 * @param {ParamMap}
 * @returns {MLWord2VecModel}
 *  @private
 *
 */
MLWord2VecModel.prototype.copy = function (extra) {
    throw "not implemented by ElairJS";
//   var extra_uw = Utils.unwrapObject(extra);
//   return  this.getJavaObject().copy(extra_uw);
}


/**
 * @returns {MLWriter}
 *  @private
 *
 */
MLWord2VecModel.prototype.write = function () {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().write();
}
//
// static methods
//


/**
 * @param {string}
 * @returns {MLWord2Vec}
 *  @private
 *
 */
MLWord2Vec.load = function (path) {
    throw "not implemented by ElairJS";
//   return  org.apache.spark.ml.feature.MLWord2Vec.load(path);
}


/**
 * @returns {MLReader}
 *  @private
 *
 */
MLWord2VecModel.read = function () {
    throw "not implemented by ElairJS";
//   return  org.apache.spark.ml.feature.MLWord2VecModel.read();
}


/**
 * @param {string}
 * @returns {MLWord2VecModel}
 *  @private
 *
 */
MLWord2VecModel.load = function (path) {
    throw "not implemented by ElairJS";
//   return  org.apache.spark.ml.feature.Word2VecModel.load(path);
}
<file_sep>package org.eclairjs.nashorn;

import jdk.nashorn.api.scripting.ScriptObjectMirror;
import org.apache.commons.lang.ArrayUtils;
import org.apache.spark.api.java.function.PairFlatMapFunction;
import scala.Tuple2;

import javax.script.Invocable;
import javax.script.ScriptEngine;
import java.util.ArrayList;

public class JSPairFlatMapFunction implements PairFlatMapFunction {

    private String func = null;
    private Object args[] = null;
    private String functionName = null;

    public JSPairFlatMapFunction(String func,  Object[] o) {
        this.functionName = Utils.getUniqeFunctionName();
        this.func = "var " + this.functionName +" = " + func;
        this.args = o;
    }

    @Override
    public Iterable<Tuple2> call(Object o) throws Exception {

        ScriptEngine e =  NashornEngineSingleton.getEngine();

        e.eval(this.func);
        Invocable invocable = (Invocable) e;
        Object arg0 = Utils.javaToJs(o, e);
        Object params[] = {arg0};

        params = ArrayUtils.addAll(params, this.args);
        ScriptObjectMirror ret = (ScriptObjectMirror)invocable.invokeFunction(this.functionName, params);

        ArrayList l = new ArrayList(ret.values());
        ArrayList<Tuple2> l2 = new ArrayList<Tuple2>(ret.size());

        for(Object t : l) {
            ArrayList al = new ArrayList(((ScriptObjectMirror)t).values());
            Object t1 = Utils.jsToJava(al.get(0));
            Object t2 = Utils.jsToJava(al.get(1));
            Tuple2 tuple = new Tuple2(t1, t2);

            l2.add(tuple);
        }

        return l2;
    }
}
<file_sep><!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="utf-8">
    <title>JSDoc: Source: streaming/StreamingContext.js</title>
    
    <script src="scripts/prettify/prettify.js"> </script>
    <script src="scripts/prettify/lang-css.js"> </script>
    <!--[if lt IE 9]>
      <script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
    <![endif]-->
    <link type="text/css" rel="stylesheet" href="styles/prettify-tomorrow.css">
    <link type="text/css" rel="stylesheet" href="styles/jsdoc-default.css">
</head>

<body>

<div id="main">
    
    <h1 class="page-title">Source: streaming/StreamingContext.js</h1>
    
    


    
    <section>
        <article>
            <pre class="prettyprint source"><code>/*
 * Copyright 2015 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

var JavaStreamingContext = 
    Java.type('org.apache.spark.streaming.api.java.JavaStreamingContext');
/**
 * @constructor
 * @classdesc Main entry point for Spark Streaming functionality. It provides methods used to create DStreams from various input sources.
 *  It can be either created by providing a Spark master URL and an appName, or from a org.apache.spark.SparkConf configuration 
 *  (see core Spark documentation), or from an existing org.apache.spark.SparkContext. The associated SparkContext can be accessed 
 *  using context.sparkContext. After creating and transforming DStreams, the streaming computation can be started and stopped using 
 *  context.start() and context.stop(), respectively. context.awaitTermination() allows the current thread to wait for the termination 
 *  of the context by stop() or by an exception.
 *  @param {SparkContex} sparkContext
 *  @param {Duration} duration
 */
var StreamingContext = function(sparkContext, duration) {
	var jvmObj = 
        new JavaStreamingContext(Utils.unwrapObject(sparkContext), 
    							 Utils.unwrapObject(duration)
    							 );
	this.logger = Logger.getLogger("streaming.Duration_js");
	JavaWrapper.call(this, jvmObj);
};

StreamingContext.prototype = Object.create(JavaWrapper.prototype); 

StreamingContext.prototype.constructor = StreamingContext;
/**
 * Wait for the execution to stop
 */
StreamingContext.prototype.awaitTermination = function() {
    this.getJavaObject().awaitTermination();
};
/**
 * Wait for the execution to stop, or timeout
 * @param {long} millis
 * @returns {boolean}
 */
StreamingContext.prototype.awaitTerminationOrTimeout = function(millis) {
    return this.getJavaObject().awaitTerminationOrTimeout(millis);
};
/**
 * Start the execution of the streams.
 */
StreamingContext.prototype.start = function() {
    this.getJavaObject().start();
};
/**
 * Stops the execution of the streams.
 */
StreamingContext.prototype.stop = function() {
    if(arguments.length == 1) {
        this.getJavaObject().stop(arguments[0]);
    } else {
        this.getJavaObject().stop();
    }
};

StreamingContext.prototype.close = function() {
    this.getJavaObject().close();
};

/**
 * Create a input stream from TCP source hostname:port.
 * @param {string} host
 * @param {string} port
 * @returns {DStream}
 */
StreamingContext.prototype.socketTextStream = function(host, port) {
    var jDStream = this.getJavaObject().socketTextStream(host, port);

    return new DStream(jDStream, this);
};




</code></pre>
        </article>
    </section>




</div>

<nav>
    <h2><a href="index.html">Index</a></h2><h3>Classes</h3><ul><li><a href="Accumulable.html">Accumulable</a></li><li><a href="AccumulableParam.html">AccumulableParam</a></li><li><a href="Accumulator.html">Accumulator</a></li><li><a href="ALS.html">ALS</a></li><li><a href="ArrayType.html">ArrayType</a></li><li><a href="AssociationRules.html">AssociationRules</a></li><li><a href="BinaryType.html">BinaryType</a></li><li><a href="BisectingKMeans.html">BisectingKMeans</a></li><li><a href="BisectingKMeansModel.html">BisectingKMeansModel</a></li><li><a href="BooleanType.html">BooleanType</a></li><li><a href="CalendarIntervalType.html">CalendarIntervalType</a></li><li><a href="Column.html">Column</a></li><li><a href="DataFrame.html">DataFrame</a></li><li><a href="DataFrameNaFunctions.html">DataFrameNaFunctions</a></li><li><a href="DataFrameReader.html">DataFrameReader</a></li><li><a href="DataFrameStatFunctions.html">DataFrameStatFunctions</a></li><li><a href="DataFrameWriter.html">DataFrameWriter</a></li><li><a href="DataType.html">DataType</a></li><li><a href="DataTypes.html">DataTypes</a></li><li><a href="DateType.html">DateType</a></li><li><a href="DenseVector.html">DenseVector</a></li><li><a href="DoubleType.html">DoubleType</a></li><li><a href="DStream.html">DStream</a></li><li><a href="Duration.html">Duration</a></li><li><a href="FloatAccumulatorParam.html">FloatAccumulatorParam</a></li><li><a href="FloatType.html">FloatType</a></li><li><a href="FPGrowth.html">FPGrowth</a></li><li><a href="FPGrowthModel.html">FPGrowthModel</a></li><li><a href="FreqItemset.html">FreqItemset</a></li><li><a href="functions.html">functions</a></li><li><a href="FutureAction.html">FutureAction</a></li><li><a href="GroupedData.html">GroupedData</a></li><li><a href="HashPartitioner.html">HashPartitioner</a></li><li><a href="IntAccumulatorParam.html">IntAccumulatorParam</a></li><li><a href="IntegerType.html">IntegerType</a></li><li><a href="LabeledPoint.html">LabeledPoint</a></li><li><a href="LinearRegressionModel.html">LinearRegressionModel</a></li><li><a href="LinearRegressionWithSGD.html">LinearRegressionWithSGD</a></li><li><a href="List.html">List</a></li><li><a href="MapType.html">MapType</a></li><li><a href="MatrixFactorizationModel.html">MatrixFactorizationModel</a></li><li><a href="Metadata.html">Metadata</a></li><li><a href="MLWord2Vec.html">MLWord2Vec</a></li><li><a href="MLWord2VecModel.html">MLWord2VecModel</a></li><li><a href="NullType.html">NullType</a></li><li><a href="NumericType.html">NumericType</a></li><li><a href="PartialResult.html">PartialResult</a></li><li><a href="Partitioner.html">Partitioner</a></li><li><a href="RangePartitioner.html">RangePartitioner</a></li><li><a href="Rating.html">Rating</a></li><li><a href="RDD.html">RDD</a></li><li><a href="Row.html">Row</a></li><li><a href="RowFactory.html">RowFactory</a></li><li><a href="Rule.html">Rule</a></li><li><a href="SparkConf.html">SparkConf</a></li><li><a href="SparkContext.html">SparkContext</a></li><li><a href="SparkFiles.html">SparkFiles</a></li><li><a href="SparkStatusTracker.html">SparkStatusTracker</a></li><li><a href="SparseVector.html">SparseVector</a></li><li><a href="SQLContext.html">SQLContext</a></li><li><a href="SQLContext.QueryExecution.html">QueryExecution</a></li><li><a href="SQLContext.SparkPlanner.html">SparkPlanner</a></li><li><a href="SQLContext.SQLSession.html">SQLSession</a></li><li><a href="SqlDate.html">SqlDate</a></li><li><a href="SqlTimestamp.html">SqlTimestamp</a></li><li><a href="StorageLevel.html">StorageLevel</a></li><li><a href="StreamingContext.html">StreamingContext</a></li><li><a href="StringType.html">StringType</a></li><li><a href="StructField.html">StructField</a></li><li><a href="StructType.html">StructType</a></li><li><a href="Time.html">Time</a></li><li><a href="TimestampType.html">TimestampType</a></li><li><a href="Vectors.html">Vectors</a></li><li><a href="VectorUDT.html">VectorUDT</a></li><li><a href="Word2Vec.html">Word2Vec</a></li><li><a href="Word2VecModel.html">Word2VecModel</a></li></ul><h3>Global</h3><ul><li><a href="global.html#Vector">Vector</a></li></ul>
</nav>

<br clear="both">

<footer>
    Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 3.2.3-dev</a> on Thu Feb 18 2016 17:00:52 GMT-0500 (EST)
</footer>

<script> prettyPrint(); </script>
<script src="scripts/linenumber.js"> </script>
</body>
</html>
<file_sep>/*
 * Copyright 2015 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

var sparkConf = new SparkConf()
  .setAppName("Binary Classification Metrics Test")
  .setMaster("local[*]");

var sc = new SparkContext(sparkConf);
var data = MLUtils.loadLibSVMFile(sc, "examples/data/mllib/sample_binary_classification_data.txt");

//Split data into training (60%) and test (40%)
var split = data.randomSplit([0.6, 0.4], 11)
var training = split[0].cache();
var test = split[1];

var model = new LogisticRegressionWithLBFGS()
    .setNumClasses(2)
    .run(training);

var predictionAndLabels = test.mapToPair(function(lp, model) {
    return [model.predict(lp.getFeatures()),
            lp.getLabel()];
});

var metrics = new BinaryClassificationMetrics(predictionAndLabels);

// Precision by threshold
var precision = metrics.precisionByThreshold();
print("Precision by threshold: " + precision.collect());

// Recall by threshold
var recall = metrics.recallByThreshold();
print("Recall by threshold: " + recall.collect());

// F Score by threshold
var f1Score = metrics.fMeasureByThreshold();
print("F1 Score by threshold: " + f1Score.collect());

var f2Score = metrics.fMeasureByThreshold(2.0);
print("F2 Score by threshold: " + f2Score.collect());

// Precision-recall curve
var prc = metrics.pr();
print("Precision-recall curve: " + prc.collect());
<file_sep>/*
 * Copyright 2015 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.eclairjs.nashorn;

import jdk.nashorn.api.scripting.JSObject;
import jdk.nashorn.api.scripting.ScriptObjectMirror;
import javax.script.Invocable;
import javax.script.ScriptEngine;
import javax.script.ScriptException;

import org.apache.log4j.Logger;
import org.apache.spark.SparkFiles;
import org.apache.spark.mllib.regression.LinearRegressionModel;
import org.apache.spark.mllib.regression.LabeledPoint;
import org.apache.spark.rdd.RDD;
import org.apache.spark.sql.Row;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import scala.Tuple2;
import scala.collection.Seq;
import scala.collection.convert.Wrappers.IteratorWrapper;
import scala.collection.convert.Wrappers.IterableWrapper;

import java.io.FileReader;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.util.*;

/**
 * Created by bburns on 9/18/15.
 */
public class Utils {

    /*
    public static Object javaToJs(Object o) {
        if(o instanceof Tuple2) {
            Tuple2 t = (Tuple2)o;
            ArrayList l = new ArrayList();
            l.add(t._1());
            l.add(t._2());
            return l.toArray();
        } else
            return o;
    }
    */
    @SuppressWarnings({ "rawtypes", "unchecked" })
	public static Object javaToJs(Object o, ScriptEngine engine) {
    	Logger logger = Logger.getLogger(Utils.class);
    	String packageName = null;
 		if(o != null) {
			logger.debug("javaToJs className " + o.getClass().getName());
			Package pack = o.getClass().getPackage();
			if (pack != null) {
	    		packageName = pack.getName();
	    	}
		}
 		/*
 		 * Any object that belongs to Spark we will wrapper it with a JavaScript object
 		 * If we don't have a JavaScript wrapper for it then we will catch the
 		 * exception and just use the wrapObject method.
 		 */
		if ((packageName != null) && (packageName.indexOf("org.apache.spark") > -1)) {
			logger.debug("spark object");
			String className = o.getClass().getSimpleName();
    		try {
 	  			Invocable invocable = (Invocable) engine;
 	  			if (className.endsWith("$")) {
 	  				//anonymous class
 	  				logger.debug("getSuperClass for " + className);
 	  				className = o.getClass().getSuperclass().getSimpleName();
 	  			}
 	  			if ( className.equals("JavaRDD")) {
 	  				/*
 	  				 * Map JavaRDD to RDD for JavaScript
 	  				 */
 	  				className = "RDD"; //o.getClass().getSimpleName();
 	  			} else if (className.equals("Word2Vec") || className.equals("Word2VecModel")) {
					if (packageName.indexOf("org.apache.spark.ml") > -1) {
						/*
							ML
						 */
						className = "ML" + o.getClass().getSimpleName();
					}

				}
 	  			logger.debug("create " + className);
	  			Object params[] = {className, o};
	  			Object parm = invocable.invokeFunction("createJavaWrapperObject", params);
	  			logger.debug(parm);
	  			return parm;
  			} catch (Exception e) {
    			logger.warn(className + " convertion error, will just wrapObject " + e);
    			logger.debug(className + " javaToJs instanceof  " + o.getClass());
    			return wrapObject(o);
    		}

    	} else if (o instanceof Tuple2) {
            Tuple2 t = (Tuple2)o;
            logger.info("Tupple2 - " + t.toString());
            Object er = null;
            Object o1 = javaToJs(t._1(),engine);
            Object o2 = javaToJs(t._2(), engine);
            logger.debug("o1 = " + o1);
             try {
				Invocable invocable = (Invocable) engine;
				 Object params[] = {o1, o2};
				 er  = invocable.invokeFunction("convertJavaTuple2",params);
			} catch (ScriptException | NoSuchMethodException e) {
				logger.error(" Tuple2 convertion " + e);
			}
            return er;
        } else if (o instanceof IteratorWrapper) {
            logger.debug("Iterator " + o.toString());
        	ArrayList alist = new ArrayList();
        	while(((IteratorWrapper) o).hasMoreElements()) {
        		alist.add(javaToJs(((IteratorWrapper) o).nextElement(),engine));
        	}
        	return wrapObject(alist);
        } else if (o instanceof IterableWrapper) {
            logger.debug("Iterable " + o.toString());
            ArrayList alist = new ArrayList();
            Iterator iter=((IterableWrapper) o).iterator();
            while(iter.hasNext()) {
                alist.add(javaToJs(iter.next(),engine));
            }
            return wrapObject(alist);
        } else if (o instanceof JSONObject) {
        	Object er = null;
        	try {
        		logger.debug("JSONObject " + o.toString());
				Invocable invocable = (Invocable) engine;
				 Object params[] = {o.toString()};
				 er  = invocable.invokeFunction("convertJavaJSONObject",params);
			} catch (ScriptException | NoSuchMethodException e) {
				logger.error(" JSONObject convertion " + e);
			}
        	return er;
        } else {
        	logger.debug(" jsToJava wrapObject " + o);
            return wrapObject(o);
        }


    }

    public static Object jsToJava(Object o) {
    	Logger logger = Logger.getLogger(Utils.class);
		if(o != null)
			logger.debug("jsToJava" + o.getClass().getName());
    	if ( (o instanceof ScriptObjectMirror) && ((ScriptObjectMirror) o).hasMember("getJavaObject") ) {
    		Object r = ((ScriptObjectMirror) o).callMember("getJavaObject");
    		logger.debug("getJavaObject" + r.getClass().getName());
    		return r;
    	} else if(o instanceof JSObject) {
            Object obj = ScriptObjectMirror.wrapAsJSONCompatible(o, null);
            String j = JSONValue.toJSONString(obj);
            return JSONValue.parse(j);
        }

        return o;
    }

    public static String getUniqeFunctionName() {
        return "EXPORTEDFUNCTION" + java.util.UUID.randomUUID().toString().replace("-", "_");
    }

    public static ScriptEngine addScopeVarsToEngine(HashMap scopeVars, ScriptEngine engine) {

    	Logger logger = Logger.getLogger(Utils.class);
    	logger.debug("addScopeVarsToEngine");
    	if (scopeVars != null) {
        	Iterator it = scopeVars.entrySet().iterator();
            while (it.hasNext()) {
                Map.Entry pair = (Map.Entry)it.next();
                logger.debug("adding " + pair.getKey() + " value " + pair.getValue());
                engine.put((String)pair.getKey(), pair.getValue());
            }
    	}
    	return engine;
    }

    private static Object wrapObject(Object o) {
    	Logger logger = Logger.getLogger(Utils.class);
        if(o instanceof String ||
           o instanceof Number) {
            return o;
        }
        logger.debug("wrapAsJSONCompatible " + o);
        return ScriptObjectMirror.wrapAsJSONCompatible(o,null);
    }

    public static String jarLoc() {
    	Logger logger = Logger.getLogger(Utils.class);
    	String jarPath = null;
    	Map<String, String> env = System.getenv();
    	jarPath = env.get("ECLAIR_JAR_LOC");
    	if (jarPath == null) {
    		String path = Utils.class.getProtectionDomain().getCodeSource().getLocation().getPath();
    		logger.info("jar path = " + path);
            String decodedPath = null;
    		try {
    			decodedPath = URLDecoder.decode(path, "UTF-8");
    		} catch (UnsupportedEncodingException e) {
    			// TODO Auto-generated catch block
    			e.printStackTrace();
    		}
    		jarPath = decodedPath;
    	}
    	logger.info("env = "+ jarPath);
    	return jarPath;

    }

    /**
     * Takes an array of objects and returns a scala Seq
     * @param {Object[]} o
     * @return {scala.collection.Seq}
     */
    @SuppressWarnings({ "rawtypes", "unchecked" })
	public static Seq toScalaSeq(Object[] o) {
    	ArrayList list = new ArrayList();
    	for (int i = 0; i < o.length; i++) {
    		list.add(o[i]);
    	}
  		return scala.collection.JavaConversions.asScalaBuffer(list).toList();

    }


}
<file_sep>/*                                                                         
* Copyright 2016 IBM Corp.                                                 
*                                                                          
* Licensed under the Apache License, Version 2.0 (the "License");          
* you may not use this file except in compliance with the License.         
* You may obtain a copy of the License at                                  
*                                                                          
*      http://www.apache.org/licenses/LICENSE-2.0                          
*                                                                          
* Unless required by applicable law or agreed to in writing, software      
* distributed under the License is distributed on an "AS IS" BASIS,        
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
* See the License for the specific language governing permissions and      
* limitations under the License.                                           
*/ 



/**
 * :: Experimental ::
 *
 * Generates association rules from a [[RDD[FreqItemset[Item]]]. This method only generates
 * association rules which have a single item as the consequent.
 *
 * @classdesc
 */

/**
 * Constructs a default instance with default parameters {minConfidence = 0.8}.
 *  @class
 */
var AssociationRules = function() {
	 
    this.logger = Logger.getLogger("AssociationRules_js");
    var jvmObject;
	if (arguments.length < 1) {
        jvmObject = new org.apache.spark.mllib.fpm.AssociationRules();
    } else {
        jvmObject = arguments[0];
    }
	 JavaWrapper.call(this, jvmObject);

};

AssociationRules.prototype = Object.create(JavaWrapper.prototype);

AssociationRules.prototype.constructor = AssociationRules;



/**
 * Sets the minimal confidence (default: `0.8`).
 * @param {float} minConfidence
 * @returns {AssociationRules}
 */
AssociationRules.prototype.setMinConfidence = function(minConfidence) {
   var javaObject =  this.getJavaObject().setMinConfidence(minConfidence);
   return new AssociationRules(javaObject);
};


/**
 * Computes the association rules with confidence above {@link minConfidence}.
 * @param {RDD} freqItemsets  frequent itemset model obtained from {@link FPGrowth}
 *
 * @returns {RDD}  a [[Set[Rule[Item]]] containing the assocation rules.
 */
AssociationRules.prototype.run = function(freqItemsets) {
   var freqItemsets_uw = Utils.unwrapObject(freqItemsets);
   var javaObject =  this.getJavaObject().run(freqItemsets_uw);
   return new RDD(javaObject);
};


/**
 * An association rule between sets of items.
 *
 * @classdesc
 * @class
 */
var Rule = function(jvmObject) {

    this.logger = Logger.getLogger("Rule_js");
    this.logger.debug("constructor");
    JavaWrapper.call(this, jvmObject);

};

Rule.prototype = Object.create(JavaWrapper.prototype);

Rule.prototype.constructor = Rule;

/**
 * Returns antecedent
 * @returns {object}
 */
Rule.prototype.antecedent = function() {
    /*
    public java.util.List<Item> javaAntecedent()
    Returns antecedent in a Java List.
    */
   return this.getJavaObject().javaAntecedent();
};

/**
 * Returns consequent
 * @returns {object}
 */
Rule.prototype.consequent = function() {
    /*
     public java.util.List<Item> javaConsequent()
     Returns consequent in a Java List.
     */
    return this.getJavaObject().javaConsequent();
};

/**
 * Returns the confidence of the rule.
 * @returns {float}
 */
Rule.prototype.confidence = function() {
    /*
     public double confidence()
     Returns the confidence of the rule.
     */
    return this.getJavaObject().confidence();
};

/**
 *
 * @returns {string}
 */
Rule.prototype.toString = function() {

   return this.getJavaObject().toString();
};<file_sep>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

var sparkConf = new SparkConf().setAppName("JavaScriptSparkSQL").setMaster("local[*]");
var ctx = new SparkContext(sparkConf);
var sqlContext = new SQLContext(ctx);

// Load a text file and convert each line to a JavaScript Object.
var people = ctx.textFile("examples/data/people.txt").map(function(line) {
	var parts = line.split(",");
	return person = {
    				name: parts[0], 
    				age: parseInt(parts[1].trim())
    		};
});

//Generate the schema
var fields = [];
fields.push(DataTypes.createStructField("name", DataTypes.StringType, true));
fields.push(DataTypes.createStructField("age", DataTypes.IntegerType, true));
var schema = DataTypes.createStructType(fields);

// Convert records of the RDD (people) to Rows.
var rowRDD = people.map(function(person){
	return RowFactory.create([person.name, person.age]);
});


//Apply the schema to the RDD.
var peopleDataFrame = sqlContext.createDataFrame(rowRDD, schema);

// Register the DataFrame as a table.
peopleDataFrame.registerTempTable("people");

// SQL can be run over RDDs that have been registered as tables.
var results = sqlContext.sql("SELECT name FROM people");

//The results of SQL queries are DataFrames and support all the normal RDD operations.
//The columns of a row in the result can be accessed by ordinal.
var names = results.toRDD().map(function(row) {
	return "Name: " + row.getString(0);
});

print("names = " + names.take(10));
var dataFrame = sqlContext.read().json("examples/data/test.json");
var gd = dataFrame.groupBy(dataFrame.col("first"));
var df2 = gd.count();

df2.show();
df2.count();
<file_sep><!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="utf-8">
    <title>JSDoc: Source: sql/Column.js</title>
    
    <script src="scripts/prettify/prettify.js"> </script>
    <script src="scripts/prettify/lang-css.js"> </script>
    <!--[if lt IE 9]>
      <script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
    <![endif]-->
    <link type="text/css" rel="stylesheet" href="styles/prettify-tomorrow.css">
    <link type="text/css" rel="stylesheet" href="styles/jsdoc-default.css">
</head>

<body>

<div id="main">
    
    <h1 class="page-title">Source: sql/Column.js</h1>
    
    


    
    <section>
        <article>
            <pre class="prettyprint source"><code>/*
 * Copyright 2015 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/**
 * 
 * @constructor
 * @classdesc A column in a DataFrame.
 * @param {string} column name of the column
 */
var Column = function(column) {
	var jvmObj;
	if (typeof column === 'string' || column instanceof String) {
		jvmObj = new org.apache.spark.sql.Column(column);
	} else {
		jvmObj = column;
    
	}
	this.logger = Logger.getLogger("sql.Column_js");
    JavaWrapper.call(this, jvmObj);
};

Column.prototype = Object.create(JavaWrapper.prototype); 

//Set the "constructor" property to refer to Column
Column.prototype.constructor = Column;


/**
 * Gives the column an alias. Same as as.
 * @param {string} alias
 * @returns {Column}
 * @example
 * // Renames colA to colB in select output.
 * df.select(df.col("colA").alias("colB"))
 */
Column.prototype.alias = function(a) {
	return new Column(this.getJavaObject().alias(a));
}
/**
 * Boolean AND.
 * @param {Column} other
 * @returns {Column}
 * @example
 * people.select( people.col("inSchool").and(people.col("isEmployed")));
 */
Column.prototype.and = function(other) {
	return new Column(this.getJavaObject().and(Utils.unwrapObject(other)));
}
/**
 * Extracts a value or values from a complex type.
 * The following types of extraction are supported:
 * - Given an Array, an integer ordinal can be used to retrieve a single value.
 * - Given a Map, a key of the correct type can be used to retrieve an individual value.
 * - Given a Struct, a string fieldName can be used to extract that field.
 * - Given an Array of Structs, a string fieldName can be used to extract filed
 *   of every struct in that array, and return an Array of fields
 * @param {string}
 * @returns {Column} 
 * @private
 */
Column.prototype.apply = function(extraction) {
	throw "not implemented by ElairJS";
	//return new Column(this.getJavaObject().apply(extraction));
}
/**
 * Gives the column an alias. 
 * @param {string | string[]} aliases, if array of strings assigns the given aliases to the results of a table generating function.
 * @param {Metadata} metadata Optional, not valid with string array
 * @returns {Column} 
 */
Column.prototype.as = function(aliases, metadata) {
	var c;
	if (!Array.isArray(aliases) && metadata) {
		c = this.getJavaObject().as(aliases, metadata);
	} else {
		c = this.getJavaObject().as(aliases);
	}
	return new Column(c);
}
/**
 * Returns an ordering used in sorting.
 * @returns {Column}
 * @example
 * df.sort(df.col("age").asc());
 */
Column.prototype.asc = function() {
	return new Column(this.getJavaObject().asc());
};
/**
 * True if the current column is between the lower bound and upper bound, inclusive.
 * @param {object} lowerBound
 * @param {object} upperBound
 * @returns {Column}
 * @example
 * var col = new Column("age");
 * var testCol = col.between(10, 29);
 * var results = peopleDataFrame.select(testCol);
 */
Column.prototype.between = function(lowerBound, upperBound) {
	this.logger.debug("between");
	return new Column(this.getJavaObject().between(lowerBound, upperBound));
};
/**
 * Compute bitwise AND of this expression with another expression.
 * @example
 *   df.select(df.col("colA").bitwiseAND(df.col("colB")));
 *
 * @since EclairJS 0.1 Spark  1.4.0
 * @param {Column} other
 * @returns {Column} 
 */
Column.prototype.bitwiseAND = function(other) {
	var javaObject =  this.getJavaObject().bitwiseAND(Utils.unwrapObject(other));
	return new Column(javaObject);
};
/**
 * Compute bitwise OR of this expression with another expression.
 * @example
 *   df.select(df.col("colA").bitwiseOR(df.col("colB")));
 *
 * @since EclairJS 0.1 Spark  1.4.0
 * @param {Column} other
 * @returns {Column} 
 */
Column.prototype.bitwiseOR = function(other) {
	var javaObject =  this.getJavaObject().bitwiseOR(Utils.unwrapObject(other));
	return new Column(javaObject);
};
/**
 * Compute bitwise XOR of this expression with another expression.
 * @example
 *   df.select(df.col("colA").bitwiseXOR(df.col("colB")));
 *
 * @since EclairJS 0.1 Spark  1.4.0
 * @param {Column} other
 * @returns {Column} 
 */
Column.prototype.bitwiseXOR = function(other) {
	var javaObject =  this.getJavaObject().bitwiseXOR(Utils.unwrapObject(other));
	return new Column(javaObject);
};
/**
 * Casts the column to a different data type.
 * @example
 *   // Casts colA to IntegerType.
 *   df.select(df("colA").cast(DataTpes.IntegerType))
 *
 *   // equivalent to
 *   df.select(df.col("colA").cast("int"))
 *
 * @since EclairJS 0.1 Spark  1.3.0
 * @param {DataType | string} to If string supported types are: `string`, `boolean`, `int`,
 * `float`, `double`, `date`, `timestamp`.
 * @returns {Column} 
 */
Column.prototype.cast = function(to) {
	var javaObject =  this.getJavaObject().cast(Utils.unwrapObject(to));
	return new Column(javaObject);
};
/**
 * Contains the other element.
 *
 * @since EclairJS 0.1 Spark  1.3.0
 * @param {object}
 * @returns {Column} 
 */
Column.prototype.contains = function(other) {
	var javaObject =  this.getJavaObject().contains(other);
	return new Column(javaObject);
}
/**
 * Returns an ordering used in sorting.
 * @returns {Column}
 * @example
 * df.sort(df.col("age").desc());
 */
Column.prototype.desc = function() {
	return new Column(this.getJavaObject().desc());
};
/**
 * Division this expression by another expression.
 * @example
 *   people.select( people.col("height").divide(people.col("weight")) );
 *
 * @since EclairJS 0.1 Spark  1.3.0
 * @param {Column} other
 * @returns {Column} 
 */
Column.prototype.divide = function(other) {
	var javaObject =  this.getJavaObject().divide(Utils.unwrapObject(other));
	return new Column(javaObject);
}
/**
 * String ends with.
 * with another string literal
 * @since EclairJS 0.1 Spark  1.3.0
 * @param {string | Column} other, if string ends with another string literal.
 * @returns {Column} 
 */
Column.prototype.endsWith = function(other) {
	var javaObject =  this.getJavaObject().endsWith(Utils.unwrapObject(other));
	return new Column(javaObject);
}
/**
 * Equality test that is safe for null values.
 *
 * @since EclairJS 0.1 Spark  1.3.0
 * @param {object} other
 * @returns {Column} 
 */
Column.prototype.eqNullSafe = function(other) {
	var javaObject =  this.getJavaObject().eqNullSafe(Utils.unwrapObject(other));
	return new Column(javaObject);
}
/**
 * Equality test
 * @param {object}
 * @returns {boolean}
 */
Column.prototype.equals = function(that) {
	return  this.getJavaObject().equals(Utils.unwrapObject(that));
};
/**
 * Equality test
 * @param {object}
 * @returns {Column}
 */
Column.prototype.equalTo = function(obj) {
    return new Column(this.getJavaObject().equalTo(Utils.unwrapObject(obj)));
};
/**
 * Prints the expression to the console for debugging purpose.
 * @parma {boolean} extended
 * @since EclairJS 0.1 Spark  1.3.0
 */
Column.prototype.explain = function(extended) {
	var ex = extended ? true : false;
	this.getJavaObject().explain(extended);
};
/**
 * Greater than or equal to an expression.
 * @example
 * people.select( people.col("age").geq(21) )
 * @param {object}
 * @returns {Column}
 */
Column.prototype.geq = function(obj) {
	return new Column(this.getJavaObject().geq(Utils.unwrapObject(obj)));
};
/**
 * An expression that gets a field by name in a {@link StructType}.
 *
 * @since EclairJS 0.1 Spark  1.3.0
 * @param {string} fieldName
 * @returns {Column} 
 */
Column.prototype.getField = function(fieldName) {
	var javaObject =  this.getJavaObject().getField(fieldName);
	return new Column(javaObject);
};
/**
 * An expression that gets an item at position `ordinal` out of an array,
 * or gets a value by key `key` in a {@link MapType}.
 *
 * @since EclairJS 0.1 Spark  1.3.0
 * @param {object} key
 * @returns {Column} 
 */
Column.prototype.getItem = function(key) {
	var javaObject =  this.getJavaObject().getItem(Utils.unwrapObject(key));
	return new Column(javaObject);
};
/**
 * Greater than.
 * @example
 *   people.select( people.col("age").gt(21) );
 * @param {object}
 * @returns {Column}
 */
Column.prototype.gt = function(obj) {
	return new Column(this.getJavaObject().gt(Utils.unwrapObject(obj)));
};
/**
 * @returns {integer}
 */
Column.prototype.hashCode = function() {
	return  this.getJavaObject().hashCode();
};
/**
 * A boolean expression that is evaluated to true if the value of this expression is contained
 * by the evaluated values of the arguments.
 * @example
 * var col = peopleDataFrame.col("age");
 * var testCol = col.in([20, 19]);
 * var results = peopleDataFrame.select(testCol);
 * 
 * @since EclairJS 0.1 Spark  1.3.0
 * @param {array}
 * @returns {Column} 
 */
Column.prototype.in = function(list) {
	var javaObject =  this.getJavaObject().in(list);
	return new Column(javaObject);
};
/**
 * A boolean expression that is evaluated to true if the value of this expression is contained
 * by the evaluated values of the arguments.
 * 
 * @since EclairJS 0.1 Spark  1.3.0
 * @param {array}
 * @returns {Column} 
 */
Column.prototype.isin = function(list) {
	var javaObject =  this.getJavaObject().isin(list);
	return new Column(javaObject);
};
/**
 * True if the current expression is NaN.
 *
 * @since EclairJS 0.1 Spark  1.5.0
 * @returns {Column} 
 */
Column.prototype.isNaN = function() {
	var javaObject =  this.getJavaObject().isNaN();
	return new Column(javaObject);
};
/**
 * True if the current expression is null.
 *
 * @since EclairJS 0.1 Spark  1.3.0
 * @returns {Column} 
 */
Column.prototype.isNull = function() {
	var javaObject =  this.getJavaObject().isNull();
	return new Column(javaObject);
};
/**
 * True if the current expression is NOT null.
 *
 * @since EclairJS 0.1 Spark  1.3.0
 * @returns {Column} 
 */
Column.prototype.isNotNull = function() {
	var javaObject =  this.getJavaObject().isNotNull();
	return new Column(javaObject);
};
/**
 * Less than or equal to.
 * @example
 *   people.select( people.col("age").leq(21) );
 *
 * @since EclairJS 0.1 Spark  1.3.0
 * @param {object} other
 * @returns {Column} 
 */
Column.prototype.leq = function(other) {
	var javaObject =  this.getJavaObject().leq(Utils.unwrapObject(other));
	return new Column(javaObject);
};
/**
 * SQL like expression.
 *
 * @since EclairJS 0.1 Spark  1.3.0
 * @param {string} literal
 * @returns {Column} 
 */
Column.prototype.like = function(literal) {
	var javaObject =  this.getJavaObject().like(literal);
	return new Column(javaObject);
};
/**
 * Less than.
 * @example
 *   people.select( people.col("age").lt(21) );
 *
 * @since EclairJS 0.1 Spark  1.3.0
 * @param {object} other
 * @returns {Column} 
 */
Column.prototype.lt = function(other) {
	var javaObject =  this.getJavaObject().lt(Utils.unwrapObject(other));
	return new Column(javaObject);
};
/**
 * Subtraction. Subtract the other expression from this expression.
 * @example
 *   people.select( people.col("height").minus(people.col("weight")) );
 *
 * @since EclairJS 0.1 Spark  1.3.0
 * @param {Column} other
 * @returns {Column} 
 */
Column.prototype.minus = function(other) {
	var javaObject =  this.getJavaObject().minus(Utils.unwrapObject(other));
	return new Column(javaObject);
};
/**
 * Modulo (a.k.a. remainder) expression.
 *
 * @since EclairJS 0.1 Spark  1.3.0
 * @param {object} other
 * @returns {Column} 
 */
Column.prototype.mod = function(other) {
	var javaObject =  this.getJavaObject().mod(Utils.unwrapObject(other));
	return new Column(javaObject);
};
/**
 * Multiplication of this expression and another expression.
 * @example
 *   people.select( people.col("height").multiply(people.col("weight")) );
 *
 * @since EclairJS 0.1 Spark  1.3.0
 * @param {Column} other
 * @returns {Column} 
 */
Column.prototype.multiply = function(other) {
	var javaObject =  this.getJavaObject().multiply(Utils.unwrapObject(other));
	return new Column(javaObject);
};
/**
 * Inequality test.
 * @example
 *   df.filter( df.col("colA").notEqual(df.col("colB")) );
 *
 * @since EclairJS 0.1 Spark  1.3.0
 * @param {Column} other
 * @returns {Column} 
 */
Column.prototype.notEqual = function(other) {
	var javaObject =  this.getJavaObject().notEqual(Utils.unwrapObject(other));
	return new Column(javaObject);
};
/**
 * Boolean OR.
 * @example
 *   people.filter( people.col("inSchool").or(people.col("isEmployed")) );
 *
 * @since EclairJS 0.1 Spark  1.3.0
 * @param {Column} other
 * @returns {Column} 
 */
Column.prototype.or = function(other) {
	var other_uw = Utils.unwrapObject(other);
	var javaObject =  this.getJavaObject().or(other_uw);
	return new Column(javaObject);
};
/**
 * Evaluates a list of conditions and returns one of multiple possible result expressions.
 * If otherwise is not defined at the end, null is returned for unmatched conditions.
 *
 * @example
 *   people.select(functions.when(people.col("gender").equalTo("male"), 0)
 *     .when(people.col("gender").equalTo("female"), 1)
 *     .otherwise(2))
 *
 * @since EclairJS 0.1 Spark  1.4.0
 * @param {object} value
 * @returns {Column} 
 */
Column.prototype.otherwise = function(value) {
	var javaObject =  this.getJavaObject().otherwise(value);
	return new Column(javaObject);
};
/**
 * Evaluates a list of conditions and returns one of multiple possible result expressions.
 * If otherwise is not defined at the end, null is returned for unmatched conditions.
 *
 * @example
 *   people.select(functions.when(people.col("gender").equalTo("male"), 0)
 *     .when(people.col("gender").equalTo("female"), 1)
 *     .otherwise(2))
 *     
 * @since EclairJS 0.1 Spark  1.4.0
 * @returns {Column} 
 */
Column.prototype.when = function(condition,value) {
	var condition_uw = Utils.unwrapObject(condition);
	var javaObject =  this.getJavaObject().when(condition_uw,value);
	return new Column(javaObject);
}












</code></pre>
        </article>
    </section>




</div>

<nav>
    <h2><a href="index.html">Index</a></h2><h3>Classes</h3><ul><li><a href="Accumulable.html">Accumulable</a></li><li><a href="AccumulableParam.html">AccumulableParam</a></li><li><a href="Accumulator.html">Accumulator</a></li><li><a href="ALS.html">ALS</a></li><li><a href="ArrayType.html">ArrayType</a></li><li><a href="AssociationRules.html">AssociationRules</a></li><li><a href="BinaryType.html">BinaryType</a></li><li><a href="BisectingKMeans.html">BisectingKMeans</a></li><li><a href="BisectingKMeansModel.html">BisectingKMeansModel</a></li><li><a href="BooleanType.html">BooleanType</a></li><li><a href="CalendarIntervalType.html">CalendarIntervalType</a></li><li><a href="Column.html">Column</a></li><li><a href="DataFrame.html">DataFrame</a></li><li><a href="DataFrameNaFunctions.html">DataFrameNaFunctions</a></li><li><a href="DataFrameReader.html">DataFrameReader</a></li><li><a href="DataFrameStatFunctions.html">DataFrameStatFunctions</a></li><li><a href="DataFrameWriter.html">DataFrameWriter</a></li><li><a href="DataType.html">DataType</a></li><li><a href="DataTypes.html">DataTypes</a></li><li><a href="DateType.html">DateType</a></li><li><a href="DenseVector.html">DenseVector</a></li><li><a href="DoubleType.html">DoubleType</a></li><li><a href="DStream.html">DStream</a></li><li><a href="Duration.html">Duration</a></li><li><a href="FloatAccumulatorParam.html">FloatAccumulatorParam</a></li><li><a href="FloatType.html">FloatType</a></li><li><a href="FPGrowth.html">FPGrowth</a></li><li><a href="FPGrowthModel.html">FPGrowthModel</a></li><li><a href="FreqItemset.html">FreqItemset</a></li><li><a href="functions.html">functions</a></li><li><a href="FutureAction.html">FutureAction</a></li><li><a href="GroupedData.html">GroupedData</a></li><li><a href="HashPartitioner.html">HashPartitioner</a></li><li><a href="IntAccumulatorParam.html">IntAccumulatorParam</a></li><li><a href="IntegerType.html">IntegerType</a></li><li><a href="LabeledPoint.html">LabeledPoint</a></li><li><a href="LinearRegressionModel.html">LinearRegressionModel</a></li><li><a href="LinearRegressionWithSGD.html">LinearRegressionWithSGD</a></li><li><a href="List.html">List</a></li><li><a href="MapType.html">MapType</a></li><li><a href="MatrixFactorizationModel.html">MatrixFactorizationModel</a></li><li><a href="Metadata.html">Metadata</a></li><li><a href="MLWord2Vec.html">MLWord2Vec</a></li><li><a href="MLWord2VecModel.html">MLWord2VecModel</a></li><li><a href="NullType.html">NullType</a></li><li><a href="NumericType.html">NumericType</a></li><li><a href="PartialResult.html">PartialResult</a></li><li><a href="Partitioner.html">Partitioner</a></li><li><a href="RangePartitioner.html">RangePartitioner</a></li><li><a href="Rating.html">Rating</a></li><li><a href="RDD.html">RDD</a></li><li><a href="Row.html">Row</a></li><li><a href="RowFactory.html">RowFactory</a></li><li><a href="Rule.html">Rule</a></li><li><a href="SparkConf.html">SparkConf</a></li><li><a href="SparkContext.html">SparkContext</a></li><li><a href="SparkFiles.html">SparkFiles</a></li><li><a href="SparkStatusTracker.html">SparkStatusTracker</a></li><li><a href="SparseVector.html">SparseVector</a></li><li><a href="SQLContext.html">SQLContext</a></li><li><a href="SQLContext.QueryExecution.html">QueryExecution</a></li><li><a href="SQLContext.SparkPlanner.html">SparkPlanner</a></li><li><a href="SQLContext.SQLSession.html">SQLSession</a></li><li><a href="SqlDate.html">SqlDate</a></li><li><a href="SqlTimestamp.html">SqlTimestamp</a></li><li><a href="StorageLevel.html">StorageLevel</a></li><li><a href="StreamingContext.html">StreamingContext</a></li><li><a href="StringType.html">StringType</a></li><li><a href="StructField.html">StructField</a></li><li><a href="StructType.html">StructType</a></li><li><a href="Time.html">Time</a></li><li><a href="TimestampType.html">TimestampType</a></li><li><a href="Vectors.html">Vectors</a></li><li><a href="VectorUDT.html">VectorUDT</a></li><li><a href="Word2Vec.html">Word2Vec</a></li><li><a href="Word2VecModel.html">Word2VecModel</a></li></ul><h3>Global</h3><ul><li><a href="global.html#Vector">Vector</a></li></ul>
</nav>

<br clear="both">

<footer>
    Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 3.2.3-dev</a> on Thu Feb 18 2016 17:00:52 GMT-0500 (EST)
</footer>

<script> prettyPrint(); </script>
<script src="scripts/linenumber.js"> </script>
</body>
</html>
<file_sep>/*
 * Copyright 2016 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

var BinaryClassificationMetrics = function(rdd) {
    var javaRdd = rdd.getJavaObject();
    this.classTag = javaRdd.classTag()
    var r = rdd.getJavaObject().rdd();
    JavaWrapper.call(this,
                     new org.apache.spark.mllib.evaluation.BinaryClassificationMetrics(r));
};

BinaryClassificationMetrics.prototype = Object.create(JavaWrapper.prototype); 

/**
 * Returns the (threshold, precision) curve.
 */
BinaryClassificationMetrics.prototype.precisionByThreshold = function() {
    var rdd = this.getJavaObject().precisionByThreshold();
    return new RDD(org.apache.spark.api.java.JavaRDD.fromRDD(rdd, this.classTag));
};

/**
 * Returns the (threshold, recall) curve.
 */
BinaryClassificationMetrics.prototype.recallByThreshold = function() {
    var rdd = this.getJavaObject().recallByThreshold();
    return new RDD(org.apache.spark.api.java.JavaRDD.fromRDD(rdd, this.classTag));
};

/**
 * Returns the (threshold, F-Measure) curve with beta = 1.0.
 */
BinaryClassificationMetrics.prototype.fMeasureByThreshold = function(t) {
    if(arguments.length == 0) {
        var rdd = this.getJavaObject().fMeasureByThreshold();
        return new RDD(org.apache.spark.api.java.JavaRDD.fromRDD(rdd, this.classTag));
    } else {
        var rdd = this.getJavaObject().fMeasureByThreshold(t);
        return new RDD(org.apache.spark.api.java.JavaRDD.fromRDD(rdd, this.classTag));
    }
};

/**
 * Returns the precision-recall curve, which is an RDD of (recall, precision),
 * NOT (precision, recall), with (0.0, 1.0) prepended to it.
 * @see http://en.wikipedia.org/wiki/Precision_and_recall
 */
BinaryClassificationMetrics.prototype.pr = function() {
    var rdd = this.getJavaObject().pr();
    return new RDD(org.apache.spark.api.java.JavaRDD.fromRDD(rdd, this.classTag));
};


var RegressionMetrics = function(rdd) {
    var javaRdd = rdd.getJavaObject();
    this.classTag = javaRdd.classTag()
    JavaWrapper.call(this,
                     new org.apache.spark.mllib.evaluation.RegressionMetrics(javaRdd.rdd()));
};

RegressionMetrics.prototype = Object.create(JavaWrapper.prototype); 

/**
 * Returns the mean squared error, which is a risk function corresponding to the
 * expected value of the squared error loss or quadratic loss.
 */
RegressionMetrics.prototype.meanSquaredError = function() {
    return this.getJavaObject().meanSquaredError();
};

/**
 * Returns the root mean squared error, which is defined as the square root of
 * the mean squared error.
 */
RegressionMetrics.prototype.rootMeanSquaredError = function() {
    return this.getJavaObject().rootMeanSquaredError();
};

/**
 * Returns R^2^, the unadjusted coefficient of determination.
 * @see [[http://en.wikipedia.org/wiki/Coefficient_of_determination]]
 * In case of regression through the origin, the definition of R^2^ is to be modified.
 * @see <NAME>, Regression through the Origin. Teaching Statistics 25, 76-80 (2003)
 * [[https://online.stat.psu.edu/~ajw13/stat501/SpecialTopics/Reg_thru_origin.pdf]]
 */
RegressionMetrics.prototype.r2 = function() {
    return this.getJavaObject().r2();
};

/**
 * Returns the mean absolute error, which is a risk function corresponding to the
 * expected value of the absolute error loss or l1-norm loss.
 */
RegressionMetrics.prototype.meanAbsoluteError = function() {
    return this.getJavaObject().meanAbsoluteError();
};

/**
 * Returns the variance explained by regression.
 * explainedVariance = \sum_i (\hat{y_i} - \bar{y})^2 / n
 * @see [[https://en.wikipedia.org/wiki/Fraction_of_variance_unexplained]]
 */
RegressionMetrics.prototype.explainedVariance = function() {
    return this.getJavaObject().explainedVariance();
};

<file_sep>
/*
 * Copyright 2016 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */



/**
 * Represents a numeric vector, whose index type is Int and value type is Double.
 *
 * Note: Users should not implement this interface.
 * @classdesc
 */


var Vector = function(jvmObject) {

    this.logger = Logger.getLogger("Vector_js");
    JavaWrapper.call(this, jvmObject);

};

Vector.prototype = Object.create(JavaWrapper.prototype);

Vector.prototype.constructor = Vector;



/**
 * @returns {??}
 */
Vector.prototype.$init$ = function() {
    throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().$init$();
//   return new ??(javaObject);
};


/**
 * Size of the vector.
 * @returns {number}
 */
Vector.prototype.size = function() {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().size();
};


/**
 * Converts the instance to a double array.
 * @returns {number[]}
 */
Vector.prototype.toArray = function() {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().toArray();
};


/**
 * @param {object} other
 * @returns {boolean}
 */
Vector.prototype.equals = function(other) {
    throw "not implemented by ElairJS";
//   var other_uw = Utils.unwrapObject(other);
//   return  this.getJavaObject().equals(other_uw);
};


/**
 * Returns a hash code value for the vector. The hash code is based on its size and its first 128
 * nonzero entries, using a hash algorithm similar to {@link hashCode}.
 * @returns {number}
 */
Vector.prototype.hashCode = function() {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().hashCode();
};


/**
 * Gets the value of the ith element.
 * @param {number} i  index
 * @returns {number}
 */
Vector.prototype.apply = function(i) {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().apply(i);
};


/**
 * Makes a deep copy of this vector.
 * @returns {Vector}
 */
Vector.prototype.copy = function() {
    throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().copy();
//   return Utils.javaToJs(javaObject);
};


/**
 * Applies a function `f` to all the active elements of dense and sparse vector.
 *
 * @param {func} f  the function takes two parameters where the first parameter is the index of
 *          the vector with type `Int`, and the second parameter is the corresponding value
 *          with type `Double`.
 */
Vector.prototype.foreachActive = function(f) {
    throw "not implemented by ElairJS";
//   var sv = Utils.createJavaParams(f);
//   var fn = new org.eclairjs.nashorn.JSFunction2(sv.funcStr, sv.scopeVars);
//    this.getJavaObject().foreachActive(fn);
};


/**
 * Number of active entries.  An "active entry" is an element which is explicitly stored,
 * regardless of its value.  Note that inactive entries have value 0.
 * @returns {number}
 */
Vector.prototype.numActives = function() {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().numActives();
};


/**
 * Number of nonzero elements. This scans all active values and count nonzeros.
 * @returns {number}
 */
Vector.prototype.numNonzeros = function() {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().numNonzeros();
};


/**
 * Converts this vector to a sparse vector with all explicit zeros removed.
 * @returns {SparseVector}
 */
Vector.prototype.toSparse = function() {
    throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().toSparse();
//   return new SparseVector(javaObject);
};


/**
 * Converts this vector to a dense vector.
 * @returns {DenseVector}
 */
Vector.prototype.toDense = function() {
    throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().toDense();
//   return new DenseVector(javaObject);
};


/**
 * Returns a vector in either dense or sparse format, whichever uses less storage.
 * @returns {Vector}
 */
Vector.prototype.compressed = function() {
    throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().compressed();
//   return Utils.javaToJs(javaObject);
};


/**
 * Find the index of a maximal element.  Returns the first maximal element in case of a tie.
 * Returns -1 if vector has length 0.
 * @returns {number}
 */
Vector.prototype.argmax = function() {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().argmax();
};


/**
 * Converts the vector to a JSON string.
 * @returns {string}
 */
Vector.prototype.toJson = function() {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().toJson();
};


/**
 * :: AlphaComponent ::
 *
 * User-defined type for {@link Vector} which allows easy interaction with SQL
 * via {@link DataFrame}.
 * @classdesc
 */

/**
 * @returns {??}
 *  @class
 */
var VectorUDT = function(jvmObject) {

    this.logger = Logger.getLogger("VectorUDT_js");
    JavaWrapper.call(this, jvmObject);

};

VectorUDT.prototype = Object.create(JavaWrapper.prototype);

VectorUDT.prototype.constructor = VectorUDT;



/**
 * @returns {StructType}
 */
VectorUDT.prototype.sqlType = function() {
    throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().sqlType();
//   return new StructType(javaObject);
};


/**
 * @param {object} obj
 * @returns {InternalRow}
 */
VectorUDT.prototype.serialize = function(obj) {
    throw "not implemented by ElairJS";
//   var obj_uw = Utils.unwrapObject(obj);
//   return  this.getJavaObject().serialize(obj_uw);
};


/**
 * @param {object} datum
 * @returns {Vector}
 */
VectorUDT.prototype.deserialize = function(datum) {
    throw "not implemented by ElairJS";
//   var datum_uw = Utils.unwrapObject(datum);
//   var javaObject =  this.getJavaObject().deserialize(datum_uw);
//   return Utils.javaToJs(javaObject);
};


/**
 * @returns {string}
 */
VectorUDT.prototype.pyUDT = function() {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().pyUDT();
};


/**
 * @returns {Class}
 */
VectorUDT.prototype.userClass = function() {
    throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().userClass();
//   return new Class(javaObject);
};


/**
 * @param {object} o
 * @returns {boolean}
 */
VectorUDT.prototype.equals = function(o) {
    throw "not implemented by ElairJS";
//   var o_uw = Utils.unwrapObject(o);
//   return  this.getJavaObject().equals(o_uw);
};


/**
 * @returns {number}
 */
VectorUDT.prototype.hashCode = function() {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().hashCode();
};


/**
 * @returns {string}
 */
VectorUDT.prototype.typeName = function() {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().typeName();
};


/**
 * A dense vector represented by a value array.
 * @classdesc
 */

/**
 * @param {number[]} values
 * @returns {??}
 *  @class
 */
var DenseVector = function(arg) {
    this.logger = Logger.getLogger("DenseVector_js");
    var jvmObj;
    if (Array.isArray(arg)) {
        jvmObj = new org.apache.spark.mllib.linalg.DenseVector(arg);
    } else {
        jvmObj = arg;
    }
    Vector.call(this, jvmObj);

};

DenseVector.prototype = Object.create(Vector.prototype);

DenseVector.prototype.constructor = DenseVector;



/**
 * @returns {number}
 */
DenseVector.prototype.size = function() {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().size();
};


/**
 * @returns {string}
 */
DenseVector.prototype.toString = function() {
     return  this.getJavaObject().toString();
};


/**
 * @returns {float[]}
 */
DenseVector.prototype.toArray = function() {

    return  this.getJavaObject().toArray();
};


/**
 * @param {number} i
 * @returns {number}
 */
DenseVector.prototype.apply = function(i) {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().apply(i);
};


/**
 * @returns {DenseVector}
 */
DenseVector.prototype.copy = function() {
    throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().copy();
//   return new DenseVector(javaObject);
};


/**
 * @param {func} f
 */
DenseVector.prototype.foreachActive = function(f) {
    throw "not implemented by ElairJS";
//   var sv = Utils.createJavaParams(f);
//   var fn = new org.eclairjs.nashorn.JSFunction2(sv.funcStr, sv.scopeVars);
//    this.getJavaObject().foreachActive(fn);
};


/**
 * @returns {number}
 */
DenseVector.prototype.hashCode = function() {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().hashCode();
};


/**
 * @returns {number}
 */
DenseVector.prototype.numActives = function() {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().numActives();
};


/**
 * @returns {number}
 */
DenseVector.prototype.numNonzeros = function() {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().numNonzeros();
};


/**
 * @returns {SparseVector}
 */
DenseVector.prototype.toSparse = function() {
    throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().toSparse();
//   return new SparseVector(javaObject);
};


/**
 * @returns {number}
 */
DenseVector.prototype.argmax = function() {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().argmax();
};


/**
 * @returns {string}
 */
DenseVector.prototype.toJson = function() {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().toJson();
};


/**
 * A sparse vector represented by an index array and an value array.
 *
 * @param size size of the vector.
 * @param indices index array, assume to be strictly increasing.
 * @param values value array, must have the same length as the index array.
 * @classdesc
 */

/**
 * @param {number} size
 * @param {number[]} indices
 * @param {number[]} values
 * @returns {??}
 *  @class
 */
var SparseVector = function(size,indices,values) {
    var jvmObject = new org.apache.spark.mllib.linalg.SparseVector(size,indices,values);
    this.logger = Logger.getLogger("SparseVector_js");
    Vector.call(this, jvmObject);

};

SparseVector.prototype = Object.create(Vector.prototype);

SparseVector.prototype.constructor = SparseVector;



/**
 * @returns {string}
 */
SparseVector.prototype.toString = function() {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().toString();
};


/**
 * @returns {number[]}
 */
SparseVector.prototype.toArray = function() {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().toArray();
};


/**
 * @returns {SparseVector}
 */
SparseVector.prototype.copy = function() {
    throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().copy();
//   return new SparseVector(javaObject);
};


/**
 * @param {func} f
 */
SparseVector.prototype.foreachActive = function(f) {
    throw "not implemented by ElairJS";
//   var sv = Utils.createJavaParams(f);
//   var fn = new org.eclairjs.nashorn.JSFunction2(sv.funcStr, sv.scopeVars);
//    this.getJavaObject().foreachActive(fn);
};


/**
 * @returns {number}
 */
SparseVector.prototype.hashCode = function() {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().hashCode();
};


/**
 * @returns {number}
 */
SparseVector.prototype.numActives = function() {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().numActives();
};


/**
 * @returns {number}
 */
SparseVector.prototype.numNonzeros = function() {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().numNonzeros();
};


/**
 * @returns {SparseVector}
 */
SparseVector.prototype.toSparse = function() {
    throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().toSparse();
//   return new SparseVector(javaObject);
};


/**
 * @returns {number}
 */
SparseVector.prototype.argmax = function() {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().argmax();
};


/**
 * @returns {string}
 */
SparseVector.prototype.toJson = function() {
    throw "not implemented by ElairJS";
//   return  this.getJavaObject().toJson();
};


/**
 *
 * @constructor
 */
var Vectors = function() {
    //var jvmObject = new org.apache.spark.mllib.linalg.SparseVector(size,indices,values);
    this.logger = Logger.getLogger("Vectors_js");
    //Vector.call(this, jvmObject);

};

//
// static methods
//

/**
 * Creates a dense vector from its values.
 * @param {number} firstValue
 * @param {...number} otherValues
 * @returns {Vector}
 */
Vectors.densewithOtherValues = function(firstValue,otherValues) {
    throw "not implemented by ElairJS";
// // TODO: handle repeated parm 'otherValues'
//   var javaObject =  org.apache.spark.mllib.linalg.Vectors.dense(firstValue,otherValues);
//   return Utils.javaToJs(javaObject);
};


/**
 * Creates a dense vector from a double array.
 * @param {float[]} values
 * @returns {Vector}
 */
Vectors.dense = function(values) {
    var javaObject =  org.apache.spark.mllib.linalg.Vectors.dense(values);
    return Utils.javaToJs(javaObject);
};


/**
 * Creates a sparse vector providing its index array and value array.
 *
 * @param {number} size  vector size.
 * @param {number[]} indices  index array, must be strictly increasing.
 * @param {number[]} values  value array, must have the same length as indices.
 * @returns {Vector}
 */
Vectors.sparse0 = function(size,indices,values) {
    throw "not implemented by ElairJS";
//   var javaObject =  org.apache.spark.mllib.linalg.Vectors.sparse(size,indices,values);
//   return Utils.javaToJs(javaObject);
};


/**
 * Creates a sparse vector using unordered (index, value) pairs.
 *
 * @param {number} size  vector size.
 * @param {Tuple2[]} elements  vector elements in (index, value) pairs.
 * @returns {Vector}
 */
Vectors.sparse1 = function(size,elements) {
    throw "not implemented by ElairJS";
// // TODO: handle Tuple conversion for 'elements'
//   var elements_uw = Utils.unwrapObject(elements);
//   var javaObject =  org.apache.spark.mllib.linalg.Vectors.sparse(size,elements_uw);
//   return Utils.javaToJs(javaObject);
};


/**
 * Creates a sparse vector using unordered (index, value) pairs in a Java friendly way.
 *
 * @param {number} size  vector size.
 * @param {JavaIterable} elements  vector elements in (index, value) pairs.
 * @returns {Vector}
 */
Vectors.sparse2 = function(size,elements) {
    throw "not implemented by ElairJS";
// // TODO: handle Tuple conversion for 'elements'
//   var elements_uw = Utils.unwrapObject(elements);
//   var javaObject =  org.apache.spark.mllib.linalg.Vectors.sparse(size,elements_uw);
//   return Utils.javaToJs(javaObject);
};


/**
 * Creates a vector of all zeros.
 *
 * @param {number} size  vector size
 * @returns {Vector}  a zero vector
 */
Vectors.zeros = function(size) {
    throw "not implemented by ElairJS";
//   var javaObject =  org.apache.spark.mllib.linalg.Vectors.zeros(size);
//   return Utils.javaToJs(javaObject);
};


/**
 * Parses a string resulted from [[Vector.toString]] into a {@link Vector}.
 * @param {string} s
 * @returns {Vector}
 */
Vectors.parse = function(s) {
    throw "not implemented by ElairJS";
//   var javaObject =  org.apache.spark.mllib.linalg.Vectors.parse(s);
//   return Utils.javaToJs(javaObject);
};


/**
 * Parses the JSON representation of a vector into a {@link Vector}.
 * @param {string} json
 * @returns {Vector}
 */
Vectors.fromJson = function(json) {
    throw "not implemented by ElairJS";
//   var javaObject =  org.apache.spark.mllib.linalg.Vectors.fromJson(json);
//   return Utils.javaToJs(javaObject);
};


/**
 * Returns the p-norm of this vector.
 * @param {Vector} vector  input vector.
 * @param {number} p  norm.
 * @returns {number}  norm in L^p^ space.
 */
Vectors.norm = function(vector,p) {
    throw "not implemented by ElairJS";
//   var vector_uw = Utils.unwrapObject(vector);
//   return  org.apache.spark.mllib.linalg.Vectors.norm(vector_uw,p);
};


/**
 * Returns the squared distance between two Vectors.
 * @param {Vector} v1  first Vector.
 * @param {Vector} v2  second Vector.
 * @returns {number}  squared distance between two Vectors.
 */
Vectors.sqdist = function(v1,v2) {
    throw "not implemented by ElairJS";
//   var v1_uw = Utils.unwrapObject(v1);
//   var v2_uw = Utils.unwrapObject(v2);
//   return  org.apache.spark.mllib.linalg.Vectors.sqdist(v1_uw,v2_uw);
};


/**
 * @param {DenseVector} dv
 * @returns {Array}
 */
DenseVector.unapply = function(dv) {
    throw "not implemented by ElairJS";
//   var dv_uw = Utils.unwrapObject(dv);
//   return  org.apache.spark.mllib.linalg.DenseVector.unapply(dv_uw);
};


/**
 * @param {SparseVector} sv
 * @returns {Tuple3}
 */
SparseVector.unapply = function(sv) {
    throw "not implemented by ElairJS";
//   var sv_uw = Utils.unwrapObject(sv);
//   var javaObject =  org.apache.spark.mllib.linalg.SparseVector.unapply(sv_uw);
//   return new Tuple3(javaObject);
};
<file_sep>/*
 * Copyright 2015 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/**
 * @constructor
 * @classdec A set of methods for aggregations on a DataFrame, created by DataFrame.groupBy.
 */
var GroupedData = function(jvmGroupedData) {
	
    JavaWrapper.call(this, jvmGroupedData);

	  // Initialize our Row-specific properties
	this.logger = Logger.getLogger("sql.GroupData_js");
}

GroupedData.prototype = Object.create(JavaWrapper.prototype); 

//Set the "constructor" property to refer to GroupedData
GroupedData.prototype.constructor = GroupedData;

/**
 * Compute aggregates by specifying a series of aggregate columns. Note that this function by default retains the grouping columns in its output. 
 * To not retain grouping columns, set spark.sql.retainGroupColumns to false.
 * The available aggregate methods are defined in {@link functions}.
 * @example
 * // Java:
 * df.groupBy("department").agg(max("age"), sum("expense"));
 * @since EclairJS 0.1 Spark  1.3.0
 * @param {Column | string} columnExpr,...columnExpr or columnName, ...columnName
 * @returns {DataFrame} 
 */
GroupedData.prototype.agg = function() {
	/*
	 * First convert any strings to Columns
	 */

	var args = Utils.createJavaObjectArguments(arguments, Column);
	/*
	 * Create a argument list we can send to Java
	 */
	var str = "this.getJavaObject().agg("
	for (var i = 0; i < args.length; i++) {
		var spacer = i < 1 ? "" : ",";
		str += spacer + "args[" + i + "]";
	}	
	str += ");";

	var javaObject = eval(str);
    return new DataFrame(javaObject);
};
/**
 * Compute the avg value for each numeric columns for each group.
 * @param {string[]} cols
 * @returns {DataFrame}
 */
GroupedData.prototype.avg = function(cols) {
    return new DataFrame(this.getJavaObject().avg(cols));
};

GroupedData.prototype.apply = function(cols) {
	throw "not implemented by ElairJS";
};
/**
 * Count the number of rows for each group.
 * @returns {DataFrame}
 */
GroupedData.prototype.count = function() {
    var jdf = this.getJavaObject().count();
    var df = new DataFrame(jdf);

    return df;
};
/**
 * Compute the max value for each numeric columns for each group.
 * @param {string[]} cols
 * @returns {DataFrame}
 */
GroupedData.prototype.max = function(cols) {
    return new DataFrame(this.getJavaObject().max(cols));
};
/**
 * Compute the mean value for each numeric columns for each group.
 * @param {string[]} cols
 * @returns {DataFrame}
 */
GroupedData.prototype.mean = function(cols) {
    return new DataFrame(this.getJavaObject().mean(cols));
};
/**
 * Compute the min value for each numeric columns for each group.
 * @param {string[]} cols
 * @returns {DataFrame}
 */
GroupedData.prototype.min = function(cols) {
    return new DataFrame(this.getJavaObject().min(cols));
};
/**
 * Compute the sum value for each numeric columns for each group.
 * @param {string[]} cols
 * @returns {DataFrame}
 */
GroupedData.prototype.sum = function(cols) {
    return new DataFrame(this.getJavaObject().sum(cols));
};

<file_sep>/*                                                                         
* Copyright 2016 IBM Corp.                                                 
*                                                                          
* Licensed under the Apache License, Version 2.0 (the "License");          
* you may not use this file except in compliance with the License.         
* You may obtain a copy of the License at                                  
*                                                                          
*      http://www.apache.org/licenses/LICENSE-2.0                          
*                                                                          
* Unless required by applicable law or agreed to in writing, software      
* distributed under the License is distributed on an "AS IS" BASIS,        
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
* See the License for the specific language governing permissions and      
* limitations under the License.                                           
*/ 



/**
 * Word2Vec creates vector representation of words in a text corpus.
 * The algorithm first constructs a vocabulary from the corpus
 * and then learns vector representation of words in the vocabulary.
 * The vector representation can be used as features in
 * natural language processing and machine learning algorithms.
 *
 * We used skip-gram model in our implementation and hierarchical softmax
 * method to train the model. The variable names in the implementation
 * matches the original C implementation.
 *
 * For original C implementation, see https://code.google.com/p/word2vec/
 * For research papers, see
 * Efficient Estimation of Word Representations in Vector Space
 * and
 * Distributed Representations of Words and Phrases and their Compositionality.
 * @classdesc
 *  @class
 */
var Word2Vec = function(jvmObject) {
	 
	 this.logger = Logger.getLogger("Word2Vec_js");
	if (!jvmObject) {
		jvmObject = new org.apache.spark.mllib.feature.Word2Vec();
	}
	 JavaWrapper.call(this, jvmObject);

};

Word2Vec.prototype = Object.create(JavaWrapper.prototype);

Word2Vec.prototype.constructor = Word2Vec;



/**
 * Sets vector size (default: 100).
 * @param {number} vectorSize
 * @returns {} 
 */
Word2Vec.prototype.setVectorSize = function(vectorSize) {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().setVectorSize(vectorSize);
//   return new (javaObject);
};


/**
 * Sets initial learning rate (default: 0.025).
 * @param {number} learningRate
 * @returns {} 
 */
Word2Vec.prototype.setLearningRate = function(learningRate) {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().setLearningRate(learningRate);
//   return new (javaObject);
};


/**
 * Sets number of partitions (default: 1). Use a small number for accuracy.
 * @param {number} numPartitions
 * @returns {} 
 */
Word2Vec.prototype.setNumPartitions = function(numPartitions) {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().setNumPartitions(numPartitions);
//   return new (javaObject);
};


/**
 * Sets number of iterations (default: 1), which should be smaller than or equal to number of
 * partitions.
 * @param {number} numIterations
 * @returns {} 
 */
Word2Vec.prototype.setNumIterations = function(numIterations) {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().setNumIterations(numIterations);
//   return new (javaObject);
};


/**
 * Sets random seed (default: a random long integer).
 * @param {number} seed
 * @returns {} 
 */
Word2Vec.prototype.setSeed = function(seed) {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().setSeed(seed);
//   return new (javaObject);
};


/**
 * Sets the window of words (default: 5)
 * @param {number} window
 * @returns {} 
 */
Word2Vec.prototype.setWindowSize = function(window) {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().setWindowSize(window);
//   return new (javaObject);
};


/**
 * Sets minCount, the minimum number of times a token must appear to be included in the word2vec
 * model's vocabulary (default: 5).
 * @param {number} minCount
 * @returns {} 
 */
Word2Vec.prototype.setMinCount = function(minCount) {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().setMinCount(minCount);
//   return new (javaObject);
};


/**
 * Computes the vector representation of each word in vocabulary.
 * @param {RDD} dataset  an RDD of words
 * @returns {Word2VecModel}  a Word2VecModel
 */
Word2Vec.prototype.fit = function(dataset) {
   var dataset_uw = Utils.unwrapObject(dataset);
   var javaObject =  this.getJavaObject().fit(dataset_uw);
   return new Word2VecModel(javaObject);
};

/**
 * Word2Vec model
 * @param wordIndex maps each word to an index, which can retrieve the corresponding
 *                  vector from wordVectors
 * @param wordVectors array of length numWords * vectorSize, vector corresponding
 *                    to the word mapped with index i can be retrieved by the slice
 *                    (i * vectorSize, i * vectorSize + vectorSize)
 * @classdesc
 */

/**
 * @param {Map} model
 * @returns {??} 
 *  @class
 */
var Word2VecModel = function(jvmObject) {

	 //var jvmObject = new org.apache.spark.mllib.feature.Word2VecModel(model);
	 this.logger = Logger.getLogger("Word2VecModel_js");
	 JavaWrapper.call(this, jvmObject);

};

Word2VecModel.prototype = Object.create(JavaWrapper.prototype);

Word2VecModel.prototype.constructor = Word2VecModel;



/**
 * @param {SparkContext} sc
 * @param {string} path
 */
Word2VecModel.prototype.save = function(sc,path) {
throw "not implemented by ElairJS";
//   var sc_uw = Utils.unwrapObject(sc);
//    this.getJavaObject().save(sc_uw,path);
};


/**
 * Transforms a word to its vector representation
 * @param {string} word  a word
 * @returns {Vector}  vector representation of word
 */
Word2VecModel.prototype.transform = function(word) {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().transform(word);
//   return Utils.javaToJs(javaObject);
};


/**
 * Find synonyms of a word
 * @param {string} word  a word
 * @param {number} num  number of synonyms to find
 * @returns {Tuple2[]}  array of (word, cosineSimilarity)
 */
Word2VecModel.prototype.findSynonyms = function(word,num) {
    var javaObject =  this.getJavaObject().findSynonyms(word,num);
    return Utils.javaToJs(javaObject);
};


/**
 * Find synonyms of the vector representation of a word
 * @param {Vector} vector  vector representation of a word
 * @param {number} num  number of synonyms to find
 * @returns {Tuple2[]}  array of (word, cosineSimilarity)
 */
Word2VecModel.prototype.findSynonymswithnumber = function(vector,num) {
throw "not implemented by ElairJS";
//   var vector_uw = Utils.unwrapObject(vector);
//   var javaObject =  this.getJavaObject().findSynonyms(vector_uw,num);
//   return Utils.javaToJs(javaObject);
};


/**
 * Returns a map of words to their vector representations.
 * @returns {Map} 
 */
Word2VecModel.prototype.getVectors = function() {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().getVectors();
//   return new Map(javaObject);
};
//
// static methods
//


/**
 * @param {SparkContext} sc
 * @param {string} path
 * @returns {Word2VecModel} 
 */
Word2VecModel.load = function(sc,path) {
throw "not implemented by ElairJS";
//   var sc_uw = Utils.unwrapObject(sc);
//   var javaObject =  org.apache.spark.mllib.feature.Word2VecModel.load(sc_uw,path);
//   return new Word2VecModel(javaObject);
};
<file_sep>/*
 * Copyright 2015 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * We need to load SparkContext.js and SparkConf.js in order to create SparkContext
 * The SparkContext will load the rest of sparkJS files. So these are the oly two 
 * the user has to explicitly load. 
 */

var sparkContext = new SparkContext("local[*]", "mllib Unit test");

var LinearRegressionWithSGDTest = function(file) {
	
	var sc = sparkContext;
	
	var data = sc.textFile(file).cache();
	var scopeVars = {};
	var parsedData = data.map( function(s) { 
		var parts = s.split(",");
		var features = parts[1].split(" "); 
		return new LabeledPoint(parts[0], new DenseVector(features));
	 });
	//var t = parsedData.take(5);
	//print("take 5 = " + JSON.stringify(parsedData.take(5)));
	var numIterations = 3;
	/* var */ linearRegressionModel = LinearRegressionWithSGD.train(parsedData, numIterations); // Due to JUNIT scoping these need to be global
	/* var */ delta = 17; // Due to JUNIT scoping these need to be global
	var valuesAndPreds = parsedData.mapToPair(function(lp, linearRegressionModel, delta) {
		var label = lp.getLabel();
		var f = lp.getFeatures();
		var prediction = linearRegressionModel.predict(f) + delta;
		return [prediction, label];
	}, [linearRegressionModel, delta]); // end MapToPair
	
	//print("valuesAndPreds: " + valuesAndPreds.take(10).toString());
	return valuesAndPreds.take(10).toString();
}

var AssociationRulesTest = function() {
    load("examples/mllib/association_rules_example.js");
    return run(sparkContext);
}

var BisectingKMeansExample = function() {
    load("examples/mllib/bisecting_k_means_example.js");
    return JSON.stringify(run(sparkContext));
}


<file_sep>/*
 * Copyright 2015 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * We need to load SparkContext.js and SparkConf.js in order to create SparkContext
 * The SparkContext will load the rest of sparkJS files. So these are the oly two 
 * the user has to explicitly load. 
 */

var sparkConf = new SparkConf()
  .setAppName("Linear Regression Test")
  .setMaster("local[*]");
  //.setMaster("spark://MacBook-Pro.local:7077");

var sc = new SparkContext(sparkConf);

var data = sc.textFile("examples/data/lpsa.data").cache();
var scopeVars = {};
var parsedData = data.map( function(s) { 
	var parts = s.split(",");
	var features = parts[1].split(" "); 
	return new LabeledPoint(parts[0], new DenseVector(features));
 });
var t = parsedData.take(5);
print("take 5 = " + JSON.stringify(parsedData.take(5)));
var numIterations = 3;
var linearRegressionModel = LinearRegressionWithSGD.train(parsedData, numIterations);

var delta = 17;
var valuesAndPreds = parsedData.mapToPair(function(lp, linearRegressionModel, delta) { // FIXME
	var label = lp.getLabel();
	var f = lp.getFeatures();
	var prediction = linearRegressionModel.predict(f) + delta;
	return [prediction, label];
}, [linearRegressionModel, delta]); // end MapToPair

print("valuesAndPreds: " + valuesAndPreds.take(10).toString());




<file_sep>/*                                                                         
* Copyright 2015 IBM Corp.                                                 
*                                                                          
* Licensed under the Apache License, Version 2.0 (the "License");          
* you may not use this file except in compliance with the License.         
* You may obtain a copy of the License at                                  
*                                                                          
*      http://www.apache.org/licenses/LICENSE-2.0                          
*                                                                          
* Unless required by applicable law or agreed to in writing, software      
* distributed under the License is distributed on an "AS IS" BASIS,        
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
* See the License for the specific language governing permissions and      
* limitations under the License.                                           
*/ 




/**
  * @constructor
  * @classdesc PartialResult
 */
var PartialResult = function(initialVal,isFinal) {
    var jvmObject = new org.apache.spark.partial.PartialResult(initialVal,isFinal);
    this.logger = Logger.getLogger("PartialResult_js");
    JavaWrapper.call(this, jvmObject);
};

PartialResult.prototype = Object.create(JavaWrapper.prototype);

PartialResult.prototype.constructor = PartialResult;



PartialResult.prototype.initialValue = function() {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().initialValue();
//   return new object(javaObject);
}



PartialResult.prototype.isInitialValueFinal = function() {
throw "not implemented by ElairJS";
//   return  this.getJavaObject().isInitialValueFinal();
}


/**
 * Blocking method to wait for and return the final value.
 * @returns {object} 
 */
PartialResult.prototype.getFinalValue = function() {
//throw "not implemented by ElairJS";
   var javaObject =  this.getJavaObject().getFinalValue();
   return new object(javaObject);
}


/**
 * Set a handler to be called when this PartialResult completes. Only one completion handler
 * is supported per PartialResult.
 * @returns {PartialResult} 
 */
PartialResult.prototype.onComplete = function(handler, bindArgs) {
//throw "not implemented by ElairJS";
   var fn = Utils.createLambdaFunction(handler, org.eclairjs.nashorn.JSFunction, bindArgs);
   var javaObject =  this.getJavaObject().onComplete(fn);
   return new PartialResult(javaObject);
}


/**
 * Set a handler to be called if this PartialResult's job fails. Only one failure handler
 * is supported per PartialResult.
 */
PartialResult.prototype.onFail = function(handler, bindArgs) {
throw "not implemented by ElairJS";
//      var fn = Utils.createLambdaFunction(handler, org.eclairjs.nashorn.JSFunction, bindArgs);
//    this.getJavaObject().onFail(fn);
}


/**
 * Transform this PartialResult into a PartialResult of type T.
 * @returns {PartialResult} 
 */
PartialResult.prototype.map = function(func, bindArgs) {
throw "not implemented by ElairJS";
//  var fn = Utils.createLambdaFunction(func, org.eclairjs.nashorn.JSFunction, bindArgs);
//   var javaObject =  this.getJavaObject().map(fn);
//   return new PartialResult(javaObject);
}



PartialResult.prototype.toString = function() {
throw "not implemented by ElairJS";
   return  this.getJavaObject().toString();
}
<file_sep>/*
 * Copyright 2015 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/** 
 * Construct a LinearRegression object with default parameters: {stepSize: 1.0, numIterations: 100, miniBatchFraction: 1.0}.
 * @constructor
 * @classdesc Train a linear regression model with no regularization using Stochastic Gradient Descent. 
 * This solves the least squares regression formulation f(weights) = 1/n ||A weights-y||^2^ (which is the mean squared error). 
 * Here the data matrix has n rows, and the input RDD holds the set of rows of A, each with its corresponding right hand side label y. 
 * See also the documentation for the precise formulation.
 */
var LinearRegressionWithSGD = {}

LinearRegressionWithSGD.DEFAULT_NUM_ITERATIONS = 100;

/**
 * Train a LinearRegression model given an RDD of (label, features) pairs.
 * @param {RDD} rdd of LabeledPoints
 * @param {integer} numIterations
 * @returns {LinearRegressionModel}
 */
LinearRegressionWithSGD.train = function(rdd, numIterations) {
	var logger = Logger.getLogger("LinearRegressionWithSGD_js");
	logger.debug("JavaRDD " + rdd);
	var jo = rdd.getJavaObject();
	logger.debug("jo = " + jo);
	var lrdd = org.apache.spark.api.java.JavaRDD.toRDD(jo);
	logger.debug("calling train");
	var model = org.apache.spark.mllib.regression.LinearRegressionWithSGD.train(lrdd, numIterations);
	logger.debug("return model");
	return new LinearRegressionModel(model);

};





<file_sep>/*
 * Copyright 2015 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


/**
 * @constructor
 * @classdec Represents a Discretized Stream (DStream), the basic abstraction in Spark Streaming, 
 * is a continuous sequence of RDDs (of the same type) representing a continuous stream of data.
 * @param {object} jDStream 
  */
var DStream = function(jDStream, streamingContext) {
	var jvmObj = jDStream;
	this.streamingContext = streamingContext;
	this.logger = Logger.getLogger("streaming.dtream.DStream_js");
	JavaWrapper.call(this, jvmObj);
};

DStream.prototype = Object.create(JavaWrapper.prototype); 

DStream.prototype.constructor = DStream;

/**
 * Persist RDDs of this DStream with the default storage level (MEMORY_ONLY_SER)
 * @returns {DStream}
 */
DStream.prototype.cache = function() {
    return new DStream(this.getJavaObject().cache(),
                       this.streamingContext);
};
/**
 * Generate an RDD for the given duration
 * @param {Time} validTime
 * @returns {RDD}
 */
DStream.prototype.compute = function(validTime) {
    return new DStream(this.getJavaObject().compute(validTime.getJavaObject()),
                       this.streamingContext);
};
/**
 * Return a new DStream containing only the elements that satisfy a predicate.
 * @param {function} func
 * @param {Object[]} bindArgs - Optional array whose values will be added to func's argument list.
 * @returns {DStream}
 */
DStream.prototype.filter = function(func, bindArgs) {
    var fn = Utils.createLambdaFunction(func, org.eclairjs.nashorn.JSFunction, bindArgs);
    return new DStream(this.getJavaObject().filter(fn), this.streamingContext);
};
/**
 * Persist RDDs of this DStream with the storage level
 * @param {StorageLevel} Optional (MEMORY_ONLY_SER) by default
 * @return {DStream}
 */
DStream.prototype.persist = function() {
    if(arguments.length == 1) {
        return new DStream(
            this.getJavaObject().persist(arguments[0].getJavaObject()),
            this.streamingContext);
    } else {
        return new DStream(this.getJavaObject().perist(), this.streamingContext);
    }
};
/**
 * Return a new DStream with an increased or decreased level of parallelism.
 * @param {int} numPartitions
 * @returns {DStream}
 */
DStream.prototype.repartition = function(numPartitions) {
    return new DStream(this.getJavaObject().repartition(numPartitions),
                       this.streamingContext);
};
/**
 * Return a new DStream by unifying data of another DStream with this DStream.
 * @param {DStream} that
 * @returns {DStream}
 */
DStream.prototype.union = function(that) {
    return new DStream(this.getJavaObject().union(Utils.unwrapObject(that)),
                       this.streamingContext);
}
/**
 * Return a new DStream in which each RDD contains all the elements in seen in a sliding window of time over this DStream. 
 * The new DStream generates RDDs with the same interval as this DStream.
 * @param duration - width of the window; must be a multiple of this DStream's interval.
 * @param slideInterval - Optional
 * @returns {DStream}
 */
DStream.prototype.window = function(duration) {
    if(arguments.length == 1) {
        return new DStream(this.getJavaObject().window(
            Utils.unwrapObject(arguments[0])),
                           this.streamingContext);
    } else {
        return new DStream(this.getJavaObject().window(
            Utils.unwrapObject(arguments[0]),
            Utils.unwrapObject(arguments[1])),
                           this.streamingContext);
    }
};
/**
 * Enable periodic checkpointing of RDDs of this DStream.
 * @param {Duration} interval
 * @return {DStream}
 */
DStream.prototype.checkpoint = function(interval) {
    return new DStream(this.getJavaObject().checkpoint(interval.getJavaObject()),
                       this.streamingContext);
};
/**
 * Return the StreamingContext associated with this DStream
 * @returns {StreamingContext}
 */
DStream.prototype.context = function() {
    return this.streamingContext;
};
/**
 * Return a new DStream in which each RDD has a single element generated by 
 * counting each RDD of this DStream.
 * @returns {DStream}
 */
DStream.prototype.count = function() {
    return new DStream(this.getJavaObject().count(), this.streamingContext);
};
/**
 * Return a new DStream in which each RDD contains the counts of each distinct
 * value in each RDD of this DStream. Hash partitioning is used to generate the
 * RDDs with Spark's default number of partitions if numPartions is not
 * specified.
 * @param {int} numPartitions
 * @returns {DStream}
 */
DStream.prototype.countByValue = function() {
    if(arguments.length > 0) {
        return new DStream(this.getJavaObject().countByValue(arguments[0]),
                           this.stremaingContext);
    } else {
        return new DStream(this.getJavaObject().countByValue(),
                           this.streamingContext);
    }
};
/**
 * Return a new DStream in which each RDD contains the count of distinct
 * elements in RDDs in a sliding window over this DStream. Hash partitioning is
 * used to generate the RDDs with Spark's default number of partitions if
 * numPartitions is not specified.
 * @param {Duration} windowDuration
 * @param {Duration} slideDuration
 * @param {int} numPartitions
 * @returns {DStream}
 */
DStream.prototype.countByValueAndWindow = function() {
    if(arguments.length == 2) {
        return new DStream(
            this.getJavaObject().countByValueAndWindow(
                arguments[0].getJavaObject(),
                arguments[1].getJavaObject()
            ), this.streamingContext
        )
    } else {
        return new DStream(
            this.getJavaObject().countByValueAndWindow(
                arguments[0].getJavaObject(),
                arguments[1].getJavaObject(),
                arguments[2]
            ), this.streamingContext
        )
    }
};
/**
 * Return a new DStream in which each RDD has a single element generated by
 * counting the number of elements in a window over this DStream. windowDuration
 * and slideDuration are as defined in the window() operation. This is
 * equivalent to window(windowDuration, slideDuration).count()
 * @param {Duration} windowDuration
 * @param {Duration} slideDuration
 * @returns {DStream}
 */
DStream.prototype.countByWindow = function(windowDuration, slideDuration) {
    return new DStream(
        this.getJavaObject().countByWindow(
            windowDuration.getJavaObject(),
            slideDuration.getJavaObject()
        ), this.streamingContext)
};
/**
 * Return a new DStream by first applying a function to all elements of this 
 * DStream, and then flattening the results.
 * @param func
 * @param {Object[]} bindArgs - Optional array whose values will be added to func's argument list.
 * @returns {DStream}
 */
DStream.prototype.flatMap = function(func, bindArgs) {
    var fn = Utils.createLambdaFunction(func, org.eclairjs.nashorn.JSFlatMapFunction, bindArgs);
    return new DStream(this.getJavaObject().flatMap(fn), this.streamingContext);
};
/**
 * Return a new DStream by applying a function to all elements of this DStream,
 * and then flattening the results
 * @param func
 * @param {Object[]} bindArgs - Optional array whose values will be added to func's argument list.
 * @returns {DStream}
 */
DStream.prototype.flatMapToPair = function(func, bindArgs) {
    var fn = Utils.createLambdaFunction(func, org.eclairjs.nashorn.JSPairFlatMapFunction, bindArgs);
    return new DStream(this.getJavaObject().flatMapToPair(fn),
                       this.streamingContext);
};
/**
 * Return a new DStream in which each RDD is generated by applying glom() to
 * each RDD of this DStream. Applying glom() to an RDD coalesces all elements
 * within each partition into an array.
 */
DStream.prototype.glom = function() {
    return new DStream(this.getJavaObject().glom(), this.streamingContext);
}

/**
 * Return a new DStream by applying a function to all elements of this DStream.
 * @param func
 * @param {Object[]} bindArgs - Optional array whose values will be added to func's argument list.
 * @returns {DStream}
 */
DStream.prototype.map = function(func, bindArgs) {
    var fn = Utils.createLambdaFunction(func, org.eclairjs.nashorn.JSFunction, bindArgs);
    return new DStream(this.getJavaObject().map(fn), this.streamingContext);
};

/**
 * Return a new DStream by applying a function to all elements of this DStream.
 * @param func
 * @param {Object[]} bindArgs - Optional array whose values will be added to func's argument list.
 * @returns {DStream}
 */
DStream.prototype.mapToPair = function(func, bindArgs) {
    var fn = Utils.createLambdaFunction(func, org.eclairjs.nashorn.JSPairFunction, bindArgs);
    return new DStream(this.getJavaObject().mapToPair(fn), this.streamingContext);
};

/**
 * Return a new DStream in which each RDD is generated by applying
 * mapPartitions() to each RDDs of this DStream. Applying mapPartitions() to an
 * RDD applies a function to each partition of the RDD.
 * @param func
 * @param {Object[]} bindArgs - Optional array whose values will be added to func's argument list.
 * @returns {DStream}
 */
DStream.prototype.mapPartitions = function(func, bindArgs) {
    var fn = Utils.createLambdaFunction(func, org.eclairjs.nashorn.JSFlatMapFunction, bindArgs);
    return new DStream(this.getJavaObject().mapPartitions(fn),
                       this.streamingContext);
};

/**
 * Apply a function to each RDD in this DStream. This is an output operator, so 'this' DStream will be registered as an output 
 * stream and therefore materialized.
 * @param func
 * @param {Object[]} bindArgs - Optional array whose values will be added to func's argument list.
 * @returns {void}
 */
DStream.prototype.foreachRDD = function(func, bindArgs) {
    var fn = Utils.createLambdaFunction(func, org.eclairjs.nashorn.JSFunction, bindArgs);
    this.getJavaObject().foreachRDD(fn);
};

/**
 * Print the first ten elements of each RDD generated in this DStream. This is an output operator, so this DStream will be 
 * registered as an output stream and there materialized.
 * @returns {void}
 */
DStream.prototype.print = function() {
    this.getJavaObject().print();
};
<file_sep><!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="utf-8">
    <title>JSDoc: Source: HashPartitioner.js</title>
    
    <script src="scripts/prettify/prettify.js"> </script>
    <script src="scripts/prettify/lang-css.js"> </script>
    <!--[if lt IE 9]>
      <script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
    <![endif]-->
    <link type="text/css" rel="stylesheet" href="styles/prettify-tomorrow.css">
    <link type="text/css" rel="stylesheet" href="styles/jsdoc-default.css">
</head>

<body>

<div id="main">
    
    <h1 class="page-title">Source: HashPartitioner.js</h1>
    
    


    
    <section>
        <article>
            <pre class="prettyprint source"><code>/*
* Copyright 2015 IBM Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*      http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/



/**
 * @constructor
 * A {@link Partitioner} that implements hash-based partitioning using
 * Java's `Object.hashCode`.
 *
 * Java arrays have hashCodes that are based on the arrays' identities rather than their contents,
 * so attempting to partition an RDD[Array[_]] or RDD[(Array[_], _)] using a HashPartitioner will
 * produce an unexpected or incorrect result.
 * @classdesc
 * 
 */

var HashPartitioner = function(partitions) {
  var jvmObject = new org.apache.spark.HashPartitioner(partitions);
  this.logger = Logger.getLogger("HashPartitioner_js");
  JavaWrapper.call(this, jvmObject);

};

HashPartitioner.prototype = Object.create(JavaWrapper.prototype);

HashPartitioner.prototype.constructor = HashPartitioner;




HashPartitioner.prototype.numPartitions = function() {
  return  this.getJavaObject().numPartitions();
}



HashPartitioner.prototype.getPartition = function(key) {
  return  this.getJavaObject().getPartition(key);
}



HashPartitioner.prototype.equals = function(other) {
  var other_uw = Utils.unwrapObject(other);
  return  this.getJavaObject().equals(other_uw);
}



HashPartitioner.prototype.hashCode = function() {
  return  this.getJavaObject().hashCode();
}
</code></pre>
        </article>
    </section>




</div>

<nav>
    <h2><a href="index.html">Index</a></h2><h3>Classes</h3><ul><li><a href="result..html">A {@link Partitioner} that implements hash-based partitioning using
Java's `Object.hashCode`.

Java arrays have hashCodes that are based on the arrays' identities rather than their contents,
so attempting to partition an RDD[Array[_]] or RDD[(Array[_], _)] using a HashPartitioner will
produce an unexpected or incorrect result.</a></li><li><a href="Accumulator.html">Accumulator</a></li><li><a href="ArrayType.html">ArrayType</a></li><li><a href="BinaryType.html">BinaryType</a></li><li><a href="BooleanType.html">BooleanType</a></li><li><a href="ByteType.html">ByteType</a></li><li><a href="CalendarIntervalType.html">CalendarIntervalType</a></li><li><a href="Column.html">Column</a></li><li><a href="DataFrame.html">DataFrame</a></li><li><a href="DataFrameNaFunctions.html">DataFrameNaFunctions</a></li><li><a href="DataFrameReader.html">DataFrameReader</a></li><li><a href="DataFrameStatFunctions.html">DataFrameStatFunctions</a></li><li><a href="DataFrameWriter.html">DataFrameWriter</a></li><li><a href="DataType.html">DataType</a></li><li><a href="DataTypes.html">DataTypes</a></li><li><a href="DateType.html">DateType</a></li><li><a href="DecimalType.html">DecimalType</a></li><li><a href="DenseVector.html">DenseVector</a></li><li><a href="DoubleType.html">DoubleType</a></li><li><a href="DStream.html">DStream</a></li><li><a href="Duration.html">Duration</a></li><li><a href="FloatType.html">FloatType</a></li><li><a href="functions.html">functions</a></li><li><a href="GroupedData.html">GroupedData</a></li><li><a href="IntegerType.html">IntegerType</a></li><li><a href="LabeledPoint.html">LabeledPoint</a></li><li><a href="LinearRegressionModel.html">LinearRegressionModel</a></li><li><a href="LinearRegressionWithSGD.html">LinearRegressionWithSGD</a></li><li><a href="LongType.html">LongType</a></li><li><a href="MapType.html">MapType</a></li><li><a href="Metadata.html">Metadata</a></li><li><a href="NullType.html">NullType</a></li><li><a href="NumericType.html">NumericType</a></li><li><a href="PartialResult.html">PartialResult</a></li><li><a href="Partitioner.html">Partitioner</a></li><li><a href="RangePartitioner.html">RangePartitioner</a></li><li><a href="RDD.html">RDD</a></li><li><a href="Row.html">Row</a></li><li><a href="RowFactory.html">RowFactory</a></li><li><a href="ShortType.html">ShortType</a></li><li><a href="SparkConf.html">SparkConf</a></li><li><a href="SparkContext.html">SparkContext</a></li><li><a href="SparkFiles.html">SparkFiles</a></li><li><a href="SQLContext.html">SQLContext</a></li><li><a href="SQLContext.QueryExecution.html">QueryExecution</a></li><li><a href="SQLContext.SparkPlanner.html">SparkPlanner</a></li><li><a href="SQLContext.SQLSession.html">SQLSession</a></li><li><a href="SqlDate.html">SqlDate</a></li><li><a href="SqlTimestamp.html">SqlTimestamp</a></li><li><a href="StorageLevel.html">StorageLevel</a></li><li><a href="StreamingContext.html">StreamingContext</a></li><li><a href="StringType.html">StringType</a></li><li><a href="StructField.html">StructField</a></li><li><a href="StructType.html">StructType</a></li><li><a href="Time.html">Time</a></li><li><a href="TimestampType.html">TimestampType</a></li><li><a href="Vector.html">Vector</a></li></ul>
</nav>

<br clear="both">

<footer>
    Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 3.2.3-dev</a> on Tue Jan 19 2016 17:58:48 GMT-0500 (EST)
</footer>

<script> prettyPrint(); </script>
<script src="scripts/linenumber.js"> </script>
</body>
</html>
<file_sep>/*
 * Copyright 2015 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

var Utils = {};

Utils.logger = Logger.getLogger("Utils_js");

  /**
   * This function needs to parse the arguments that are being passed to the LAMDA function
   * and get references to the arguments that will need to be added to the closer of the Nashorn
   * engine when the LAMDA function runs on the worker. A standard spark LAMDA would look like:
   * function(sparkArg){...}, but we need any variables declared outside the closer of the LAMDA
   * to be passed into the LAMDA so we can add them to the args when we call the LAMDA function from
   * a new Nashorn engine context. Are LAMDA function must include the out of closer variables ex.
   * function(sparkArg, scopeArg1, scopeArg2, .....)
   * @param  {function} func LAMDA function that will be passed to spark. The functions
   *                      will have the format function(sparkArg, scopeArg1, scopeArg2, .....)
   * @param  {sparkArgumentsPassed} the number of arguments passed to the LAMDA by spark defaults to 1
   *
   * @return {Object}   {
   * 						funcStr:  stringified funciton that was passed in,
   *                        scopeVars: Array of references to the out of closer args
   *                      }
   */
  Utils.createJavaParams = function(func, sparkArgumentsPassed) {
	  Utils.logger.debug("createJavaParams func: " + func + " sparkArgumentsPassed: " + sparkArgumentsPassed);
	  var scopeVarsStartingPosion = sparkArgumentsPassed ? sparkArgumentsPassed : 1;
  	  var parmas = {};
  	  parmas.scopeVars = null;
  	  /*
  	   * First we stringify the function
  	   */
  	  parmas.funcStr = func.toString();
      /*
       * Start parsing the arguments passed to the function
       */
      var start = parmas.funcStr.indexOf("(");
      var stop = parmas.funcStr.indexOf(")");
      var agrsStr = parmas.funcStr.substring(start +1, stop);
      var args = agrsStr.split(","); // get all the arguments names
      parmas.scopeVars = [];
      for (var i = scopeVarsStartingPosion; i < args.length; i++) {
    	  // unwrapObjects or we can have serialization problems
    	  Utils.logger.debug("scopeVar = " + args[i]);
    	  var a = eval(args[i]);
    	  Utils.logger.debug("got a ref to  = " + a);
		  parmas.scopeVars.push(Utils.unwrapObject(eval(args[i]))); // eval the argument name to get a reference to the variable
	  }
      return parmas;

  };

  Utils.javaToJs = function(javaObj) {
      if (Array.isArray(javaObj) || javaObj.getClass().isArray() )
      {
          var ret = [];
          for (var i=0;i<javaObj.length;i++) {
              ret.push(org.eclairjs.nashorn.Utils.javaToJs(javaObj[i], org.eclairjs.nashorn.NashornEngineSingleton.getEngine()));
          }
          return ret;

      }
      else
          return org.eclairjs.nashorn.Utils.javaToJs(javaObj,org.eclairjs.nashorn.NashornEngineSingleton.getEngine());
  };

  Utils.unwrapObject = function(obj) {
      if (Array.isArray(obj))
      {
          var unObj = [];
          for (var i=0;i<obj.length;i++) {
              unObj.push(Utils.unwrapObject(obj[i]));
          }
          return unObj;
      }
      else
          return (obj && obj.getJavaObject) ? obj.getJavaObject() : obj;
  };


  Utils.unwrapTuple = function(obj) {
        Utils.logger.debug("unwrapTuple = " + obj);
    if (Array.isArray(obj) && obj.length>1)
    {
      var Tuple2 = Java.type('scala.Tuple2');
       return new Tuple2(Utils.unwrapObject(obj[0]),Utils.unwrapObject(obj[1]));
    }
    else
      throw "Expecting tuple, i.e. [1,2] ";
  };

  /**
   * Creates a argument list of Spark Java objects that can be passed to a Spark Java method.
   * If the objects passed in the argument list are an instanceof "type" then the object will be
   * unwrapped else will will create an instanceof "type" for that object.
   * If the object
   * for example:
   * // Spark Java
   * GoupedData.agg(Column expr, Column... exprs)
   * @private
   * @param {object | string} object,...object
   * @param {function} type this is the constructor of the desired object type for example Column
   * @returns {object[]} array of Java spark objects
   */
  Utils.createJavaObjectArguments = function(args, type) {
	  /*
		 * First convert any strings to Objects of type
		 */
		var a = Array.prototype.slice.call(args);
		for (var i = 0; i < a.length; i++) {
			var o = a[i];
			if (!(o instanceof type)) {
				o = new type(o);
			}
			a[i] = Utils.unwrapObject(o);
		}
		return a;
  };
  /**
   * Creates a Java HashMap from a JavaScript object.
   * @private
   * @param {object} obj hashMap
   * @returns {HashMap} java.util.HashMap
   */
  Utils.createJavaHashMap = function(obj, javaMapObj) {
	  var map = javaMapObj ? javaMapObj : new java.util.HashMap();
	   for(var colName in obj){
		   if (typeof obj[colName] === 'number') {
			   map.put(new java.lang.Double(colName), new java.lang.Double(obj[colName]));
		   } else {
			   map.put(colName, obj[colName]);
		   }

	   }
	   return map;
  };

  function convertJavaTuple2(o1, o2) {
	  return [o1 ,o2];
  };

  function convertJavaJSONObject(str) {
	  return JSON.parse(str);
  };

  function createJavaWrapperObject(className, obj) {
	  return eval("new " + className + "(obj)");
  };


  Utils.createLambdaFunction = function(func, clazz, bindArgs) {
    return new clazz(func.toString(), bindArgs ? Utils.unwrapObject(bindArgs) : [])
  };<file_sep>/*                                                                         
* Copyright 2016 IBM Corp.                                                 
*                                                                          
* Licensed under the Apache License, Version 2.0 (the "License");          
* you may not use this file except in compliance with the License.         
* You may obtain a copy of the License at                                  
*                                                                          
*      http://www.apache.org/licenses/LICENSE-2.0                          
*                                                                          
* Unless required by applicable law or agreed to in writing, software      
* distributed under the License is distributed on an "AS IS" BASIS,        
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
* See the License for the specific language governing permissions and      
* limitations under the License.                                           
*/ 



/**
 * Model representing the result of matrix factorization.
 *
 * Note: If you create the model directly using constructor, please be aware that fast prediction
 * requires cached user/product features and their associated partitioners.
 *
 * @param rank Rank for the features in this model.
 * @param userFeatures RDD of tuples where each tuple represents the userId and
 *                     the features computed for this user.
 * @param productFeatures RDD of tuples where each tuple represents the productId
 *                        and the features computed for this product.
 * @classdesc
 */

/**
 * @param {number} rank
 * @param {RDD} userFeatures
 * @param {RDD} productFeatures
 *  @class
 */
var MatrixFactorizationModel = function() {
	this.logger = Logger.getLogger("MatrixFactorizationModel_js");
	var jvmObject = arguments[0];
	if (arguments.length > 1) {
		var userFeatures = Utils.unwrapObject(arguments[1]);
		var productFeatures = Utils.unwrapObject(arguments[2]);
		jvmObject = new org.apache.spark.mllib.recommendation.MatrixFactorizationModel(arguments[0],userFeatures,productFeatures);
	}

	 JavaWrapper.call(this, jvmObject);

};

MatrixFactorizationModel.prototype = Object.create(JavaWrapper.prototype);

MatrixFactorizationModel.prototype.constructor = MatrixFactorizationModel;



/**
 * @param {number} user
 * @param {number} product
 * @returns {number} 
 */
MatrixFactorizationModel.prototype.predict0 = function(user,product) {
throw "not implemented by ElairJS";
//   return  this.getJavaObject().predict(user,product);
};


/**
 * Predict the rating of many users for many products.
 * The output RDD has an element per each element in the input RDD (including all duplicates)
 * unless a user or product is missing in the training set.
 *
 * @param {RDD} usersProducts   RDD of (user, product) pairs.
 * @returns {RDD}  RDD of Ratings.
 */
MatrixFactorizationModel.prototype.predict1 = function(usersProducts) {
throw "not implemented by ElairJS";
// // TODO: handle Tuple conversion for 'usersProducts'
//   var usersProducts_uw = Utils.unwrapObject(usersProducts);
//   var javaObject =  this.getJavaObject().predict(usersProducts_uw);
//   return new RDD(javaObject);
};


/**
 * Java-friendly version of {@link predict}.
 * @param {JavaPairRDD} usersProducts
 * @returns {JavaRDD} 
 */
MatrixFactorizationModel.prototype.predict2 = function(usersProducts) {
throw "not implemented by ElairJS";
//   var usersProducts_uw = Utils.unwrapObject(usersProducts);
//   var javaObject =  this.getJavaObject().predict(usersProducts_uw);
//   return new JavaRDD(javaObject);
};


/**
 * Recommends products to a user.
 *
 * @param {number} user  the user to recommend products to
 * @param {number} num  how many products to return. The number returned may be less than this.
 *  "score" in the rating field. Each represents one recommended product, and they are sorted
 *  by score, decreasing. The first returned is the one predicted to be most strongly
 *  recommended to the user. The score is an opaque value that indicates how strongly
 *  recommended the product is.
 * @returns {Rating[]}  [[Rating]] objects, each of which contains the given user ID, a product ID, and a
 */
MatrixFactorizationModel.prototype.recommendProducts = function(user,num) {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().recommendProducts(user,num);
//   return Utils.javaToJs(javaObject);
};


/**
 * Recommends users to a product. That is, this returns users who are most likely to be
 * interested in a product.
 *
 * @param {number} product  the product to recommend users to
 * @param {number} num  how many users to return. The number returned may be less than this.
 *  "score" in the rating field. Each represents one recommended user, and they are sorted
 *  by score, decreasing. The first returned is the one predicted to be most strongly
 *  recommended to the product. The score is an opaque value that indicates how strongly
 *  recommended the user is.
 * @returns {Rating[]}  [[Rating]] objects, each of which contains a user ID, the given product ID, and a
 */
MatrixFactorizationModel.prototype.recommendUsers = function(product,num) {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().recommendUsers(product,num);
//   return Utils.javaToJs(javaObject);
};


/**
 * Save this model to the given path.
 *
 * This saves:
 *  - human-readable (JSON) model metadata to path/metadata/
 *  - Parquet formatted data to path/data/
 *
 * The model may be loaded using {@link load}.
 *
 * @param {SparkContext} sc   Spark context used to save model data.
 * @param {string} path   Path specifying the directory in which to save this model.
 *              If the directory already exists, this method throws an exception.
 */
MatrixFactorizationModel.prototype.save = function(sc,path) {
throw "not implemented by ElairJS";
//   var sc_uw = Utils.unwrapObject(sc);
//    this.getJavaObject().save(sc_uw,path);
};


/**
 * Recommends topK products for all users.
 *
 * @param {number} num  how many products to return for every user.
 * rating objects which contains the same userId, recommended productID and a "score" in the
 * rating field. Semantics of score is same as recommendProducts API
 * @returns {RDD}  [(Int, Array[Rating])] objects, where every tuple contains a userID and an array of
 */
MatrixFactorizationModel.prototype.recommendProductsForUsers = function(num) {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().recommendProductsForUsers(num);
//   return new RDD(javaObject);
};


/**
 * Recommends topK users for all products.
 *
 * @param {number} num  how many users to return for every product.
 * of rating objects which contains the recommended userId, same productID and a "score" in the
 * rating field. Semantics of score is same as recommendUsers API
 * @returns {RDD}  [(Int, Array[Rating])] objects, where every tuple contains a productID and an array
 */
MatrixFactorizationModel.prototype.recommendUsersForProducts = function(num) {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().recommendUsersForProducts(num);
//   return new RDD(javaObject);
};
/**
 *
 * @returns {RDD}
 */
MatrixFactorizationModel.prototype.userFeatures = function() {
   var javaObject =  this.getJavaObject().userFeatures();
   return new RDD(javaObject.toJavaRDD());
};
/**
 *
 * @returns {RDD}
 */
MatrixFactorizationModel.prototype.productFeatures = function() {
    var javaObject =  this.getJavaObject().productFeatures();
    return new RDD(javaObject.toJavaRDD());
};


//
// static methods
//


/**
 * Load a model from the given path.
 *
 * The model should have been saved by {@link save}.
 *
 * @param {SparkContext} sc   Spark context used for loading model files.
 * @param {string} path   Path specifying the directory to which the model was saved.
 * @returns {MatrixFactorizationModel}   Model instance
 */
MatrixFactorizationModel.load = function(sc,path) {
throw "not implemented by ElairJS";
//   var sc_uw = Utils.unwrapObject(sc);
//   var javaObject =  org.apache.spark.mllib.recommendation.MatrixFactorizationModel.load(sc_uw,path);
//   return new MatrixFactorizationModel(javaObject);
};

<file_sep>/*
* Copyright 2015 IBM Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*      http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/



/**
 * @constructor
 * @classdesc An object that defines how the elements in a key-value pair RDD are partitioned by key.
 * Maps each key to a partition ID, from 0 to `numPartitions - 1`.
 */
var Partitioner = function(jvmObject) {
  throw "con't do 'new' on abstract class 'Partitioner'"
    // this.logger = Logger.getLogger("Partitioner_js");
    // JavaWrapper.call(this, jvmObject);
};

Partitioner.prototype = Object.create(JavaWrapper.prototype);

Partitioner.prototype.constructor = Partitioner;




Partitioner.prototype.numPartitions = function() {
throw "not implemented by ElairJS";
//   return  this.getJavaObject().numPartitions();
}



Partitioner.prototype.getPartition = function(key) {
throw "not implemented by ElairJS";
//   return  this.getJavaObject().getPartition(key);
}


/**
 * @constructor
 * @classdesc A {@link Partitioner} that implements hash-based partitioning using
 * Java's `Object.hashCode`.
 *
 * Java arrays have hashCodes that are based on the arrays' identities rather than their contents,
 * so attempting to partition an RDD[Array[_]] or RDD[(Array[_], _)] using a HashPartitioner will
 * produce an unexpected or incorrect result.
 * 
 */

var HashPartitioner = function(partitions) {
  var jvmObject = new org.apache.spark.HashPartitioner(partitions);
  this.logger = Logger.getLogger("HashPartitioner_js");
  JavaWrapper.call(this, jvmObject);

};

HashPartitioner.prototype = Object.create(JavaWrapper.prototype);

HashPartitioner.prototype.constructor = HashPartitioner;




HashPartitioner.prototype.numPartitions = function() {
  return  this.getJavaObject().numPartitions();
}



HashPartitioner.prototype.getPartition = function(key) {
  return  this.getJavaObject().getPartition(key);
}



HashPartitioner.prototype.equals = function(other) {
  var other_uw = Utils.unwrapObject(other);
  return  this.getJavaObject().equals(other_uw);
}



HashPartitioner.prototype.hashCode = function() {
  return  this.getJavaObject().hashCode();
}


/**
 * @constructor
 * @classdesc A {@link Partitioner} that partitions sortable records by range into roughly
 * equal ranges. The ranges are determined by sampling the content of the RDD passed in.
 *
 * Note that the actual number of partitions created by the RangePartitioner might not be the same
 * as the `partitions` parameter, in the case where the number of sampled records is less than
 * the value of `partitions`.
 * 
 */

var RangePartitioner = function(partitions,rdd,ascending) {
  var jvmObject = new org.apache.spark.RangePartitioner(partitions,rdd,ascending);
  this.logger = Logger.getLogger("RangePartitioner_js");
  JavaWrapper.call(this, jvmObject);

};

RangePartitioner.prototype = Object.create(JavaWrapper.prototype);

RangePartitioner.prototype.constructor = RangePartitioner;




RangePartitioner.prototype.numPartitions = function() {
  return  this.getJavaObject().numPartitions();
}



RangePartitioner.prototype.getPartition = function(key) {
  return  this.getJavaObject().getPartition(key);
}



RangePartitioner.prototype.equals = function(other) {
  var other_uw = Utils.unwrapObject(other);
  return  this.getJavaObject().equals(other_uw);
}



RangePartitioner.prototype.hashCode = function() {
  return  this.getJavaObject().hashCode();
}
//
// static methods
//


/**
 * Choose a partitioner to use for a cogroup-like operation between a number of RDDs.
 *
 * If any of the RDDs already has a partitioner, choose that one.
 *
 * Otherwise, we use a default HashPartitioner. For the number of partitions, if
 * spark.default.parallelism is set, then we'll use the value from SparkContext
 * defaultParallelism, otherwise we'll use the max number of upstream partitions.
 *
 * Unless spark.default.parallelism is set, the number of partitions will be the
 * same as the number of partitions in the largest upstream RDD, as this should
 * be least likely to cause out-of-memory errors.
 *
 * We use two method parameters (rdd, others) to enforce callers passing at least 1 RDD.
 * @returns {Partitioner}
 */
Partitioner.defaultPartitioner = function(rdd,others) {
  var rdd_uw = Utils.unwrapObject(rdd);
  var others_uw = Utils.unwrapObject(others);
  var javaObject =  org.apache.spark.HashPartitioner.defaultPartitioner(rdd_uw,others_uw);
  return Utils.javaToJs(javaObject);
}
<file_sep><!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="utf-8">
    <title>JSDoc: Source: mllib/feature/Word2Vec.js</title>
    
    <script src="scripts/prettify/prettify.js"> </script>
    <script src="scripts/prettify/lang-css.js"> </script>
    <!--[if lt IE 9]>
      <script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
    <![endif]-->
    <link type="text/css" rel="stylesheet" href="styles/prettify-tomorrow.css">
    <link type="text/css" rel="stylesheet" href="styles/jsdoc-default.css">
</head>

<body>

<div id="main">
    
    <h1 class="page-title">Source: mllib/feature/Word2Vec.js</h1>
    
    


    
    <section>
        <article>
            <pre class="prettyprint source"><code>/*                                                                         
* Copyright 2016 IBM Corp.                                                 
*                                                                          
* Licensed under the Apache License, Version 2.0 (the "License");          
* you may not use this file except in compliance with the License.         
* You may obtain a copy of the License at                                  
*                                                                          
*      http://www.apache.org/licenses/LICENSE-2.0                          
*                                                                          
* Unless required by applicable law or agreed to in writing, software      
* distributed under the License is distributed on an "AS IS" BASIS,        
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
* See the License for the specific language governing permissions and      
* limitations under the License.                                           
*/ 



/**
 * Word2Vec creates vector representation of words in a text corpus.
 * The algorithm first constructs a vocabulary from the corpus
 * and then learns vector representation of words in the vocabulary.
 * The vector representation can be used as features in
 * natural language processing and machine learning algorithms.
 *
 * We used skip-gram model in our implementation and hierarchical softmax
 * method to train the model. The variable names in the implementation
 * matches the original C implementation.
 *
 * For original C implementation, see https://code.google.com/p/word2vec/
 * For research papers, see
 * Efficient Estimation of Word Representations in Vector Space
 * and
 * Distributed Representations of Words and Phrases and their Compositionality.
 * @classdesc
 *  @class
 */
var Word2Vec = function(jvmObject) {
	 
	 this.logger = Logger.getLogger("Word2Vec_js");
	if (!jvmObject) {
		jvmObject = new org.apache.spark.mllib.feature.Word2Vec();
	}
	 JavaWrapper.call(this, jvmObject);

};

Word2Vec.prototype = Object.create(JavaWrapper.prototype);

Word2Vec.prototype.constructor = Word2Vec;



/**
 * Sets vector size (default: 100).
 * @param {number} vectorSize
 * @returns {} 
 */
Word2Vec.prototype.setVectorSize = function(vectorSize) {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().setVectorSize(vectorSize);
//   return new (javaObject);
};


/**
 * Sets initial learning rate (default: 0.025).
 * @param {number} learningRate
 * @returns {} 
 */
Word2Vec.prototype.setLearningRate = function(learningRate) {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().setLearningRate(learningRate);
//   return new (javaObject);
};


/**
 * Sets number of partitions (default: 1). Use a small number for accuracy.
 * @param {number} numPartitions
 * @returns {} 
 */
Word2Vec.prototype.setNumPartitions = function(numPartitions) {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().setNumPartitions(numPartitions);
//   return new (javaObject);
};


/**
 * Sets number of iterations (default: 1), which should be smaller than or equal to number of
 * partitions.
 * @param {number} numIterations
 * @returns {} 
 */
Word2Vec.prototype.setNumIterations = function(numIterations) {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().setNumIterations(numIterations);
//   return new (javaObject);
};


/**
 * Sets random seed (default: a random long integer).
 * @param {number} seed
 * @returns {} 
 */
Word2Vec.prototype.setSeed = function(seed) {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().setSeed(seed);
//   return new (javaObject);
};


/**
 * Sets the window of words (default: 5)
 * @param {number} window
 * @returns {} 
 */
Word2Vec.prototype.setWindowSize = function(window) {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().setWindowSize(window);
//   return new (javaObject);
};


/**
 * Sets minCount, the minimum number of times a token must appear to be included in the word2vec
 * model's vocabulary (default: 5).
 * @param {number} minCount
 * @returns {} 
 */
Word2Vec.prototype.setMinCount = function(minCount) {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().setMinCount(minCount);
//   return new (javaObject);
};


/**
 * Computes the vector representation of each word in vocabulary.
 * @param {RDD} dataset  an RDD of words
 * @returns {Word2VecModel}  a Word2VecModel
 */
Word2Vec.prototype.fit = function(dataset) {
   var dataset_uw = Utils.unwrapObject(dataset);
   var javaObject =  this.getJavaObject().fit(dataset_uw);
   return new Word2VecModel(javaObject);
};

/**
 * Word2Vec model
 * @param wordIndex maps each word to an index, which can retrieve the corresponding
 *                  vector from wordVectors
 * @param wordVectors array of length numWords * vectorSize, vector corresponding
 *                    to the word mapped with index i can be retrieved by the slice
 *                    (i * vectorSize, i * vectorSize + vectorSize)
 * @classdesc
 */

/**
 * @param {Map} model
 * @returns {??} 
 *  @class
 */
var Word2VecModel = function(jvmObject) {

	 //var jvmObject = new org.apache.spark.mllib.feature.Word2VecModel(model);
	 this.logger = Logger.getLogger("Word2VecModel_js");
	 JavaWrapper.call(this, jvmObject);

};

Word2VecModel.prototype = Object.create(JavaWrapper.prototype);

Word2VecModel.prototype.constructor = Word2VecModel;



/**
 * @param {SparkContext} sc
 * @param {string} path
 */
Word2VecModel.prototype.save = function(sc,path) {
throw "not implemented by ElairJS";
//   var sc_uw = Utils.unwrapObject(sc);
//    this.getJavaObject().save(sc_uw,path);
};


/**
 * Transforms a word to its vector representation
 * @param {string} word  a word
 * @returns {Vector}  vector representation of word
 */
Word2VecModel.prototype.transform = function(word) {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().transform(word);
//   return Utils.javaToJs(javaObject);
};


/**
 * Find synonyms of a word
 * @param {string} word  a word
 * @param {number} num  number of synonyms to find
 * @returns {Tuple2[]}  array of (word, cosineSimilarity)
 */
Word2VecModel.prototype.findSynonyms = function(word,num) {
    var javaObject =  this.getJavaObject().findSynonyms(word,num);
    return Utils.javaToJs(javaObject);
};


/**
 * Find synonyms of the vector representation of a word
 * @param {Vector} vector  vector representation of a word
 * @param {number} num  number of synonyms to find
 * @returns {Tuple2[]}  array of (word, cosineSimilarity)
 */
Word2VecModel.prototype.findSynonymswithnumber = function(vector,num) {
throw "not implemented by ElairJS";
//   var vector_uw = Utils.unwrapObject(vector);
//   var javaObject =  this.getJavaObject().findSynonyms(vector_uw,num);
//   return Utils.javaToJs(javaObject);
};


/**
 * Returns a map of words to their vector representations.
 * @returns {Map} 
 */
Word2VecModel.prototype.getVectors = function() {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().getVectors();
//   return new Map(javaObject);
};
//
// static methods
//


/**
 * @param {SparkContext} sc
 * @param {string} path
 * @returns {Word2VecModel} 
 */
Word2VecModel.load = function(sc,path) {
throw "not implemented by ElairJS";
//   var sc_uw = Utils.unwrapObject(sc);
//   var javaObject =  org.apache.spark.mllib.feature.Word2VecModel.load(sc_uw,path);
//   return new Word2VecModel(javaObject);
};
</code></pre>
        </article>
    </section>




</div>

<nav>
    <h2><a href="index.html">Index</a></h2><h3>Classes</h3><ul><li><a href="Accumulable.html">Accumulable</a></li><li><a href="AccumulableParam.html">AccumulableParam</a></li><li><a href="Accumulator.html">Accumulator</a></li><li><a href="ALS.html">ALS</a></li><li><a href="ArrayType.html">ArrayType</a></li><li><a href="AssociationRules.html">AssociationRules</a></li><li><a href="BinaryType.html">BinaryType</a></li><li><a href="BisectingKMeans.html">BisectingKMeans</a></li><li><a href="BisectingKMeansModel.html">BisectingKMeansModel</a></li><li><a href="BooleanType.html">BooleanType</a></li><li><a href="CalendarIntervalType.html">CalendarIntervalType</a></li><li><a href="Column.html">Column</a></li><li><a href="DataFrame.html">DataFrame</a></li><li><a href="DataFrameNaFunctions.html">DataFrameNaFunctions</a></li><li><a href="DataFrameReader.html">DataFrameReader</a></li><li><a href="DataFrameStatFunctions.html">DataFrameStatFunctions</a></li><li><a href="DataFrameWriter.html">DataFrameWriter</a></li><li><a href="DataType.html">DataType</a></li><li><a href="DataTypes.html">DataTypes</a></li><li><a href="DateType.html">DateType</a></li><li><a href="DenseVector.html">DenseVector</a></li><li><a href="DoubleType.html">DoubleType</a></li><li><a href="DStream.html">DStream</a></li><li><a href="Duration.html">Duration</a></li><li><a href="FloatAccumulatorParam.html">FloatAccumulatorParam</a></li><li><a href="FloatType.html">FloatType</a></li><li><a href="FPGrowth.html">FPGrowth</a></li><li><a href="FPGrowthModel.html">FPGrowthModel</a></li><li><a href="FreqItemset.html">FreqItemset</a></li><li><a href="functions.html">functions</a></li><li><a href="FutureAction.html">FutureAction</a></li><li><a href="GroupedData.html">GroupedData</a></li><li><a href="HashPartitioner.html">HashPartitioner</a></li><li><a href="IntAccumulatorParam.html">IntAccumulatorParam</a></li><li><a href="IntegerType.html">IntegerType</a></li><li><a href="LabeledPoint.html">LabeledPoint</a></li><li><a href="LinearRegressionModel.html">LinearRegressionModel</a></li><li><a href="LinearRegressionWithSGD.html">LinearRegressionWithSGD</a></li><li><a href="List.html">List</a></li><li><a href="MapType.html">MapType</a></li><li><a href="MatrixFactorizationModel.html">MatrixFactorizationModel</a></li><li><a href="Metadata.html">Metadata</a></li><li><a href="MLWord2Vec.html">MLWord2Vec</a></li><li><a href="MLWord2VecModel.html">MLWord2VecModel</a></li><li><a href="NullType.html">NullType</a></li><li><a href="NumericType.html">NumericType</a></li><li><a href="PartialResult.html">PartialResult</a></li><li><a href="Partitioner.html">Partitioner</a></li><li><a href="RangePartitioner.html">RangePartitioner</a></li><li><a href="Rating.html">Rating</a></li><li><a href="RDD.html">RDD</a></li><li><a href="Row.html">Row</a></li><li><a href="RowFactory.html">RowFactory</a></li><li><a href="Rule.html">Rule</a></li><li><a href="SparkConf.html">SparkConf</a></li><li><a href="SparkContext.html">SparkContext</a></li><li><a href="SparkFiles.html">SparkFiles</a></li><li><a href="SparkStatusTracker.html">SparkStatusTracker</a></li><li><a href="SparseVector.html">SparseVector</a></li><li><a href="SQLContext.html">SQLContext</a></li><li><a href="SQLContext.QueryExecution.html">QueryExecution</a></li><li><a href="SQLContext.SparkPlanner.html">SparkPlanner</a></li><li><a href="SQLContext.SQLSession.html">SQLSession</a></li><li><a href="SqlDate.html">SqlDate</a></li><li><a href="SqlTimestamp.html">SqlTimestamp</a></li><li><a href="StorageLevel.html">StorageLevel</a></li><li><a href="StreamingContext.html">StreamingContext</a></li><li><a href="StringType.html">StringType</a></li><li><a href="StructField.html">StructField</a></li><li><a href="StructType.html">StructType</a></li><li><a href="Time.html">Time</a></li><li><a href="TimestampType.html">TimestampType</a></li><li><a href="Vectors.html">Vectors</a></li><li><a href="VectorUDT.html">VectorUDT</a></li><li><a href="Word2Vec.html">Word2Vec</a></li><li><a href="Word2VecModel.html">Word2VecModel</a></li></ul><h3>Global</h3><ul><li><a href="global.html#Vector">Vector</a></li></ul>
</nav>

<br clear="both">

<footer>
    Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 3.2.3-dev</a> on Thu Feb 18 2016 17:00:52 GMT-0500 (EST)
</footer>

<script> prettyPrint(); </script>
<script src="scripts/linenumber.js"> </script>
</body>
</html>
<file_sep>/*
 * Copyright 2016 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

var LogisticRegressionModel = function(jvmObj) {
    JavaWrapper.call(this, jvmObj);
};

LogisticRegressionModel.prototype = Object.create(JavaWrapper.prototype); 
LogisticRegressionModel.prototype.constructor = LogisticRegressionModel;

LogisticRegressionModel.prototype.clearThreshold = function() {
    this.getJavaObject().clearThreshold();
};

LogisticRegressionModel.prototype.predict = function(testData) {
	var p = this.getJavaObject().predict(Utils.unwrapObject(testData));
    return p;
}


var LogisticRegressionWithLBFGS = function(jvmObj) {

    if(jvmObj == undefined) {
        jvmObj = 
            new org.apache.spark.mllib.classification.LogisticRegressionWithLBFGS();
    }

    JavaWrapper.call(this, jvmObj);
};

LogisticRegressionWithLBFGS.prototype = Object.create(JavaWrapper.prototype);

LogisticRegressionWithLBFGS.prototype.setNumClasses = function(n) {
    return new LogisticRegressionWithLBFGS(
        this.getJavaObject().setNumClasses(n));
};

LogisticRegressionWithLBFGS.prototype.run = function(input, initialWeights) {
    if(initialWeights == undefined) {
        return new LogisticRegressionModel(
            this.getJavaObject().run(input.getJavaObject().rdd()));
    } else {
        return new LogisticRegressionModel(
            this.getJavaObject().run(input.getJavaObject().rdd(),
                                     initialWeights.getJavaObject()));
    }
};
<file_sep>/*
 * Copyright 2016 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */




  var exampleApacheLogs = [
    "10.10.10.10 - \"FRED\" [18/Jan/2013:17:56:07 +1100] \"GET http://images.com/2013/Generic.jpg " +
      "HTTP/1.1\" 304 315 \"http://referall.com/\" \"Mozilla/4.0 (compatible; MSIE 7.0; " +
      "Windows NT 5.1; GTB7.4; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.04506.648; " +
      ".NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 1.0.3705; .NET CLR 1.1.4322; .NET CLR " +
      "3.5.30729; Release=ARP)\" \"UD-1\" - \"image/jpeg\" \"whatever\" 0.350 \"-\" - \"\" 265 923 934 \"\" " +
      "192.168.3.11 images.com 1358492167 - Whatup",
    "10.10.10.10 - \"FRED\" [18/Jan/2013:18:02:37 +1100] \"GET http://images.com/2013/Generic.jpg " +
      "HTTP/1.1\" 304 306 \"http:/referall.com\" \"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; " +
      "GTB7.4; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.04506.648; .NET CLR " +
      "3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 1.0.3705; .NET CLR 1.1.4322; .NET CLR  " +
      "3.5.30729; Release=ARP)\" \"UD-1\" - \"image/jpeg\" \"whatever\" 0.352 \"-\" - \"\" 256 977 988 \"\" " +
      "0 172.16.31.10 images.com 1358492557 - Whatup"];



  var apacheLogRegex =
    /^([\d.]+) (\S+) (\S+) \[([\w\d:/]+\s[+\-]\d{4})\] "(.+?)" (\d{3}) ([\d\-]+) "([^"]+)" "([^"]+)"/;



function extractStats(line)
{
 var match = line.match(apacheLogRegex);
 if (match)
 {
      var bytes = 0+match[7];
      return { count:1, bytes:bytes};
  } else {
      return { count:1, bytes:0};
 }

}



function extractKey(line) {
 var match = line.match(apacheLogRegex);
 if (match)
 {
      var ip = match[1];
      var user = match[3];
      var query = match[5];
      if (user !='"-"') {
        return { ip:ip, user:user, query:query};
      }
    }
    return {ip:null, user:null, query:null};
  }


var conf = new SparkConf().setAppName("JavaScript Log Query").setMaster("local[*]");
var sparkContext = new SparkContext(conf);

var dataSet = (arguments.length == 1) ? sparkContext.textFile(arguments[0]) : sparkContext.parallelize(exampleApacheLogs);

var extracted = dataSet.mapToPair(function(line){
  return [extractKey(line),extractStats(line)];
});

 var counts = extracted.reduceByKey(function(stats1,stats2){
    return {count:stats1.count+stats2.count,bytes:stats1.bytes+stats2.bytes};
 });

var output = counts.collect();

for (var i=0;i<output.length;i++)
  print("user="+JSON.stringify(output[i][0])+"\tbytes="+output[i][1].bytes+"\tn="+output[i][1].count);


sparkContext.stop();


<file_sep>/*
 * Copyright 2015 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.eclairjs.nashorn;

import org.junit.Test;
import static org.junit.Assert.*;

import javax.script.Invocable;
import javax.script.ScriptEngine;

public class MlLibTest {

	@Test
    public void LinearRegressionTest() throws Exception {
        ScriptEngine engine = TestUtils.getEngine();
        String file = TestUtils.resourceToFile("/data/mllib/lpsa.data");

        TestUtils.evalJSResource(engine, "/mllib/mllibtest.js");
        Object ret = ((Invocable)engine).invokeFunction("LinearRegressionWithSGDTest", file);
        
        String expected = "34.802055592544406,-0.4307829,32.26369105545771,-0.1625189,27.073768640300933,-0.1625189,32.956369807610656,-0.1625189,26.589176152816094,0.3715636,34.161678328568854,0.7654678,24.3647041765334,0.8544153,26.661937949806784,1.2669476,28.790597957841044,1.2669476,20.51350661135643,1.2669476";
        assertEquals("failure - strings are not equal", expected, ret.toString());

    }

    /*
        tests
        SparkContext.parallelize([FreqItemset])
        FreqItemset()
        AssociationRules()
        AssociationRules.setMinConfidence()
        AssociationRules.run()
        Rule.antecedent()
        Rule.consequent()
        Rule.confidence()
        RDD.collect()
     */
    @Test
    public void AssociationRulesTest() throws Exception {
        ScriptEngine engine = TestUtils.getEngine();
        //String file = TestUtils.resourceToFile("/data/mllib/lpsa.data");

        TestUtils.evalJSResource(engine, "/mllib/mllibtest.js");
        Object ret = ((Invocable)engine).invokeFunction("AssociationRulesTest");

        String expected = "[a] => [b], 0.8";
        assertEquals("failure - strings are not equal", expected, ret.toString());

    }

    @Test
    public void BisectingKMeansExample() throws Exception {
        ScriptEngine engine = TestUtils.getEngine();
        //String file = TestUtils.resourceToFile("/data/mllib/lpsa.data");

        TestUtils.evalJSResource(engine, "/mllib/mllibtest.js");
        Object ret = ((Invocable)engine).invokeFunction("BisectingKMeansExample");

        String expected = "{\"Compute_Cost\":0.07999999999994545," +
                "\"Cluster_Center_0\":\"[0.2]\",\"Cluster_Center_1\":\"[10.2]\"," +
                "\"Cluster_Center_2\":\"[20.200000000000003]\",\"Cluster_Center_3\":\"[30.200000000000003]\"}";

        assertEquals("failure - strings are not equal", expected, ret.toString());

    }
    
}
<file_sep>/*
 * Copyright 2016 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


/**
 * Computes the PageRank of URLs from an input file. Input file should
 * be in format of:
 * URL         neighbor URL
 * URL         neighbor URL
 * URL         neighbor URL
 * ...
 * where URL and their neighbors are separated by space(s).
 *
 * This is an example implementation for learning how to use Spark. For more conventional use,
 * please refer to org.apache.spark.graphx.lib.PageRank
 * @class
 */


var conf = new SparkConf().setAppName("JavaScript Page Rank").setMaster("local[*]");
var sparkContext = new SparkContext(conf);

var filename=(arguments.length > 0) ?  arguments[1] : "examples/data/pagerank_data.txt";
var iters =  (arguments.length > 1) ? 0+arguments[1] : 10;


    // Loads in input file. It should be in format of:
    //     URL         neighbor URL
    //     URL         neighbor URL
    //     URL         neighbor URL
    //     ...
    var lines = sparkContext.textFile(filename, 1);

    // Loads all URLs from input file and initialize their neighbors.
    var links = lines.mapToPair(function(s) {
        var parts = s.split(/\s+/);
        return  [parts[0], parts[1]];
    }).distinct().groupByKey().cache();


    // Loads all URLs with other URL(s) link to from input file and initialize ranks of them to one.
    var ranks = links.mapValues(function() {
        return 1.0;
    });

    // Calculates and updates URL ranks continuously using PageRank algorithm.
    for (var current = 0; current < iters; current++) {
      // Calculates URL contributions to the rank of other URLs.
      var contribs = links.join(ranks).values()
        .flatMapToPair(function(tuple)  {
            var urlCount = tuple[0].length;
            var results = [];
            for (var n=0;n<urlCount;n++) {
              results.push([tuple[0][n], tuple[1] / urlCount]);
            }
            return results;
      });

      // Re-calculates URL ranks based on neighbor contributions.
      ranks = contribs.reduceByKey(function(a,b){return a+b;}).
        mapValues(function(sum) {
          return 0.15 + sum * 0.85;
      });
    }


    // Collects all URL ranks and dump them to console.
    var output = ranks.collect();
    for (var i=0;i<output.length;i++) {
        print(output[i][0] + " has rank: " + output[i][1] + ".");
    }

sparkContext.stop();


<file_sep>/*
 * Copyright 2015 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

var conf = new SparkConf().setAppName("JavaScript streaming").setMaster("local[*]"); 
var sparkContext = new SparkContext(conf);
var streamingContext = new StreamingContext(sparkContext, new Duration(2000));
//var dstream = streamingContext.socketTextStream("localhost", 9999);
print("KafkaUtils");
print(KafkaUtils);
print(KafkaUtils.createStream);
var dstream = KafkaUtils
    .createStream(streamingContext,
                  "192.168.127.12:2181",
                  "floyd",
                  "airline")
    //.window(new Duration(1000 * 60 * 15))
    .flatMap(function(chunk) {
       return chunk[1].split('\n');
    })
    .map(function(line) {
        var lineArr = line.split(",");
        var str = JSON.stringify({
          "origin": lineArr[16],
          "carrier": lineArr[8],
          "flight_num": lineArr[9],
          "destination": lineArr[17],
          "take_off_delay_mins": parseInt(lineArr[15])
        })

        return str;
    })


var sqlContext = new org.apache.spark.sql.hive.HiveContext(sparkContext.getJavaObject().sc());
sqlContext.sql("CREATE TABLE IF NOT EXISTS airline (origin STRING, carrier STRING, flight_num STRING, destination STRING, take_off_delay_mins INT)")
dstream.foreachRDD(function(rdd) {
    var df = sqlContext.read().json(rdd)
    var writer = df.write().mode(org.apache.spark.sql.SaveMode.Append)
    //df.show()
    writer.saveAsTable("airline");
    //print(rdd.collect());
});

streamingContext.start();
streamingContext.awaitTermination();
<file_sep><project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <groupId>com.ibm</groupId>
    <artifactId>eclairjs-nashorn</artifactId>
    <version>0.1</version>

    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <executions>
                    <execution>
                        <phase>compile</phase>
                        <goals>
                            <goal>compile</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.3</version>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-shade-plugin</artifactId>
                <version>2.4.1</version>
                <executions>
                    <execution>
                        <phase>package</phase>
                        <goals>
                            <goal>shade</goal>
                        </goals>
                        <configuration>
                            <filters>
                                <filter>
                                  <artifact>*:*</artifact>
                                  <excludes>
                                    <exclude>META-INF/*.SF</exclude>
                                    <exclude>META-INF/*.DSA</exclude>
                                    <exclude>META-INF/*.RSA</exclude>
                                  </excludes>
                                </filter>
                            </filters>
                            <artifactSet>
                                <excludes>
                                    <exclude>classworlds:classworlds</exclude>
                                    <exclude>junit:junit</exclude>
                                    <exclude>jmock:*</exclude>
                                    <exclude>*:xml-apis</exclude>
                                    <exclude>org.scala-lang:*</exclude>
                                    <exclude>org.apache.maven:lib:tests</exclude>
                                    <exclude>log4j:log4j:jar:</exclude>
                                    <exclude>org.eclipse.jetty:*</exclude>
                                </excludes>
                            </artifactSet>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
        </plugins>
        <pluginManagement>
        	<plugins>
        		<!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.-->
        		<plugin>
        			<groupId>org.eclipse.m2e</groupId>
        			<artifactId>lifecycle-mapping</artifactId>
        			<version>1.0.0</version>
        			<configuration>
        				<lifecycleMappingMetadata>
        					<pluginExecutions>
        						<pluginExecution>
        							<pluginExecutionFilter>
        								<groupId>
        									com.phasebash.jsdoc
        								</groupId>
        								<artifactId>
        									jsdoc3-maven-plugin
        								</artifactId>
        								<versionRange>
        									[1.1.0,)
        								</versionRange>
        								<goals>
        									<goal>jsdoc3</goal>
        								</goals>
        							</pluginExecutionFilter>
        							<action>
        								<ignore></ignore>
        							</action>
        						</pluginExecution>
        					</pluginExecutions>
        				</lifecycleMappingMetadata>
        			</configuration>
        		</plugin>
        	</plugins>
        </pluginManagement>
    </build>
     <properties>
        <spark.version>1.6.0</spark.version>
    </properties>
    <dependencies>

        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-library</artifactId>
            <version>2.10.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.10</artifactId>
            <version>${spark.version}</version>
        </dependency>
        <dependency>
		    <groupId>org.apache.spark</groupId>
		    <artifactId>spark-sql_2.10</artifactId>
		    <version>${spark.version}</version>
		</dependency>
		<dependency>
		    <groupId>org.apache.spark</groupId>
		    <artifactId>spark-catalyst_2.10</artifactId>
		    <version>${spark.version}</version>
		</dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-hive_2.10</artifactId>
            <version>${spark.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-streaming_2.10</artifactId>
            <version>${spark.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-mllib_2.10</artifactId>
            <version>${spark.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-streaming-kafka_2.10</artifactId>
            <version>${spark.version}</version>
        </dependency>
        <dependency>
            <groupId>com.googlecode.json-simple</groupId>
            <artifactId>json-simple</artifactId>
            <version>1.1</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>commons-io</groupId>
            <artifactId>commons-io</artifactId>
            <version>2.4</version>
            <scope>test</scope>
        </dependency>
    </dependencies>
    <profiles>
    <profile>
        <id>notebook</id>
        <dependencies>
        <dependency>
            <groupId>org.apache.toree</groupId>
            <artifactId>toree-kernel-api_2.10</artifactId>
            <version>0.0.0-dev-SNAPSHOT</version>
            <scope>provided</scope>
            <exclusions>
                <exclusion>
                    <groupId>javax.servlet</groupId>
                    <artifactId>servlet-api</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        </dependencies>
        <build>
        <plugins>
        <plugin>
            <groupId>net.alchim31.maven</groupId>
            <artifactId>scala-maven-plugin</artifactId>
            <executions>
                <execution>
                    <id>scala-compile-first</id>
                    <phase>process-resources</phase>
                    <goals>
                        <goal>add-source</goal>
                        <goal>compile</goal>
                    </goals>
                </execution>
                <execution>
                    <id>scala-test-compile</id>
                    <phase>process-test-resources</phase>
                    <goals>
                        <goal>testCompile</goal>
                    </goals>
                </execution>
            </executions>
        </plugin>
        </plugins>
        </build>
    </profile>
       <profile>
        <id>docs</id>
        <build>
        <plugins>
            <plugin>
                <groupId>com.phasebash.jsdoc</groupId>
                <artifactId>jsdoc3-maven-plugin</artifactId>
                <version>1.1.0</version>
                <executions>
                    <execution>
                        <phase>compile</phase>
                        <goals>
                            <goal>jsdoc3</goal>
                        </goals>
                    </execution>
                </executions>
                <configuration>
                    <recursive>true</recursive>
                    <directoryRoots>
                        <directoryRoot>src/main/resources</directoryRoot>
                    </directoryRoots>
                     <outputDirectory>docs/jsdoc</outputDirectory>
                </configuration>
            </plugin>

        </plugins>
        </build>
    </profile>
    </profiles>
</project>
<file_sep>/*
 * Copyright 2016 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


  function showWarning() {
    var warning = "WARN: This is a naive implementation of Logistic Regression " +
            "and is given as an example!\n" +
            "Please use either org.apache.spark.mllib.classification.LogisticRegressionWithSGD " +
            "or org.apache.spark.mllib.classification.LogisticRegressionWithLBFGS " +
            "for more conventional use.";
    print(warning);
  }

  function printWeights( a) {
    print(a);
  }



  function dot(  a,  b) {
    var x = 0;
    for (var i = 0; i < D; i++) {
      x += a[i] * b[i];
    }
    return x;
  }


showWarning();


var D = 10;   // Number of dimensions

var file=(arguments.length>0) ? arguments[0] : "./examples/data/lr_data.txt";
var ITERATIONS =  (arguments.length > 1) ? 0+arguments[1] : 10;

var conf = new SparkConf().setAppName("JavaScript Logistic Regression ").setMaster("local[*]");
var sc = new SparkContext(conf);

    var lines = sc.textFile(file);
    var points = lines.map(function(line){
        var tok = line.split(/\s+/);
        var y = tok[0];
        var x = [];
        for (var i = 0; i < D; i++) {
          x[i] = tok[i + 1];
        }
      return {x:x,y:y} ;

    }).cache();

    // Initialize w to a random value
    var weights= [];
    for (var i = 0; i < D; i++) {
      weights[i] = 2 * Math.random() - 1;
    }

    print("Initial w: ");
    printWeights(weights);

    for (var i = 1; i <= ITERATIONS; i++) {
      print("On iteration " + i);

      var gradient = points.map( function (datapoint){
        var gradient = [];
        for (var i = 0; i < D; i++) {
          var d  = dot(weights, datapoint.x);
          gradient[i] = (1 / (1 + Math.exp(-datapoint.y * d)) - 1) * datapoint.y * datapoint.x[i];
        }
        return gradient;
      }).reduce(function(a,b){
        var result = [];
        for (var j = 0; j < D; j++) {
          result[j] = a[j] + b[j];
        }
        return result;
      });

      for (var j = 0; j < D; j++) {
        weights[j] -= gradient[j];
      }

    }

    print("Final w: ");
    printWeights(weights);


sc.stop();


<file_sep>/*
 * Copyright 2016 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

var conf = new SparkConf().setAppName("JavaScript Spark Pi").setMaster("local[*]");
var sparkContext = new SparkContext(conf);


/**
 * Computes an approximation to pi
 */


var slices =  10;

var n = 100000 * slices;
var l = [];
for (var i = 0; i < n; i++) {
  l.push(i);
}


 var dataSet = sparkContext.parallelize(l, slices);



 var count = dataSet.map(function(i) {
        var x = Math.random() * 2 - 1;
        var y = Math.random() * 2 - 1;
        return (x * x +   y * y < 1) ? 1 : 0;
    }).reduce(function(int1,int2) {
        return int1 + int2;
    });


  print("Pi is roughly " + (4.0 * count / n));


sparkContext.stop();


<file_sep>/*
 * Copyright 2015 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

var JavaStreamingContext = 
    Java.type('org.apache.spark.streaming.api.java.JavaStreamingContext');
/**
 * @constructor
 * @classdesc Main entry point for Spark Streaming functionality. It provides methods used to create DStreams from various input sources.
 *  It can be either created by providing a Spark master URL and an appName, or from a org.apache.spark.SparkConf configuration 
 *  (see core Spark documentation), or from an existing org.apache.spark.SparkContext. The associated SparkContext can be accessed 
 *  using context.sparkContext. After creating and transforming DStreams, the streaming computation can be started and stopped using 
 *  context.start() and context.stop(), respectively. context.awaitTermination() allows the current thread to wait for the termination 
 *  of the context by stop() or by an exception.
 *  @param {SparkContex} sparkContext
 *  @param {Duration} duration
 */
var StreamingContext = function(sparkContext, duration) {
	var jvmObj = 
        new JavaStreamingContext(Utils.unwrapObject(sparkContext), 
    							 Utils.unwrapObject(duration)
    							 );
	this.logger = Logger.getLogger("streaming.Duration_js");
	JavaWrapper.call(this, jvmObj);
};

StreamingContext.prototype = Object.create(JavaWrapper.prototype); 

StreamingContext.prototype.constructor = StreamingContext;
/**
 * Wait for the execution to stop
 */
StreamingContext.prototype.awaitTermination = function() {
    this.getJavaObject().awaitTermination();
};
/**
 * Wait for the execution to stop, or timeout
 * @param {long} millis
 * @returns {boolean}
 */
StreamingContext.prototype.awaitTerminationOrTimeout = function(millis) {
    return this.getJavaObject().awaitTerminationOrTimeout(millis);
};
/**
 * Start the execution of the streams.
 */
StreamingContext.prototype.start = function() {
    this.getJavaObject().start();
};
/**
 * Stops the execution of the streams.
 */
StreamingContext.prototype.stop = function() {
    if(arguments.length == 1) {
        this.getJavaObject().stop(arguments[0]);
    } else {
        this.getJavaObject().stop();
    }
};

StreamingContext.prototype.close = function() {
    this.getJavaObject().close();
};

/**
 * Create a input stream from TCP source hostname:port.
 * @param {string} host
 * @param {string} port
 * @returns {DStream}
 */
StreamingContext.prototype.socketTextStream = function(host, port) {
    var jDStream = this.getJavaObject().socketTextStream(host, port);

    return new DStream(jDStream, this);
};




<file_sep>/*
 * Copyright 2015 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
var sparkContext = new SparkContext("local[*]", "accumulators test");
var accum = sparkContext.accumulator([0]);
var addInt = function(){
	
	print("accum " + accum);
	sparkContext.parallelize([1, 2, 3, 4]).foreach(function(x, accum) {
		accum.add(x);
	}, [accum]);
	print( accum.value());
}
	

addInt();

<file_sep>/*                                                                         
* Copyright 2016 IBM Corp.                                                 
*                                                                          
* Licensed under the Apache License, Version 2.0 (the "License");          
* you may not use this file except in compliance with the License.         
* You may obtain a copy of the License at                                  
*                                                                          
*      http://www.apache.org/licenses/LICENSE-2.0                          
*                                                                          
* Unless required by applicable law or agreed to in writing, software      
* distributed under the License is distributed on an "AS IS" BASIS,        
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
* See the License for the specific language governing permissions and      
* limitations under the License.                                           
*/ 



/**
 * Model trained by {@link FPGrowth}, which holds frequent itemsets.
 * @param freqItemsets frequent itemset, which is an RDD of {@link FreqItemset}
 * @classdesc
 */

/**
 * @param {RDD} freqItemsets
 * @returns {??} 
 *  @class
 */
var FPGrowthModel = function(freqItemsets) {
	 var jvmObject = new org.apache.spark.mllib.fpm.FPGrowthModel(freqItemsets);
	 this.logger = Logger.getLogger("FPGrowthModel_js");
	 JavaWrapper.call(this, jvmObject);

};

FPGrowthModel.prototype = Object.create(JavaWrapper.prototype);

FPGrowthModel.prototype.constructor = FPGrowthModel;



/**
 * Generates association rules for the [[Item]]s in {@link freqItemsets}.
 * @param {number} confidence  minimal confidence of the rules produced
 * @returns {RDD} 
 */
FPGrowthModel.prototype.generateAssociationRules = function(confidence) {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().generateAssociationRules(confidence);
//   return new RDD(javaObject);
};


/**
 * A parallel FP-growth algorithm to mine frequent itemsets. The algorithm is described in
 * [[http://dx.doi.org/10.1145/1454008.1454027 Li et al., PFP: Parallel FP-Growth for Query
 *  Recommendation]]. PFP distributes computation in such a way that each worker executes an
 * independent group of mining tasks. The FP-Growth algorithm is described in
 * [[http://dx.doi.org/10.1145/335191.335372 Han et al., Mining frequent patterns without candidate
 *  generation]].
 *
 * @param minSupport the minimal support level of the frequent pattern, any pattern appears
 *                   more than (minSupport * size-of-the-dataset) times will be output
 * @param numPartitions number of partitions used by parallel FP-growth
 *
 * @see [[http://en.wikipedia.org/wiki/Association_rule_learning Association rule learning
 *       (Wikipedia)]]
 *
 * @classdesc
 */

/**
 * Constructs a default instance with default parameters {minSupport: `0.3`, numPartitions: same
 * as the input data}.
 *
 * @returns {??} 
 *  @class
 */
var FPGrowth = function(jvmObject) {
	 
	 this.logger = Logger.getLogger("FPGrowth_js");
	 JavaWrapper.call(this, jvmObject);

};

FPGrowth.prototype = Object.create(JavaWrapper.prototype);

FPGrowth.prototype.constructor = FPGrowth;



/**
 * Sets the minimal support level (default: `0.3`).
 *
 * @param {number} minSupport
 * @returns {} 
 */
FPGrowth.prototype.setMinSupport = function(minSupport) {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().setMinSupport(minSupport);
//   return new (javaObject);
};


/**
 * Sets the number of partitions used by parallel FP-growth (default: same as input data).
 *
 * @param {number} numPartitions
 * @returns {} 
 */
FPGrowth.prototype.setNumPartitions = function(numPartitions) {
throw "not implemented by ElairJS";
//   var javaObject =  this.getJavaObject().setNumPartitions(numPartitions);
//   return new (javaObject);
};


/**
 * Computes an FP-Growth model that contains frequent itemsets.
 * @param {RDD} data  input data set, each element contains a transaction
 *
 * @returns {FPGrowthModel}  an [[FPGrowthModel]]
 */
FPGrowth.prototype.runwithRDD = function(data) {
throw "not implemented by ElairJS";
//   var data_uw = Utils.unwrapObject(data);
//   var javaObject =  this.getJavaObject().run(data_uw);
//   return new FPGrowthModel(javaObject);
};


/**
 * @param {JavaRDD} data
 * @returns {FPGrowthModel} 
 */
FPGrowth.prototype.runwithJavaRDD = function(data) {
throw "not implemented by ElairJS";
//   var data_uw = Utils.unwrapObject(data);
//   var javaObject =  this.getJavaObject().run(data_uw);
//   return new FPGrowthModel(javaObject);
};

/**
 * Frequent itemset. param: items items in this itemset. Java users should call javaItems() instead. param: freq frequency
 * @classdesc
 * @param {object} items
 * @param {integer} freq
 * @constructor
 */
var FreqItemset = function() {

	this.logger = Logger.getLogger("FreqItemset_js");
	var jvmObject = arguments[0];
	if (arguments.length > 1) {
        var items = Utils.unwrapObject(arguments[0]);
        if (Array.isArray(items)) {
            var list = new java.util.ArrayList();
            items.forEach(function(item){
                list.add(Utils.unwrapObject(item));
            });
            items = list.toArray();
        }
		jvmObject = new org.apache.spark.mllib.fpm.FPGrowth.FreqItemset(items, arguments[1]);
	}
	JavaWrapper.call(this, jvmObject);

};

FreqItemset.prototype = Object.create(JavaWrapper.prototype);

FreqItemset.prototype.constructor = FreqItemset;

FreqItemset.prototype.items = function() {

   var javaObject =  this.getJavaObject().javaItems();
   return javaObject;
};