text:
package com.amazonaws.services.sagemaker.model.transform;

import java.util.Map;
import java.util.List;
import javax.annotation.Generated;

import com.amazonaws.SdkClientException;
import com.amazonaws.services.sagemaker.model.*;

import com.amazonaws.protocol.*;
import com.amazonaws.annotation.SdkInternalApi;

/**
 * CreateTransformJobRequestMarshaller
 */
@Generated("com.amazonaws:aws-java-sdk-code-generator")
@SdkInternalApi
public class CreateTransformJobRequestMarshaller {

    private static final MarshallingInfo<String> TRANSFORMJOBNAME_BINDING = MarshallingInfo.builder(MarshallingType.STRING)
            .marshallLocation(MarshallLocation.PAYLOAD).marshallLocationName("TransformJobName").build();
    private static final MarshallingInfo<String> MODELNAME_BINDING = MarshallingInfo.builder(MarshallingType.STRING).marshallLocation(MarshallLocation.PAYLOAD)
            .marshallLocationName("ModelName").build();
    private static final MarshallingInfo<Integer> MAXCONCURRENTTRANSFORMS_BINDING = MarshallingInfo.builder(MarshallingType.INTEGER)
            .marshallLocation(MarshallLocation.PAYLOAD).marshallLocationName("MaxConcurrentTransforms").build();
    private static final MarshallingInfo<Integer> MAXPAYLOADINMB_BINDING = MarshallingInfo.builder(MarshallingType.INTEGER)
            .marshallLocation(MarshallLocation.PAYLOAD).marshallLocationName("MaxPayloadInMB").build();
    private static final MarshallingInfo<String> BATCHSTRATEGY_BINDING = MarshallingInfo.builder(MarshallingType.STRING)
            .marshallLocation(MarshallLocation.PAYLOAD).marshallLocationName("BatchStrategy").build();
    private static final MarshallingInfo<Map> ENVIRONMENT_BINDING = MarshallingInfo.builder(MarshallingType.MAP).marshallLocation(MarshallLocation.PAYLOAD)
            .marshallLocationName("Environment").build();
    private static final MarshallingInfo<StructuredPojo> TRANSFORMINPUT_BINDING = MarshallingInfo.builder(MarshallingType.STRUCTURED)
            .marshallLocation(MarshallLocation.PAYLOAD).marshallLocationName("TransformInput").build();
    private static final MarshallingInfo<StructuredPojo> TRANSFORMOUTPUT_BINDING = MarshallingInfo.builder(MarshallingType.STRUCTURED)
            .marshallLocation(MarshallLocation.PAYLOAD).marshallLocationName("TransformOutput").build();
    private static final MarshallingInfo<StructuredPojo> TRANSFORMRESOURCES_BINDING = MarshallingInfo.builder(MarshallingType.STRUCTURED)
            .marshallLocation(MarshallLocation.PAYLOAD).marshallLocationName("TransformResources").build();
    private static final MarshallingInfo<StructuredPojo> DATAPROCESSING_BINDING = MarshallingInfo.builder(MarshallingType.STRUCTURED)
            .marshallLocation(MarshallLocation.PAYLOAD).marshallLocationName("DataProcessing").build();
    private static final MarshallingInfo<List> TAGS_BINDING = MarshallingInfo.builder(MarshallingType.LIST).marshallLocation(MarshallLocation.PAYLOAD)
            .marshallLocationName("Tags").build();

    private static final CreateTransformJobRequestMarshaller instance = new CreateTransformJobRequestMarshaller();

    public static CreateTransformJobRequestMarshaller getInstance() {
        return instance;
    }

    /**
     * Marshall the given parameter object.
     */
    public void marshall(CreateTransformJobRequest createTransformJobRequest, ProtocolMarshaller protocolMarshaller) {

        if (createTransformJobRequest == null) {
            throw new SdkClientException("Invalid argument passed to marshall(...)");
        }

        try {
            protocolMarshaller.marshall(createTransformJobRequest.getTransformJobName(), TRANSFORMJOBNAME_BINDING);
            protocolMarshaller.marshall(createTransformJobRequest.getModelName(), MODELNAME_BINDING);
            protocolMarshaller.marshall(createTransformJobRequest.getMaxConcurrentTransforms(), MAXCONCURRENTTRANSFORMS_BINDING);
            protocolMarshaller.marshall(createTransformJobRequest.getMaxPayloadInMB(), MAXPAYLOADINMB_BINDING);
            protocolMarshaller.marshall(createTransformJobRequest.getBatchStrategy(), BATCHSTRATEGY_BINDING);
            protocolMarshaller.marshall(createTransformJobRequest.getEnvironment(), ENVIRONMENT_BINDING);
            protocolMarshaller.marshall(createTransformJobRequest.getTransformInput(), TRANSFORMINPUT_BINDING);
            protocolMarshaller.marshall(createTransformJobRequest.getTransformOutput(), TRANSFORMOUTPUT_BINDING);
            protocolMarshaller.marshall(createTransformJobRequest.getTransformResources(), TRANSFORMRESOURCES_BINDING);
            protocolMarshaller.marshall(createTransformJobRequest.getDataProcessing(), DATAPROCESSING_BINDING);
            protocolMarshaller.marshall(createTransformJobRequest.getTags(), TAGS_BINDING);
        } catch (Exception e) {
            throw new SdkClientException("Unable to marshall request to JSON: " + e.getMessage(), e);
        }
    }

}

redpajama_set_name: RedPajamaGithub


text:
package org.apache.spark.examples.sql;

// $example on:schema_merging$
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
// $example off:schema_merging$
import java.util.Properties;

// $example on:basic_parquet_example$
import org.apache.spark.api.java.function.MapFunction;
import org.apache.spark.sql.Encoders;
// $example on:schema_merging$
// $example on:json_dataset$
// $example on:csv_dataset$
// $example on:text_dataset$
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
// $example off:text_dataset$
// $example off:csv_dataset$
// $example off:json_dataset$
// $example off:schema_merging$
// $example off:basic_parquet_example$
import org.apache.spark.sql.SparkSession;

public class JavaSQLDataSourceExample {

  // $example on:schema_merging$
  public static class Square implements Serializable {
    private int value;
    private int square;

    // Getters and setters...
    // $example off:schema_merging$
    public int getValue() {
      return value;
    }

    public void setValue(int value) {
      this.value = value;
    }

    public int getSquare() {
      return square;
    }

    public void setSquare(int square) {
      this.square = square;
    }
    // $example on:schema_merging$
  }
  // $example off:schema_merging$

  // $example on:schema_merging$
  public static class Cube implements Serializable {
    private int value;
    private int cube;

    // Getters and setters...
    // $example off:schema_merging$
    public int getValue() {
      return value;
    }

    public void setValue(int value) {
      this.value = value;
    }

    public int getCube() {
      return cube;
    }

    public void setCube(int cube) {
      this.cube = cube;
    }
    // $example on:schema_merging$
  }
  // $example off:schema_merging$

  public static void main(String[] args) {
    SparkSession spark = SparkSession
      .builder()
      .appName("Java Spark SQL data sources example")
      .config("spark.some.config.option", "some-value")
      .getOrCreate();

    runBasicDataSourceExample(spark);
    runGenericFileSourceOptionsExample(spark);
    runBasicParquetExample(spark);
    runParquetSchemaMergingExample(spark);
    runJsonDatasetExample(spark);
    runCsvDatasetExample(spark);
    runTextDatasetExample(spark);
    runJdbcDatasetExample(spark);

    spark.stop();
  }

  private static void runGenericFileSourceOptionsExample(SparkSession spark) {
    // $example on:ignore_corrupt_files$
    // enable ignore corrupt files
    spark.sql("set spark.sql.files.ignoreCorruptFiles=true");
    // dir1/file3.json is corrupt from parquet's view
    Dataset<Row> testCorruptDF = spark.read().parquet(
            "examples/src/main/resources/dir1/",
            "examples/src/main/resources/dir1/dir2/");
    testCorruptDF.show();
    // +-------------+
    // |         file|
    // +-------------+
    // |file1.parquet|
    // |file2.parquet|
    // +-------------+
    // $example off:ignore_corrupt_files$
    // $example on:recursive_file_lookup$
    Dataset<Row> recursiveLoadedDF = spark.read().format("parquet")
            .option("recursiveFileLookup", "true")
            .load("examples/src/main/resources/dir1");
    recursiveLoadedDF.show();
    // +-------------+
    // |         file|
    // +-------------+
    // |file1.parquet|
    // |file2.parquet|
    // +-------------+
    // $example off:recursive_file_lookup$
    spark.sql("set spark.sql.files.ignoreCorruptFiles=false");
    // $example on:load_with_path_glob_filter$
    Dataset<Row> testGlobFilterDF = spark.read().format("parquet")
            .option("pathGlobFilter", "*.parquet") // json file should be filtered out
            .load("examples/src/main/resources/dir1");
    testGlobFilterDF.show();
    // +-------------+
    // |         file|
    // +-------------+
    // |file1.parquet|
    // +-------------+
    // $example off:load_with_path_glob_filter$
    // $example on:load_with_modified_time_filter$
    Dataset<Row> beforeFilterDF = spark.read().format("parquet")
            // Only load files modified before 7/1/2020 at 05:30
            .option("modifiedBefore", "2020-07-01T05:30:00")
            // Only load files modified after 6/1/2020 at 05:30
            .option("modifiedAfter", "2020-06-01T05:30:00")
            // Interpret both times above relative to CST timezone
            .option("timeZone", "CST")
            .load("examples/src/main/resources/dir1");
    beforeFilterDF.show();
    // +-------------+
    // |         file|
    // +-------------+
    // |file1.parquet|
    // +-------------+
    // $example off:load_with_modified_time_filter$
  }

  private static void runBasicDataSourceExample(SparkSession spark) {
    // $example on:generic_load_save_functions$
    Dataset<Row> usersDF = spark.read().load("examples/src/main/resources/users.parquet");
    usersDF.select("name", "favorite_color").write().save("namesAndFavColors.parquet");
    // $example off:generic_load_save_functions$
    // $example on:manual_load_options$
    Dataset<Row> peopleDF =
      spark.read().format("json").load("examples/src/main/resources/people.json");
    peopleDF.select("name", "age").write().format("parquet").save("namesAndAges.parquet");
    // $example off:manual_load_options$
    // $example on:manual_load_options_csv$
    Dataset<Row> peopleDFCsv = spark.read().format("csv")
      .option("sep", ";")
      .option("inferSchema", "true")
      .option("header", "true")
      .load("examples/src/main/resources/people.csv");
    // $example off:manual_load_options_csv$
    // $example on:manual_save_options_orc$
    usersDF.write().format("orc")
      .option("orc.bloom.filter.columns", "favorite_color")
      .option("orc.dictionary.key.threshold", "1.0")
      .option("orc.column.encoding.direct", "name")
      .save("users_with_options.orc");
    // $example off:manual_save_options_orc$
    // $example on:manual_save_options_parquet$
    usersDF.write().format("parquet")
        .option("parquet.bloom.filter.enabled#favorite_color", "true")
        .option("parquet.bloom.filter.expected.ndv#favorite_color", "1000000")
        .option("parquet.enable.dictionary", "true")
        .option("parquet.page.write-checksum.enabled", "false")
        .save("users_with_options.parquet");
    // $example off:manual_save_options_parquet$
    // $example on:direct_sql$
    Dataset<Row> sqlDF =
      spark.sql("SELECT * FROM parquet.`examples/src/main/resources/users.parquet`");
    // $example off:direct_sql$
    // $example on:write_sorting_and_bucketing$
    peopleDF.write().bucketBy(42, "name").sortBy("age").saveAsTable("people_bucketed");
    // $example off:write_sorting_and_bucketing$
    // $example on:write_partitioning$
    usersDF
      .write()
      .partitionBy("favorite_color")
      .format("parquet")
      .save("namesPartByColor.parquet");
    // $example off:write_partitioning$
    // $example on:write_partition_and_bucket$
    usersDF
      .write()
      .partitionBy("favorite_color")
      .bucketBy(42, "name")
      .saveAsTable("users_partitioned_bucketed");
    // $example off:write_partition_and_bucket$

    spark.sql("DROP TABLE IF EXISTS people_bucketed");
    spark.sql("DROP TABLE IF EXISTS users_partitioned_bucketed");
  }

  private static void runBasicParquetExample(SparkSession spark) {
    // $example on:basic_parquet_example$
    Dataset<Row> peopleDF = spark.read().json("examples/src/main/resources/people.json");

    // DataFrames can be saved as Parquet files, maintaining the schema information
    peopleDF.write().parquet("people.parquet");

    // Read in the Parquet file created above.
    // Parquet files are self-describing so the schema is preserved
    // The result of loading a parquet file is also a DataFrame
    Dataset<Row> parquetFileDF = spark.read().parquet("people.parquet");

    // Parquet files can also be used to create a temporary view and then used in SQL statements
    parquetFileDF.createOrReplaceTempView("parquetFile");
    Dataset<Row> namesDF = spark.sql("SELECT name FROM parquetFile WHERE age BETWEEN 13 AND 19");
    Dataset<String> namesDS = namesDF.map(
        (MapFunction<Row, String>) row -> "Name: " + row.getString(0),
        Encoders.STRING());
    namesDS.show();
    // +------------+
    // |       value|
    // +------------+
    // |Name: Justin|
    // +------------+
    // $example off:basic_parquet_example$
  }

  private static void runParquetSchemaMergingExample(SparkSession spark) {
    // $example on:schema_merging$
    List<Square> squares = new ArrayList<>();
    for (int value = 1; value <= 5; value++) {
      Square square = new Square();
      square.setValue(value);
      square.setSquare(value * value);
      squares.add(square);
    }

    // Create a simple DataFrame, store into a partition directory
    Dataset<Row> squaresDF = spark.createDataFrame(squares, Square.class);
    squaresDF.write().parquet("data/test_table/key=1");

    List<Cube> cubes = new ArrayList<>();
    for (int value = 6; value <= 10; value++) {
      Cube cube = new Cube();
      cube.setValue(value);
      cube.setCube(value * value * value);
      cubes.add(cube);
    }

    // Create another DataFrame in a new partition directory,
    // adding a new column and dropping an existing column
    Dataset<Row> cubesDF = spark.createDataFrame(cubes, Cube.class);
    cubesDF.write().parquet("data/test_table/key=2");

    // Read the partitioned table
    Dataset<Row> mergedDF = spark.read().option("mergeSchema", true).parquet("data/test_table");
    mergedDF.printSchema();

    // The final schema consists of all 3 columns in the Parquet files together
    // with the partitioning column appeared in the partition directory paths
    // root
    //  |-- value: int (nullable = true)
    //  |-- square: int (nullable = true)
    //  |-- cube: int (nullable = true)
    //  |-- key: int (nullable = true)
    // $example off:schema_merging$
  }

  private static void runJsonDatasetExample(SparkSession spark) {
    // $example on:json_dataset$
    // A JSON dataset is pointed to by path.
    // The path can be either a single text file or a directory storing text files
    Dataset<Row> people = spark.read().json("examples/src/main/resources/people.json");

    // The inferred schema can be visualized using the printSchema() method
    people.printSchema();
    // root
    //  |-- age: long (nullable = true)
    //  |-- name: string (nullable = true)

    // Creates a temporary view using the DataFrame
    people.createOrReplaceTempView("people");

    // SQL statements can be run by using the sql methods provided by spark
    Dataset<Row> namesDF = spark.sql("SELECT name FROM people WHERE age BETWEEN 13 AND 19");
    namesDF.show();
    // +------+
    // |  name|
    // +------+
    // |Justin|
    // +------+

    // Alternatively, a DataFrame can be created for a JSON dataset represented by
    // a Dataset<String> storing one JSON object per string.
    List<String> jsonData = Arrays.asList(
            "{\"name\":\"Yin\",\"address\":{\"city\":\"Columbus\",\"state\":\"Ohio\"}}");
    Dataset<String> anotherPeopleDataset = spark.createDataset(jsonData, Encoders.STRING());
    Dataset<Row> anotherPeople = spark.read().json(anotherPeopleDataset);
    anotherPeople.show();
    // +---------------+----+
    // |        address|name|
    // +---------------+----+
    // |[Columbus,Ohio]| Yin|
    // +---------------+----+
    // $example off:json_dataset$
  }

  private static void runCsvDatasetExample(SparkSession spark) {
    // $example on:csv_dataset$
    // A CSV dataset is pointed to by path.
    // The path can be either a single CSV file or a directory of CSV files
    String path = "examples/src/main/resources/people.csv";

    Dataset<Row> df = spark.read().csv(path);
    df.show();
    // +------------------+
    // |               _c0|
    // +------------------+
    // |      name;age;job|
    // |Jorge;30;Developer|
    // |  Bob;32;Developer|
    // +------------------+

    // Read a csv with delimiter, the default delimiter is ","
    Dataset<Row> df2 = spark.read().option("delimiter", ";").csv(path);
    df2.show();
    // +-----+---+---------+
    // |  _c0|_c1|      _c2|
    // +-----+---+---------+
    // | name|age|      job|
    // |Jorge| 30|Developer|
    // |  Bob| 32|Developer|
    // +-----+---+---------+

    // Read a csv with delimiter and a header
    Dataset<Row> df3 = spark.read().option("delimiter", ";").option("header", "true").csv(path);
    df3.show();
    // +-----+---+---------+
    // | name|age|      job|
    // +-----+---+---------+
    // |Jorge| 30|Developer|
    // |  Bob| 32|Developer|
    // +-----+---+---------+

    // You can also use options() to use multiple options
    java.util.Map<String, String> optionsMap = new java.util.HashMap<String, String>();
    optionsMap.put("delimiter",";");
    optionsMap.put("header","true");
    Dataset<Row> df4 = spark.read().options(optionsMap).csv(path);

    // "output" is a folder which contains multiple csv files and a _SUCCESS file.
    df3.write().csv("output");

    // Read all files in a folder, please make sure only CSV files should present in the folder.
    String folderPath = "examples/src/main/resources";
    Dataset<Row> df5 = spark.read().csv(folderPath);
    df5.show();
    // Wrong schema because non-CSV files are read
    // +-----------+
    // |        _c0|
    // +-----------+
    // |238val_238|
    // |  86val_86|
    // |311val_311|
    // |  27val_27|
    // |165val_165|
    // +-----------+

    // $example off:csv_dataset$
  }

  private static void runTextDatasetExample(SparkSession spark) {
    // $example on:text_dataset$
    // A text dataset is pointed to by path.
    // The path can be either a single text file or a directory of text files
    String path = "examples/src/main/resources/people.txt";

    Dataset<Row> df1 = spark.read().text(path);
    df1.show();
    // +-----------+
    // |      value|
    // +-----------+
    // |Michael, 29|
    // |   Andy, 30|
    // | Justin, 19|
    // +-----------+

    // You can use 'lineSep' option to define the line separator.
    // The line separator handles all `\r`, `\r\n` and `\n` by default.
    Dataset<Row> df2 = spark.read().option("lineSep", ",").text(path);
    df2.show();
    // +-----------+
    // |      value|
    // +-----------+
    // |    Michael|
    // |   29\nAndy|
    // | 30\nJustin|
    // |       19\n|
    // +-----------+

    // You can also use 'wholetext' option to read each input file as a single row.
    Dataset<Row> df3 = spark.read().option("wholetext", "true").text(path);
    df3.show();
    //  +--------------------+
    //  |               value|
    //  +--------------------+
    //  |Michael, 29\nAndy...|
    //  +--------------------+

    // "output" is a folder which contains multiple text files and a _SUCCESS file.
    df1.write().text("output");

    // You can specify the compression format using the 'compression' option.
    df1.write().option("compression", "gzip").text("output_compressed");

    // $example off:text_dataset$
  }

  private static void runJdbcDatasetExample(SparkSession spark) {
    // $example on:jdbc_dataset$
    // Note: JDBC loading and saving can be achieved via either the load/save or jdbc methods
    // Loading data from a JDBC source
    Dataset<Row> jdbcDF = spark.read()
      .format("jdbc")
      .option("url", "jdbc:postgresql:dbserver")
      .option("dbtable", "schema.tablename")
      .option("user", "username")
      .option("password", "password")
      .load();

    Properties connectionProperties = new Properties();
    connectionProperties.put("user", "username");
    connectionProperties.put("password", "password");
    Dataset<Row> jdbcDF2 = spark.read()
      .jdbc("jdbc:postgresql:dbserver", "schema.tablename", connectionProperties);

    // Saving data to a JDBC source
    jdbcDF.write()
      .format("jdbc")
      .option("url", "jdbc:postgresql:dbserver")
      .option("dbtable", "schema.tablename")
      .option("user", "username")
      .option("password", "password")
      .save();

    jdbcDF2.write()
      .jdbc("jdbc:postgresql:dbserver", "schema.tablename", connectionProperties);

    // Specifying create table column data types on write
    jdbcDF.write()
      .option("createTableColumnTypes", "name CHAR(64), comments VARCHAR(1024)")
      .jdbc("jdbc:postgresql:dbserver", "schema.tablename", connectionProperties);
    // $example off:jdbc_dataset$
  }
}


text:
__author__ = 'NovikovII'
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from selenium.webdriver.support.wait import WebDriverWait
from python_example.fixture.fixture import driver

def test_l08(driver):
    driver.get("http://localhost/litecart/")
    driver.implicitly_wait(10)
    driver.find_element_by_css_selector("li.product")

    popular_products = driver.find_elements_by_css_selector("li.product")

    for popular_product in popular_products:
        stickers = popular_product.find_elements_by_css_selector(".sticker")
        assert (len(stickers) == 1)


text:
"use strict";Object.defineProperty(exports, "__esModule", { value: true });exports.default = void 0;

var _graphql = require("graphql");
var _graphqlRelay = require("graphql-relay");

var _ToDosConnection = _interopRequireDefault(require("./ToDosConnection"));function _interopRequireDefault(obj) {return obj && obj.__esModule ? obj : { default: obj };} //  weak
var _default =
{
  ToDos: {
    type: _ToDosConnection.default.connectionType,

    args: {
      status: {
        type: _graphql.GraphQLString,
        defaultValue: 'any' },

      ..._graphqlRelay.connectionArgs },

    resolve: async (obj, { status, ...args }, context, { rootValue: objectManager }) => {
      const arr = await objectManager.getObjectList_async('ToDo', {});

      return (0, _graphqlRelay.connectionFromArray)(
      arr.filter(a_ToDo => status === 'any' || a_ToDo.ToDo_Complete === (status === 'completed')),
      args);

    } },

  ToDo_TotalCount: {
    type: _graphql.GraphQLInt,

    resolve: async (obj, { ...args }, context, { rootValue: objectManager }) => {
      const arr = await objectManager.getObjectList_async('ToDo', {});

      return arr.length;
    } },

  ToDo_CompletedCount: {
    type: _graphql.GraphQLInt,

    resolve: async (obj, { ...args }, context, { rootValue: objectManager }) => {
      const arr = await objectManager.getObjectList_async('ToDo', {});

      return arr.filter(a_ToDo => a_ToDo.ToDo_Complete).length;
    } } };exports.default = _default;
//# sourceMappingURL=_ViewerFields.js.map


text:
Welcome to your practical assessment for Module 3 of MIT Big Data and Social Analytics.

## Objective:
When performing an analysis, you need to validate all of your assumptions, and be able to logically describe what you want to do, before selecting a method of execution. Using the WiFi access point information contained in the StudentLife data set from Dartmouth College you will:
- Explore the fundamental ideas of first-order analysis and data exploration;
- Explore the differences between noise and bias, and how you can reduce their impact in a data analysis project; and
- Apply basic statistical and visual data interrogation methods.

Engage with the mentor team and fellow students in the forums to get help, and share your insights and learning experiences.

We hope that you enjoy this course, and that you will acquire and refine skills that are of use to you in your personal journey.

## Notebook 1: Data exploration: Test your intuition about BSSIDs
> **Notebook goal**: Justify your assumptions about the data and hypotheses using your own behaviors and patterns.

The easiest way to validate your assumptions about the data generated is by testing data using your own behaviors and patterns. Once you have developed a working hypothesis, you need to consider cases where the behaviors of others do not necessarily align with your own. This notebook introduces BSSIDs in more detail, and revisits some of the concepts around testing on yourself. It also repeats some of the data exploration steps for a single user, and offers you the chance to think about what you would expect to see in the data.

####  List of exercises:
>   - **Exercise 1:** Using Pandas's ``describe()`` and ``info()`` methods to review data.
  -  **Exercise 2:** Access point identification.
  - **Exercise 3 [Advanced]:** Trending access point information.

## Notebook 2: Exploring noise vs. bias
> **Notebook goal**: Show the difference between bias and noise.

Bias and noise (variance) are the two sources of error that are invariably found in data. Understanding the differences between these types of errors, and how you can identify and mitigate these, is an important task prior to conducting analysis. This notebook will utilize a generated data set to demonstrate the difference between noise and bias, and introduce the concept of working with image files as input to your analysis. While we do not recommend that you attempt any image processing on the supplied environment, it is important to understand that you are not just restricted to text files or databases as input to your analyses.

####  List of exercises
>   - **Exercise 1:** Reducing the effect of bias in data.
  - **Exercise 2:** Understanding the effect of increasing sample size on the shape of noisy and biased data.

## Notebook 3: Geotagging WiFi access points
> **Notebook goal**: Apply basic statistical and visual data interrogation to data.
This notebook starts by geotagging access points for a single user (in the first section) and then moves on to repeat the exercise for a large data set (in the second section). The goal is to identify stationary and non-stationary WiFi routers in the StudentLife data set. This is the first time that you will start to work with the full record set, and you will need to make sure that you shut down any other notebooks that are not in use, to ensure that you have the maximum amount of resources available to complete the exercise. You will iterate through a directory, load the various files, identify just under 700 000 geotags, and perform a number of transforms on the data set.

####  List of exercises:
>   - **Exercise 1:** Identification of stationary WiFi routers.
  - **Exercise 2 [Advanced]:** Identification of non-stationary WiFi routers.

**Notebook Contributors**:
**Andre Voges**, **Mieszko Manijak**, **Gorden Jemwa**, **Arek Stopczynski**, Xiaowen Dong, and Yves-Alexandre de Montjoye.


text:
"""
tests.test_component_history
~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Tests the history component.
"""
# pylint: disable=protected-access,too-many-public-methods
import time
import os
import unittest
from unittest.mock import patch
from datetime import timedelta

import homeassistant.core as ha
import homeassistant.util.dt as dt_util
from homeassistant.components import history, recorder

from tests.common import (
    mock_http_component, mock_state_change_event, get_test_home_assistant)

class TestComponentHistory(unittest.TestCase):
    """ Tests homeassistant.components.history module. """

    def setUp(self):  # pylint: disable=invalid-name
        """ Init needed objects. """
        self.hass = get_test_home_assistant(1)
        self.init_rec = False

    def tearDown(self):  # pylint: disable=invalid-name
        """ Stop down stuff we started. """
        self.hass.stop()

        if self.init_rec:
            recorder._INSTANCE.block_till_done()
            os.remove(self.hass.config.path(recorder.DB_FILE))

    def init_recorder(self):
        recorder.setup(self.hass, {})
        self.hass.start()
        recorder._INSTANCE.block_till_done()
        self.init_rec = True

    def test_setup(self):
        """ Test setup method of history. """
        mock_http_component(self.hass)
        self.assertTrue(history.setup(self.hass, {}))

    def test_last_5_states(self):
        """ Test retrieving the last 5 states. """
        self.init_recorder()
        states = []

        entity_id = 'test.last_5_states'

        for i in range(7):
            self.hass.states.set(entity_id, "State {}".format(i))

            if i > 1:
                states.append(self.hass.states.get(entity_id))

            self.hass.pool.block_till_done()
            recorder._INSTANCE.block_till_done()

        self.assertEqual(
            list(reversed(states)), history.last_5_states(entity_id))

    def test_get_states(self):
        """ Test getting states at a specific point in time. """
        self.init_recorder()
        states = []

        for i in range(5):
            state = ha.State(
                'test.point_in_time_{}'.format(i % 5),
                "State {}".format(i),
                {'attribute_test': i})

            mock_state_change_event(self.hass, state)
            self.hass.pool.block_till_done()

            states.append(state)

        recorder._INSTANCE.block_till_done()

        point = dt_util.utcnow() + timedelta(seconds=1)

        with patch('homeassistant.util.dt.utcnow', return_value=point):
            for i in range(5):
                state = ha.State(
                    'test.point_in_time_{}'.format(i % 5),
                    "State {}".format(i),
                    {'attribute_test': i})

                mock_state_change_event(self.hass, state)
                self.hass.pool.block_till_done()

        # Get states returns everything before POINT
        self.assertEqual(states,
                         sorted(history.get_states(point),
                                key=lambda state: state.entity_id))

        # Test get_state here because we have a DB setup
        self.assertEqual(
            states[0], history.get_state(point, states[0].entity_id))

    def test_state_changes_during_period(self):
        self.init_recorder()
        entity_id = 'media_player.test'

        def set_state(state):
            self.hass.states.set(entity_id, state)
            self.hass.pool.block_till_done()
            recorder._INSTANCE.block_till_done()

            return self.hass.states.get(entity_id)

        set_state('idle')
        set_state('YouTube')

        start = dt_util.utcnow()
        point = start + timedelta(seconds=1)
        end = point + timedelta(seconds=1)

        with patch('homeassistant.util.dt.utcnow', return_value=point):
            states = [
                set_state('idle'),
                set_state('Netflix'),
                set_state('Plex'),
                set_state('YouTube'),
            ]

        with patch('homeassistant.util.dt.utcnow', return_value=end):
            set_state('Netflix')
            set_state('Plex')

        self.assertEqual(
            {entity_id: states},
            history.state_changes_during_period(start, end, entity_id))


text:
using System;
using System.Collections.Generic;
using System.Reflection;
using System.Text;
using Castle.Windsor;
using Rhino.Commons;

namespace HierarchicalContainers
{
    class Program
    {
        static void Main(string[] args)
        {
            try
            {
                RhinoContainer container = new RhinoContainer("Windsor.boo");
                IoC.Initialize(container);
                ContainerSelector containerSelector = IoC.Resolve<ContainerSelector>();
                containerSelector.PrintChildContainers();
                using(UnitOfWork.Start())
                {
                    Console.WriteLine(
                        NHibernateUnitOfWorkFactory.CurrentNHibernateSession
                            .Connection.ConnectionString
                        );
                }
                using(containerSelector.Enter("Northwind"))
                {
                    using (UnitOfWork.Start())
                    {
                        Console.WriteLine(
                            NHibernateUnitOfWorkFactory.CurrentNHibernateSession
                                .Connection.ConnectionString
                            );
                    }
                }
                using (containerSelector.Enter("Southsand"))
                {
                    using (UnitOfWork.Start())
                    {
                        Console.WriteLine(
                            NHibernateUnitOfWorkFactory.CurrentNHibernateSession
                                .Connection.ConnectionString
                            );
                    }
                }
            }
            catch (Exception e)
            {
                System.Console.WriteLine(e);
            }
        }

    }

    public class ContainerSelector
    {
        private readonly IDictionary<string, IWindsorContainer> containers = new Dictionary<string, IWindsorContainer>();

        public void Register(string name, IWindsorContainer container)
        {
            containers.Add(name, container);
        }

        public void PrintChildContainers()
        {
            foreach (KeyValuePair<string, IWindsorContainer> pair in containers)
            {
                Console.WriteLine("Container {0}", pair.Key);
            }
        }

        public IDisposable Enter(string name)
        {
            if(containers.ContainsKey(name)==false)
            {
                throw new InvalidOperationException("Container " + name + " was not registered");
            }
            IWindsorContainer container = containers[name];
            return IoC.UseLocalContainer(container);
        }
    }
}


text:
var getUnusedPort = require('../helper/port-helper');
var port = getUnusedPort();

require('./../helper/collection')('recent_messages', port);
var assert = require('assert');
var MongoClient = require('mongodb').MongoClient

var db;

describe('test tracker indexes', function () {

    before(function (done) {
        MongoClient.connect('mongodb://localhost/testCollectionrecent_messages', function(err, adb) {
            db = adb;
            done(err);
        });
    });

    after(function (done) {
        if (db) {
            db.close();
        }
        done();
    });

    it('should have user index on messages collection', function (done) {
        db.collection('messages').indexes(function(err, indexes) {
            assert.equal(indexes[1].key['user'], 1);
            done();
        });
    });

    it('should have trackerId index on trackers collection', function (done) {
        db.collection('messages').indexes(function(err, indexes) {
            assert.equal(indexes[2].key['trackerId'], 1);
            done();
        });
    });
});


text:
package com.cdancy.etcd.rest.domain.auth;

import org.jclouds.javax.annotation.Nullable;
import org.jclouds.json.SerializedNames;

import com.cdancy.etcd.rest.error.ErrorMessage;
import com.google.auto.value.AutoValue;

@AutoValue
public abstract class Role {

    public abstract String role();

    @Nullable
    public abstract Permission permissions();

    @Nullable
    public abstract Grant grant();

    @Nullable
    public abstract Revoke revoke();

    @Nullable
    public abstract ErrorMessage errorMessage();

    Role() {
    }

    @SerializedNames({ "role", "permissions", "grant", "revoke", "errorMessage" })
    public static Role create(String role, Permission permissions, Grant grant, Revoke revoke,
            ErrorMessage errorMessage) {
        return new AutoValue_Role(role, permissions, grant, revoke, errorMessage);
    }
}


text:
"""Ring buffer for automatic storage of images"""

import os.path
import logging
from datetime import datetime
import numpy as np
from scipy.misc import imsave
import tables

class RingBuffer(object):
    """Ring buffer class.

    Attributes
    ----------
    directory : str
        Location to store the ring buffer file.
    recording : bool
        True when data is being saved to the ring buffer.
    N : int
        Number of images to store in the ring buffer.

    """

    def __init__(self, **kwargs):
        """Initialize the ring buffer.

        Keyword arguments
        -----------------
        N : int
            Number of images to store in the ring buffer.
        directory : str
            The directory to buffer images to.
        filename : str
            Filename to use for the HDF5 file.
        recording : bool
            Activate recording when True, disable when False.
        logger : str
            The name of the logger to use. Defaults to 'RingBuffer'.
        roi : list
            The currently selected region of interest.

        """
        directory = kwargs.get('directory', '.')
        filename = kwargs.get('filename', 'rbuffer.h5')
        recording = kwargs.get('recording', True)
        N = int(kwargs.get('N', 100))
        logger = kwargs.get('logger', 'RingBuffer')
        roi = kwargs.get('roi', [10, 100, 10, 100])
        assert isinstance(directory, (str, unicode))
        assert isinstance(filename, (str, unicode))
        assert isinstance(recording, (int, bool))
        assert isinstance(logger, (str, unicode))
        assert isinstance(roi, (list, tuple, np.ndarray))

        self.recording = recording
        self.N = N
        self.logger = logging.getLogger(logger)
        self.roi = roi
        self._index = 0

        # Initialize HDF5 database.
        self.filename = os.path.join(directory, filename)
        self._db = tables.open_file(self.filename, 'w', title="Ring Buffer")
        self._db.create_group('/', 'images', 'Buffered Images')

    def __enter__(self):
        return self

    def __exit__(self, type_, value, tb):
        self.close()

    def close(self):
        self._db.close()

    def get_current_index(self):
        """Return the current index. This is in a function to
        hopefully prevent the user from accessing _index directly
        which could lead to bad things if it is modified!

        """
        return self._index

    def set_recording_state(self, state):
        """Explicitly set the recording state to state."""
        assert isinstance(state, (bool, int))
        self.recording = state

    def toggle(self):
        """Toggle the recording state."""
        self.recording = not self.recording

    def write(self, data):
        """Write data to the ring buffer file."""
        if not self.recording:
            return
        name = 'img{:04d}'.format(self._index)
        try:
            self._db.get_node('/images/' + name).remove()
        except tables.NoSuchNodeError:
            pass
        finally:
            # TODO: Adapt to CArray for compression
            #filters = tables.Filters(complevel=5, complib='zlib')
            arr = self._db.create_array('/images', name, data)
            arr.attrs.timestamp = datetime.strftime(datetime.now(), '%Y-%m-%d %H:%M:%S.%f')
            arr.attrs.roi = self.roi
            arr.flush()
        self._db.flush()
        self._index = self._index + 1 if self._index < self.N - 1 else 0

    def read(self, index):
        """Return data from the ring buffer file."""
        img = self._db.get_node('/images/img{:04d}'.format(index))
        return np.array(img)

    def get_roi(self, index):
        """Return the recorded ROI for the given index."""
        return self._db.get_node('/images/img{:04d}'.format(index)).attrs.roi

    def save_as(self, filename):
        """Save the ring buffer to file filename. The output format
        will depend on the extension of filename.

        """
        self.logger.warning("Saving ring buffers to non-HDF5 formats is not yet properly implemented!")

        # Save as PNG files in a zip archive.
        if filename[-3:] == 'zip':
            for node in self._db.list_nodes('/images'):
                data = node.read()
                imsave('./img.png', data)

if __name__ == "__main__":
    from numpy import random

    size = 512
    img_size = (size, size)
    with RingBuffer(N=100) as rb:
        for i in range(200):
            img = random.random(img_size)
            rb.write(img)
        rb.save_as('test.zip')


text:
export GOVC_INSECURE=true

govc license.add "$VCSA_LICENSE" "$ESX_LICENSE" >/dev/null

govc license.assign "$VCSA_LICENSE" >/dev/null

govc find / -type h | xargs -I% -n1 govc license.assign -host % "$ESX_LICENSE" >/dev/null

echo "Assigned licenses..."
govc license.assigned.ls

echo ""
echo "License usage..."
govc license.ls


text:
package caseless

import (
	"strings"
	"github.com/src/github.com/json-iterator/go"
	"fmt"
	"errors"
	"reflect"
)

/*

   Takes a json string and flattens json values (path --> value | range | pattern | set)

   This allows to compare JSON values with their history.

   Adapted from: https://github.com/chandler767/JSON-Path-Flattener/blob/master/main.go

   JSON values may be replaced by:

   - A regex pattern (e.g. a date YYYY-MM-DD pattern)
   - A range (e.g. 1-5)
   - A set (e.g. 10,20,30 or "yes", "no")

   This is set as part of user input and is saved on DB.

*/

// a path to value mapping

type PathValue struct {
	path string
	value string
}

type Flattener struct {
	err           error
	nestedjson    map[string]interface{}
	flattenedjson map[string]string
}

// Turn a JSON body into a map of flattened paths to values --> values

func (f *Flattener) Flatten (jsonToFlatten string) {

	// make sure there are no newlines in input:

	jsonToFlatten = strings.Replace(jsonToFlatten, "\n", "", -1)

	f.flattenedjson = make(map[string]string)

	var json = jsoniter.ConfigCompatibleWithStandardLibrary
	err := json.Unmarshal([]byte(jsonToFlatten), &f.nestedjson)

	if err != nil {
		fmt.Println(errors.New("could not unmarshal JSON. A valid path, URL, or string is required"))
		panic(err)
	}

	for key, val := range f.nestedjson {
		f.process(f.flattenedjson, key, reflect.ValueOf(val))
	}

	pathValues := make([]PathValue, len(f.flattenedjson))

	for key, val := range f.flattenedjson {

		pathValues = append(pathValues, PathValue{key, val})
		fmt.Println(fmt.Sprintf(".%s=%s", key, val))
	}

	return
}

// De-serialize from DB
// FlatBody is saved in db because often values are converted to patterns, ranges or lists after user input
// This will take the DB string value and explodes it into the local map

func (f *Flattener) Load (flatBody string) {

	// Convert to slice:

	lines := strings.Split(flatBody, "\n")

	// Each line is separated by the string @testcaseless@
	// Split KEY@testcasseless@VALUE

	for ix := range lines {
		kv := strings.Split(lines[ix], "@testcaseless@")
		f.flattenedjson[kv[0]] = kv[1]
	}

}

// get flattened map as string, this is for db storage

func (f *Flattener) AsString() string {

	s := ""
	for k, v := range f.flattenedjson {
		s = s + k + "@testcaseless@" + v + "\n"
	}
	return s
}

// return map of path --> value

func (f *Flattener) AsMap() map[string]string {
    return f.flattenedjson
}

// add a path --> value to the current map

func (f *Flattener) Add(path string, value string) {

	if f.flattenedjson == nil {
		f.flattenedjson = make(map[string]string)
	}

	f.flattenedjson[path] = value
}

func (f *Flattener) Exists(path string) bool {
	if _, ok := f.flattenedjson[path]; ok {
		return true
	}
	return false
}

func (f *Flattener) process(jsontoflatten map[string]string, prefix string, val reflect.Value) {

	if val.Kind() == reflect.Interface {
		val = val.Elem()
	}

	switch val.Kind() {

	case reflect.Bool:

		if val.Bool() {
			jsontoflatten[prefix] = "true"
		} else {
			jsontoflatten[prefix] = "false"
		}

	case reflect.Int:

		jsontoflatten[prefix] = fmt.Sprintf("%d", val.Int())

	case reflect.Float64:

		jsontoflatten[prefix] = strings.TrimRight(strings.TrimRight(fmt.Sprintf("%.2f", val), "0"), ".") // Remove trailing zeros.

	case reflect.Map:

		for _, key := range val.MapKeys() {

			if key.Kind() == reflect.Interface {
				key = key.Elem()
			}
			if key.Kind() != reflect.String {
				panic(fmt.Sprintf("Key is not string: %s", key))
			}
			f.process(jsontoflatten, fmt.Sprintf("%s.%s", prefix, key.String()), val.MapIndex(key))
		}

	case reflect.Slice:

		for i := 0; i < val.Len(); i++ {
			f.process(jsontoflatten, fmt.Sprintf("%s[%d]", prefix, i), val.Index(i))
		}

	case reflect.String:

		jsontoflatten[prefix] = val.String()

	default:

		if !val.IsValid() {
			jsontoflatten[prefix] = "nil" // Insert a nil as string.
		} else {
			panic(fmt.Sprintf("Unexpected JSON value: %s", val)) // Check JSON format.
		}
	}
}


text:
        <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
        <title>W28699_text</title>
        <link rel="stylesheet" type="text/css" href="style.css" />

                «

    <p class="styleSans107.0000<enum PANGO_WEIGHT_NORMAL of type PangoWeight><enum PANGO_STYLE_NORMAL of type PangoStyle>">Emerald Oil Inc Well Summary Joel Goodsen 5-32-29H Section 32 T149N R102W McKenzie County, ND

m 7", 32#, P-110. BTC, 8rd 10,780 / 1.0
a. Assume full casing evacuation with 10.0 ppg fl '
evacuation with 1.15 psi/ft equivalent collapse strengt
b. Burst pressure based on 9000 psig max press for stimulation plus 10 ppg fluid in casing and 9.0 ppg fluid on backside—to 10,946' TVD.
Based on string weight in 9.0 ppg fluid, (305k lbs) plus 100k
C.
Float Equipment: Centralizers placed on every joint across productive zones. Cement volumes are estimates based on 7" casing set in an 8-3/ "hole with 30% excess.
Pre-flush (Spacer): 20 bbls Diesel Preflush

    <p class="styleSans24.0000<enum PANGO_WEIGHT_NORMAL of type PangoWeight><enum PANGO_STYLE_NORMAL of type PangoStyle>">'Ju bbls Tuned Spacer 120 sks of Scavenger cement 10.8 ppg, 3.39 fl3/sk.
215 sx (102 bbls) WBL 11.3 ppg, 2.59 ft3/sk, with 0.5% CPL-4 + 0.4% CDF
-4P + 0.7% LTR + 0.25 lb per sx Poly E Flake (lost circulation additive)
'55 SK (124 bblS)  35  ppg, 1,55 ft3/sk_ with 100/. Mar'l Inuunun - A --~ -—'

    <p class="styleSans9.0000<enum PANGO_WEIGHT_NORMAL of type PangoWeight><enum PANGO_STYLE_NORMAL of type PangoStyle>">Lead Slurry:

    <p class="styleSans9.0000<enum PANGO_WEIGHT_NORMAL of type PangoWeight><enum PANGO_STYLE_NORMAL of type PangoStyle>">Tail Slurry:

    <p class="styleSans21.0000<enum PANGO_WEIGHT_NORMAL of type PangoWeight><enum PANGO_STYLE_NORMAL of type PangoStyle>">V r v ,-.., ....-. .u/u l'aUI \DVVUVV) "NJ-['70 CFL'4.+ U. /"/o CFR + 0.4% CDF-4P +2% LTR & 0.25 lb per sx Poly E Flake (lost circulation additive)


text:
[![Build Status](https://travis-ci.org/doerfli/webmock.svg?branch=master)](https://travis-ci.org/doerfli/webmock) [![Dependency Status](https://dependencyci.com/github/doerfli/webmock/badge)](https://dependencyci.com/github/doerfli/webmock)

# What's webmock?

Webmock is going to be a portal to create HTTP service mocks with customizable reponses.

More documentation will follow soon ...


text:
class User(object):
    def __init__(self, server, name, id, real_name, tz):
        self.tz = tz
        self.name = name
        self.real_name = real_name
        self.server = server
        self.id = id

    def __eq__(self, compare_str):
        return self.id == compare_str or self.name == compare_str

    def __str__(self):
        data = ""
        for key in list(self.__dict__.keys()):
            if key != "server":
                data += "{0}: {1}\n".format(key, str(self.__dict__[key])[:40])
        return data

    def __repr__(self):
        return self.__str__()


text:
package org.scijava.ui;

/**
 * Represents a plane of ARGB pixel data. Useful for passing around data during
 * copy/paste operations to external programs.
 *
 * @author Barry DeZonia
 */
public class ARGBPlane {

	// -- instance variables --

	private final int width;
	private final int height;
	private final int[] data;

	// -- constructors --

	public ARGBPlane(final int width, final int height, final int[] data) {
		this.width = width;
		this.height = height;
		this.data = data;
		if (width < 1 || height < 1) {
			throw new IllegalArgumentException(
				"Both width and height must be greater than zero");
		}
		if ((long) width * height > Integer.MAX_VALUE) {
			throw new IllegalArgumentException("Plane is too large");
		}
		if (data.length != width * height) {
			throw new IllegalArgumentException("Data size mismatch");
		}
	}

	public ARGBPlane(final int width, final int height) {
		this(width, height, new int[width * height]);
	}

	// -- public interface --

	public int getWidth() {
		return width;
	}

	public int getHeight() {
		return height;
	}

	public int[] getData() {
		return data;
	}

	public int getARGB(final int x, final int y) {
		return data[index(x, y)];
	}

	public int getAlpha(final int x, final int y) {
		return alpha(data[index(x, y)]);
	}

	public int getRed(final int x, final int y) {
		return red(data[index(x, y)]);
	}

	public int getGreen(final int x, final int y) {
		return green(data[index(x, y)]);
	}

	public int getBlue(final int x, final int y) {
		return blue(data[index(x, y)]);
	}

	public void setARGB(final int x, final int y, final int argb) {
		data[index(x, y)] = argb;
	}

	public void setAlpha(final int x, final int y, final int v) {
		int argb = data[index(x, y)];
		final int component = v & 0xff;
		argb &= ~0xff000000;
		argb |= (component << 24);
		data[index(x, y)] = argb;
	}

	public void setRed(final int x, final int y, final int v) {
		int argb = data[index(x, y)];
		final int component = v & 0xff;
		argb &= ~0xff0000;
		argb |= (component << 16);
		data[index(x, y)] = argb;
	}

	public void setGreen(final int x, final int y, final int v) {
		int argb = data[index(x, y)];
		final int component = v & 0xff;
		argb &= ~0xff00;
		argb |= (component << 8);
		data[index(x, y)] = argb;
	}

	public void setBlue(final int x, final int y, final int v) {
		int argb = data[index(x, y)];
		final int component = v & 0xff;
		argb &= ~0xff;
		argb |= (component << 0);
		data[index(x, y)] = argb;
	}

	// -- helpers --

	private int index(final int x, final int y) {
		return y * width + x;
	}

	private int alpha(final int argb) {
		return (argb >> 24) & 0xff;
	}

	private int red(final int argb) {
		return (argb >> 16) & 0xff;
	}

	private int green(final int argb) {
		return (argb >> 8) & 0xff;
	}

	private int blue(final int argb) {
		return (argb >> 0) & 0xff;
	}

}