text:
package alluxio.master.table;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import alluxio.AlluxioTestDirectory;
import alluxio.grpc.table.SyncStatus;
import alluxio.job.wire.Status;
import alluxio.master.LocalAlluxioJobCluster;
import alluxio.master.table.transform.TransformJobInfo;
import alluxio.testutils.BaseIntegrationTest;
import alluxio.testutils.LocalAlluxioClusterResource;
import alluxio.util.CommonUtils;
import alluxio.util.WaitForOptions;

import com.google.common.collect.Sets;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.testcontainers.containers.Container;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.utility.MountableFile;

import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

@Ignore("Enable after docker can be used from within tests")
public final class TableIntegrationTest extends BaseIntegrationTest {
  private static final String DB_NAME = "test";
  private static final String DB_NAME_ERRORS = "test_ERRORS";
  private static final File WAREHOUSE_DIR = AlluxioTestDirectory
      .createTemporaryDirectory(new File("/tmp/alluxio-tests"), "TableIntegrationTest");
  /** The docker image name and tag. */
  private static final String HMS_IMAGE = "<REPLACE WITH DOCKER IMAGE/TAG OF HMS>";
  private static final String TEST_TABLE = "test_table";
  private static final String TEST_TABLE_RENAME = "test_table_rename";
  private static final String TEST_TABLE_PART = "test_table_part";
  private static final String TEST_TABLE_ERROR = "table_bad_location";

  private static String sHmsAddress;
  private static int sHmsPort;

  @ClassRule
  public static TemporaryFolder sFolder = new TemporaryFolder();

  @ClassRule
  public static GenericContainer sHms =
      new GenericContainer<>(HMS_IMAGE).withExposedPorts(9083)
          .withFileSystemBind(WAREHOUSE_DIR.getAbsolutePath(), WAREHOUSE_DIR.getAbsolutePath());

  @ClassRule
  public static LocalAlluxioClusterResource sLocalAlluxioClusterResource =
      new LocalAlluxioClusterResource.Builder().build();

  private static LocalAlluxioJobCluster sLocalAlluxioJobCluster;
  private static TableMaster sTableMaster;

  @BeforeClass
  public static void beforeClass() throws Exception {
    sHmsAddress = sHms.getContainerIpAddress();
    sHmsPort = sHms.getMappedPort(9083);

    File dbDir = new File(WAREHOUSE_DIR, DB_NAME + ".db");
    dbDir.mkdirs();

    // write data files for the tables, to avoid the insert statement, which is slow.
    // write data files for table: test_table
    String base = dbDir.getAbsolutePath() + "/" + TEST_TABLE + "/";
    new File(base).mkdirs();
    PrintWriter writer = new PrintWriter(base + "data.csv");
    writer.println("1,11,111,1.1,1111,11111");
    writer.println("2,22,222,2.2,2222,22222");
    writer.close();

    // write data files for table: test_table_part
    base = dbDir.getAbsolutePath() + "/" + TEST_TABLE_PART + "/partitionint6=11111/";
    new File(base).mkdirs();
    writer =
        new PrintWriter(base + "data.csv");
    writer.println("1,11,111,1.1,1111");
    writer.close();
    base = dbDir.getAbsolutePath() + "/" + TEST_TABLE_PART + "/partitionint6=22222/";
    new File(base).mkdirs();
    writer =
        new PrintWriter(base + "data.csv");
    writer.println("2,22,222,2.2,2222");
    writer.close();

    // write data files for table: test_table_rename
    base = dbDir.getAbsolutePath() + "/" + TEST_TABLE_RENAME + "/";
    new File(base).mkdirs();
    writer = new PrintWriter(base + "data.csv");
    writer.println("1,11,111,1.1,1111,11111");
    writer.println("2,22,222,2.2,2222,22222");
    writer.close();

    String[] lines = {
        String.format("CREATE DATABASE %s; USE %s;", DB_NAME, DB_NAME),
        String.format(
            "CREATE EXTERNAL TABLE %s(`int1` int, `long2` bigint, `string3` string, "
                + "`float4` float, " + "`long5` bigint, `partitionint6` int) LOCATION 'file://%s';",
            TEST_TABLE, dbDir.getAbsolutePath() + "/" + TEST_TABLE),
        String.format("MSCK REPAIR TABLE %s;", TEST_TABLE),
        String.format(
            "CREATE EXTERNAL TABLE %s(`int1` int, `long2` bigint, `string3` string, "
                + "`float4` "
                + "float, `long5` bigint) PARTITIONED BY (partitionint6 INT) LOCATION 'file://%s';",
            TEST_TABLE_PART, dbDir.getAbsolutePath() + "/" + TEST_TABLE_PART),
        String.format("MSCK REPAIR TABLE %s;", TEST_TABLE_PART),
        String.format(
            "CREATE EXTERNAL TABLE %s(`int1` int, `long2` bigint, `string3` string, "
                + "`float4` float, " + "`long5` bigint, `partitionint6` int) LOCATION 'file://%s';",
            TEST_TABLE_RENAME, dbDir.getAbsolutePath() + "/" + TEST_TABLE_RENAME),

        // this database has a table which cannot be mounted.
        String.format("CREATE DATABASE %s; USE %s;", DB_NAME_ERRORS, DB_NAME_ERRORS),
        String.format(
            "CREATE EXTERNAL TABLE %s(`int1` int, `long2` bigint, `string3` string, "
                + "`float4` float, " + "`long5` bigint, `partitionint6` int) LOCATION 'file://%s';",
            TEST_TABLE, dbDir.getAbsolutePath() + "/" + TEST_TABLE),
        String.format("MSCK REPAIR TABLE %s;", TEST_TABLE),
        String.format("CREATE EXTERNAL TABLE %s(`int1` int) LOCATION 'file:///does/not/exist';",
            TEST_TABLE_ERROR),
    };

    execBeeline(Arrays.asList(lines));

    sLocalAlluxioJobCluster = new LocalAlluxioJobCluster();
    sLocalAlluxioJobCluster.start();

    sTableMaster = sLocalAlluxioClusterResource.get().getLocalAlluxioMaster().getMasterProcess()
        .getMaster(TableMaster.class);

    sTableMaster
        .attachDatabase("hive", "thrift://" + sHmsAddress + ":" + sHmsPort, DB_NAME, DB_NAME,
            Collections.emptyMap(), false);
  }

  @AfterClass
  public static void afterClass() throws Exception {
    sLocalAlluxioJobCluster.stop();
  }

  @After
  public void after() throws Exception {
    if (Sets.newHashSet(sTableMaster.getAllDatabases()).contains(DB_NAME_ERRORS)) {
      sTableMaster.detachDatabase(DB_NAME_ERRORS);
    }
  }

  @Test
  public void attachWrongDb() throws Exception {
    try {
      sTableMaster.attachDatabase("hive", "thrift://" + sHmsAddress + ":" + sHmsPort, "wrong_db",
          "wrong_db", Collections.emptyMap(), false);
      fail("Attaching wrong db name is expected to fail.");
    } catch (Exception e) {
      // this is expected
    }
  }

  @Test
  public void attachMountFail() throws Exception {
    SyncStatus status = sTableMaster
        .attachDatabase("hive", "thrift://" + sHmsAddress + ":" + sHmsPort, DB_NAME_ERRORS,
            DB_NAME_ERRORS, Collections.emptyMap(), false);
    assertEquals(1, status.getTablesErrorsCount());
    assertTrue(status.containsTablesErrors(TEST_TABLE_ERROR));
    try {
      sTableMaster.getDatabase(DB_NAME_ERRORS);
      fail("DB with a bad table should be not be attached.");
    } catch (Exception e) {
      // this is expected
    }
    status = sTableMaster
        .attachDatabase("hive", "thrift://" + sHmsAddress + ":" + sHmsPort, DB_NAME_ERRORS,
            DB_NAME_ERRORS, Collections.emptyMap(), true);
    assertEquals(1, status.getTablesErrorsCount());
    assertTrue(status.containsTablesErrors(TEST_TABLE_ERROR));
    assertNotNull(sTableMaster.getDatabase(DB_NAME_ERRORS));
  }

  @Test
  public void attachIgnore() throws Exception {
    Map<String, String> options = new HashMap<>();
    options.put(CatalogProperty.DB_IGNORE_TABLES.getName(), TEST_TABLE_ERROR + ",other,");

    // expect no errors if the bad table is ignored
    SyncStatus status = sTableMaster
        .attachDatabase("hive", "thrift://" + sHmsAddress + ":" + sHmsPort, DB_NAME_ERRORS,
            DB_NAME_ERRORS, options, false);
    assertEquals(0, status.getTablesErrorsCount());
    assertEquals(1, status.getTablesIgnoredCount());

    // expect no errors on a sync, if the bad table is ignored
    status = sTableMaster.syncDatabase(DB_NAME_ERRORS);
    assertEquals(0, status.getTablesErrorsCount());
    assertEquals(1, status.getTablesIgnoredCount());
  }

  @Test
  public void syncUnmodifiedTable() throws Exception {
    Table table = sTableMaster.getTable(DB_NAME, TEST_TABLE);
    long version = table.getVersion();
    table = sTableMaster.getTable(DB_NAME, TEST_TABLE_PART);
    long versionPart = table.getVersion();

    sTableMaster.syncDatabase(DB_NAME);
    assertEquals(version, sTableMaster.getTable(DB_NAME, TEST_TABLE).getVersion());
    assertEquals(versionPart, sTableMaster.getTable(DB_NAME, TEST_TABLE_PART).getVersion());
  }

  @Test
  public void syncModifiedTable() throws Exception {
    Table table = sTableMaster.getTable(DB_NAME, TEST_TABLE);
    long version = table.getVersion();
    table = sTableMaster.getTable(DB_NAME, TEST_TABLE_PART);
    long versionPart = table.getVersion();
    Set<Long> partitionVersions =
        table.getPartitions().stream().map(Partition::getVersion).collect(Collectors.toSet());

    // update the tables and partitions
    String[] lines = {
        String.format("USE %s;", DB_NAME),
        String.format("ALTER TABLE %s SET LOCATION 'file://%s/test.db/%s';",
            TEST_TABLE, WAREHOUSE_DIR.getAbsolutePath(), TEST_TABLE),
        String.format(
            "ALTER TABLE %s partition(partitionint6=22222) SET LOCATION "
                + "'file://%s/test.db/%s/partitionint6=22222/';",
            TEST_TABLE_PART, WAREHOUSE_DIR.getAbsolutePath(), TEST_TABLE_PART),
    };
    execBeeline(Arrays.asList(lines));

    sTableMaster.syncDatabase(DB_NAME);
    long newVersion = sTableMaster.getTable(DB_NAME, TEST_TABLE).getVersion();
    long newVersionPart = sTableMaster.getTable(DB_NAME, TEST_TABLE_PART).getVersion();
    Set<Long> newPartitionVersions =
        sTableMaster.getTable(DB_NAME, TEST_TABLE_PART).getPartitions().stream()
            .map(Partition::getVersion).collect(Collectors.toSet());
    assertNotEquals(version, newVersion);
    assertNotEquals(versionPart, newVersionPart);
    assertNotEquals(partitionVersions, newPartitionVersions);
    sTableMaster.syncDatabase(DB_NAME);
    assertEquals(newVersion, sTableMaster.getTable(DB_NAME, TEST_TABLE).getVersion());
    assertEquals(newVersionPart, sTableMaster.getTable(DB_NAME, TEST_TABLE_PART).getVersion());
    assertEquals(newPartitionVersions,
        sTableMaster.getTable(DB_NAME, TEST_TABLE_PART).getPartitions().stream()
            .map(Partition::getVersion).collect(Collectors.toSet()));
  }

  @Test
  public void syncModifiedTransformedTable() throws Exception {
    Table table = sTableMaster.getTable(DB_NAME, TEST_TABLE);
    long version = table.getVersion();
    table = sTableMaster.getTable(DB_NAME, TEST_TABLE_PART);
    long versionPart = table.getVersion();
    Set<Long> partitionVersions =
        table.getPartitions().stream().map(Partition::getVersion).collect(Collectors.toSet());

    // transform tables
    long jobId = sTableMaster.transformTable(DB_NAME, TEST_TABLE, null);
    long jobPartId = sTableMaster.transformTable(DB_NAME, TEST_TABLE_PART, null);

    String definition1 = sTableMaster.getTransformJobInfo(jobId).getDefinition();
    String definition2 = sTableMaster.getTransformJobInfo(jobPartId).getDefinition();

    CommonUtils.waitFor("Transforms to complete", () -> {
      try {
        TransformJobInfo jobInfo1 = sTableMaster.getTransformJobInfo(jobId);
        TransformJobInfo jobInfo2 = sTableMaster.getTransformJobInfo(jobPartId);
        return jobInfo1.getJobStatus() == Status.COMPLETED
            && jobInfo2.getJobStatus() == Status.COMPLETED;
      } catch (Exception e) {
        return false;
      }
    }, WaitForOptions.defaults().setTimeoutMs(30000).setInterval(1000));

    // verify transformed partitions
    table = sTableMaster.getTable(DB_NAME, TEST_TABLE);
    assertEquals(1, table.getPartitions().size());
    assertTrue(table.getPartitions().get(0).isTransformed(definition1));
    table = sTableMaster.getTable(DB_NAME, TEST_TABLE_PART);
    assertEquals(2, table.getPartitions().size());
    assertTrue(table.getPartitions().get(0).isTransformed(definition2));
    assertTrue(table.getPartitions().get(1).isTransformed(definition2));

    // update the tables and partitions
    String[] lines = {
        String.format("USE %s;", DB_NAME),
        String.format("ALTER TABLE %s SET LOCATION 'file://%s/test.db/%s';",
            TEST_TABLE, WAREHOUSE_DIR.getAbsolutePath(), TEST_TABLE),
        String.format(
            "ALTER TABLE %s partition(partitionint6=22222) SET LOCATION "
                + "'file://%s/test.db/%s/partitionint6=22222/';",
            TEST_TABLE_PART, WAREHOUSE_DIR.getAbsolutePath(), TEST_TABLE_PART),
    };
    execBeeline(Arrays.asList(lines));

    // verify new versions after a sync
    sTableMaster.syncDatabase(DB_NAME);
    long newVersion = sTableMaster.getTable(DB_NAME, TEST_TABLE).getVersion();
    long newVersionPart = sTableMaster.getTable(DB_NAME, TEST_TABLE_PART).getVersion();
    Set<Long> newPartitionVersions =
        sTableMaster.getTable(DB_NAME, TEST_TABLE_PART).getPartitions().stream()
            .map(Partition::getVersion).collect(Collectors.toSet());
    assertNotEquals(version, newVersion);
    assertNotEquals(versionPart, newVersionPart);
    assertNotEquals(partitionVersions, newPartitionVersions);

    // verify only the updated partitions have invalidated the transformations
    table = sTableMaster.getTable(DB_NAME, TEST_TABLE);
    assertEquals(1, table.getPartitions().size());
    assertFalse(table.getPartitions().get(0).isTransformed(definition1));
    version = table.getPartitions().get(0).getVersion();
    table = sTableMaster.getTable(DB_NAME, TEST_TABLE_PART);
    assertEquals(2, table.getPartitions().size());
    assertTrue(table.getPartitions().get(0).isTransformed(definition2)
        ^ table.getPartitions().get(1).isTransformed(definition2));
    partitionVersions =
        table.getPartitions().stream().map(Partition::getVersion).collect(Collectors.toSet());

    // verify transformed partition versions are unchanged when syncing an unmodified udb
    sTableMaster.syncDatabase(DB_NAME);
    assertEquals(version,
        sTableMaster.getTable(DB_NAME, TEST_TABLE).getPartitions().get(0).getVersion());
    newPartitionVersions =
        sTableMaster.getTable(DB_NAME, TEST_TABLE_PART).getPartitions().stream()
            .map(Partition::getVersion).collect(Collectors.toSet());
    assertEquals(partitionVersions, newPartitionVersions);
  }

  @Test
  public void renameTable() throws Exception {
    String newName = TEST_TABLE_RENAME + "_new";

    // update the tables and partitions
    String[] lines = {
        String.format("USE %s;", DB_NAME),
        String.format("ALTER TABLE %s RENAME TO %s;", TEST_TABLE_RENAME, newName),
    };
    execBeeline(Arrays.asList(lines));

    Table table = sTableMaster.getTable(DB_NAME, TEST_TABLE_RENAME);
    assertNotNull(table);
    try {
      sTableMaster.getTable(DB_NAME, newName);
      fail("getting non-existing table should not succeed.");
    } catch (Exception e) {
      // expected
    }

    sTableMaster.syncDatabase(DB_NAME);

    try {
      sTableMaster.getTable(DB_NAME, TEST_TABLE_RENAME);
      fail("previous table name should not exist after the rename.");
    } catch (Exception e) {
      // expected
    }

    table = sTableMaster.getTable(DB_NAME, newName);
    assertNotNull(table);
  }

  /**
   * Runs a sequence of commands via beeline, within the HMS container.
   *
   * @param commands the list of commands
   */
  private static void execBeeline(List<String> commands) throws Exception {
    File file = sFolder.newFile();
    PrintWriter writer = new PrintWriter(file.getAbsolutePath());
    for (String line : commands) {
      writer.println(line);
    }
    writer.close();

    sHms.copyFileToContainer(MountableFile.forHostPath(file.getAbsolutePath()),
        "/opt/" + file.getName());
    Container.ExecResult result =
        sHms.execInContainer("/opt/hive/bin/beeline", "-u", "jdbc:hive2://", "-f",
            "/opt/" + file.getName());

    if (result.getExitCode() != 0) {
      throw new IOException("beeline commands failed. stderr: " + result.getStderr());
    }
  }
}


text:
import {sheet} from 'emotion';
import serializer from 'jest-glamor-react';

expect.addSnapshotSerializer(serializer(sheet));

redpajama_set_name: RedPajamaGithub


text:
gcc -o genMake genMake.c
./genMake
rm genMake
./clean.sh
./build.sh
./installOpenIndiana.sh

# compiling using make:
# make clean
# make install

# reset Makefile (modified on ARM platforms)
git checkout Makefile
git checkout build.sh


text:
package tests

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"regexp"
	"sort"
	"strings"
	"testing"

	"github.com/ethereum/go-ethereum/params"
)

var (
	baseDir            = filepath.Join(".", "testdata")
	blockTestDir       = filepath.Join(baseDir, "BlockchainTests")
	stateTestDir       = filepath.Join(baseDir, "GeneralStateTests")
	transactionTestDir = filepath.Join(baseDir, "TransactionTests")
	vmTestDir          = filepath.Join(baseDir, "VMTests")
	rlpTestDir         = filepath.Join(baseDir, "RLPTests")
	difficultyTestDir  = filepath.Join(baseDir, "BasicTests")
)

func readJson(reader io.Reader, value interface{}) error {
	data, err := ioutil.ReadAll(reader)
	if err != nil {
		return fmt.Errorf("error reading JSON file: %v", err)
	}
	if err = json.Unmarshal(data, &value); err != nil {
		if syntaxerr, ok := err.(*json.SyntaxError); ok {
			line := findLine(data, syntaxerr.Offset)
			return fmt.Errorf("JSON syntax error at line %v: %v", line, err)
		}
		return err
	}
	return nil
}

func readJsonFile(fn string, value interface{}) error {
	file, err := os.Open(fn)
	if err != nil {
		return err
	}
	defer file.Close()

	err = readJson(file, value)
	if err != nil {
		return fmt.Errorf("%s in file %s", err.Error(), fn)
	}
	return nil
}

// findLine returns the line number for the given offset into data.
func findLine(data []byte, offset int64) (line int) {
	line = 1
	for i, r := range string(data) {
		if int64(i) >= offset {
			return
		}
		if r == '\n' {
			line++
		}
	}
	return
}

// testMatcher controls skipping and chain config assignment to tests.
type testMatcher struct {
	configpat    []testConfig
	failpat      []testFailure
	skiploadpat  []*regexp.Regexp
	skipshortpat []*regexp.Regexp
}

type testConfig struct {
	p      *regexp.Regexp
	config params.ChainConfig
}

type testFailure struct {
	p      *regexp.Regexp
	reason string
}

// skipShortMode skips tests matching when the -short flag is used.
func (tm *testMatcher) skipShortMode(pattern string) {
	tm.skipshortpat = append(tm.skipshortpat, regexp.MustCompile(pattern))
}

// skipLoad skips JSON loading of tests matching the pattern.
func (tm *testMatcher) skipLoad(pattern string) {
	tm.skiploadpat = append(tm.skiploadpat, regexp.MustCompile(pattern))
}

// fails adds an expected failure for tests matching the pattern.
func (tm *testMatcher) fails(pattern string, reason string) {
	if reason == "" {
		panic("empty fail reason")
	}
	tm.failpat = append(tm.failpat, testFailure{regexp.MustCompile(pattern), reason})
}

// config defines chain config for tests matching the pattern.
func (tm *testMatcher) config(pattern string, cfg params.ChainConfig) {
	tm.configpat = append(tm.configpat, testConfig{regexp.MustCompile(pattern), cfg})
}

// findSkip matches name against test skip patterns.
func (tm *testMatcher) findSkip(name string) (reason string, skipload bool) {
	if testing.Short() {
		for _, re := range tm.skipshortpat {
			if re.MatchString(name) {
				return "skipped in -short mode", false
			}
		}
	}
	for _, re := range tm.skiploadpat {
		if re.MatchString(name) {
			return "skipped by skipLoad", true
		}
	}
	return "", false
}

// findConfig returns the chain config matching defined patterns.
func (tm *testMatcher) findConfig(name string) *params.ChainConfig {
	// TODO(fjl): name can be derived from testing.T when min Go version is 1.8
	for _, m := range tm.configpat {
		if m.p.MatchString(name) {
			return &m.config
		}
	}
	return new(params.ChainConfig)
}

// checkFailure checks whether a failure is expected.
func (tm *testMatcher) checkFailure(t *testing.T, name string, err error) error {
	// TODO(fjl): name can be derived from t when min Go version is 1.8
	failReason := ""
	for _, m := range tm.failpat {
		if m.p.MatchString(name) {
			failReason = m.reason
			break
		}
	}
	if failReason != "" {
		t.Logf("expected failure: %s", failReason)
		if err != nil {
			t.Logf("error: %v", err)
			return nil
		} else {
			return fmt.Errorf("test succeeded unexpectedly")
		}
	}
	return err
}

// walk invokes its runTest argument for all subtests in the given directory.
//
// runTest should be a function of type func(t *testing.T, name string, x <TestType>),
// where TestType is the type of the test contained in test files.
func (tm *testMatcher) walk(t *testing.T, dir string, runTest interface{}) {
	// Walk the directory.
	dirinfo, err := os.Stat(dir)
	if os.IsNotExist(err) || !dirinfo.IsDir() {
		fmt.Fprintf(os.Stderr, "can't find test files in %s, did you clone the tests submodule?\n", dir)
		t.Skip("missing test files")
	}
	err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		name := filepath.ToSlash(strings.TrimPrefix(path, dir+string(filepath.Separator)))
		if info.IsDir() {
			if _, skipload := tm.findSkip(name + "/"); skipload {
				return filepath.SkipDir
			}
			return nil
		}
		if filepath.Ext(path) == ".json" {
			t.Run(name, func(t *testing.T) { tm.runTestFile(t, path, name, runTest) })
		}
		return nil
	})
	if err != nil {
		t.Fatal(err)
	}
}

func (tm *testMatcher) runTestFile(t *testing.T, path, name string, runTest interface{}) {
	if r, _ := tm.findSkip(name); r != "" {
		t.Skip(r)
	}
	t.Parallel()

	// Load the file as map[string]<testType>.
	m := makeMapFromTestFunc(runTest)
	if err := readJsonFile(path, m.Addr().Interface()); err != nil {
		t.Fatal(err)
	}

	// Run all tests from the map. Don't wrap in a subtest if there is only one test in the file.
	keys := sortedMapKeys(m)
	if len(keys) == 1 {
		runTestFunc(runTest, t, name, m, keys[0])
	} else {
		for _, key := range keys {
			name := name + "/" + key
			t.Run(key, func(t *testing.T) {
				if r, _ := tm.findSkip(name); r != "" {
					t.Skip(r)
				}
				runTestFunc(runTest, t, name, m, key)
			})
		}
	}
}

func makeMapFromTestFunc(f interface{}) reflect.Value {
	stringT := reflect.TypeOf("")
	testingT := reflect.TypeOf((*testing.T)(nil))
	ftyp := reflect.TypeOf(f)
	if ftyp.Kind() != reflect.Func || ftyp.NumIn() != 3 || ftyp.NumOut() != 0 || ftyp.In(0) != testingT || ftyp.In(1) != stringT {
		panic(fmt.Sprintf("bad test function type: want func(*testing.T, string, <TestType>), have %s", ftyp))
	}
	testType := ftyp.In(2)
	mp := reflect.New(reflect.MapOf(stringT, testType))
	return mp.Elem()
}

func sortedMapKeys(m reflect.Value) []string {
	keys := make([]string, m.Len())
	for i, k := range m.MapKeys() {
		keys[i] = k.String()
	}
	sort.Strings(keys)
	return keys
}

func runTestFunc(runTest interface{}, t *testing.T, name string, m reflect.Value, key string) {
	reflect.ValueOf(runTest).Call([]reflect.Value{
		reflect.ValueOf(t),
		reflect.ValueOf(name),
		m.MapIndex(reflect.ValueOf(key)),
	})
}


text:
package rpc

import (
	"context"
	"net"
)

// NewInProcClient attaches an in-process connection to the given RPC server.
func DialInProc(handler *Server) *Client {
	initctx := context.Background()
	c, _ := newClient(initctx, func(context.Context) (net.Conn, error) {
		p1, p2 := net.Pipe()
		go handler.ServeCodec(NewJSONCodec(p1), OptionMethodInvocation|OptionSubscriptions)
		return p2, nil
	})
	return c
}


text:
<web-app version="2.4"
	xmlns="http://java.sun.com/xml/ns/j2ee" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://java.sun.com/xml/ns/j2ee
	http://java.sun.com/xml/ns/j2ee/web-app_2_4.xsd">

    <display-name>静态资源服务器</display-name>

    <context-param>
        <param-name>contextConfigLocation</param-name>
        <param-value>
            classpath*:config/applicationContext-*.xml
        </param-value>
    </context-param>

    <listener>
        <listener-class>
            org.springframework.web.context.ContextLoaderListener
        </listener-class>
    </listener>

    <filter>
        <filter-name>characterEncodingFilter</filter-name>
        <filter-class>org.springframework.web.filter.CharacterEncodingFilter</filter-class>
        <init-param>
            <param-name>encoding</param-name>
            <param-value>UTF-8</param-value>
        </init-param>
    </filter>

    <filter-mapping>
        <filter-name>characterEncodingFilter</filter-name>
        <url-pattern>/*</url-pattern>
    </filter-mapping>

    <!--<filter>-->
        <!--<filter-name>authFilter</filter-name>-->
        <!--<filter-class>org.interfacedesign.filter.AuthFilter</filter-class>-->
        <!--<init-param>-->
            <!--<param-name>exclude</param-name>-->
            <!--<param-value>user</param-value>-->
        <!--</init-param>-->
    <!--</filter>-->

    <!--<filter-mapping>-->
        <!--<filter-name>authFilter</filter-name>-->
        <!--<url-pattern>*</url-pattern>-->
    <!--</filter-mapping>-->

    <!-- 配置springSecurityFilter -->
    <filter>
        <filter-name>springSecurityFilterChain</filter-name>
        <filter-class>org.springframework.web.filter.DelegatingFilterProxy</filter-class>
    </filter>
    <filter-mapping>
        <filter-name>springSecurityFilterChain</filter-name>
        <url-pattern>/*</url-pattern>
    </filter-mapping>

    <servlet>
        <servlet-name>dispatcherServlet</servlet-name>
        <servlet-class>org.springframework.web.servlet.DispatcherServlet</servlet-class>
        <init-param>
            <param-name>contextConfigLocation</param-name>
            <param-value>
                classpath:config/spring-mvc.xml
            </param-value>
        </init-param>
        <load-on-startup>1</load-on-startup>
    </servlet>

    <servlet-mapping>
        <servlet-name>dispatcherServlet</servlet-name>
        <url-pattern>/</url-pattern>
    </servlet-mapping>

    <!--<filter>-->
        <!--<filter-name>hibernateFilter</filter-name>-->
        <!--<filter-class>-->
            <!--org.springframework.orm.hibernate4.support.OpenSessionInViewFilter-->
        <!--</filter-class>-->
    <!--</filter>-->

    <!--<filter-mapping>-->
        <!--<filter-name>hibernateFilter</filter-name>-->
        <!--<url-pattern>/designer</url-pattern>-->
    <!--</filter-mapping>-->

    <error-page>
        <error-code>404</error-code>
        <location>/views/error/404.jsp</location>
    </error-page>

    <welcome-file-list>
        <welcome-file>/views/index.jsp</welcome-file>
    </welcome-file-list>

    <!--<error-page>-->
        <!--<error-code>500</error-code>-->
        <!--<location>/WEB-INF/views/error/500.jsp</location>-->
    <!--</error-page>-->
</web-app>


text:
body
{ background-color:#fff; color:#333; font-family:Tahoma,Arial,sans-serif; font-size:1em; line-height:140%; margin:0; padding:0;}

h1
{ background:#00669F none repeat scroll 0 0; color:#fff; font-size:1.5em; margin:0; padding:20px; }

h2
{ border-bottom:1px solid #DDDDDD; color:#00527F; font-size:1.3em; margin:0; padding:25px 0 10px 20px; }

h3
{ border-bottom:1px solid #BBBBBB; color:#00527F; font-size:1.2em; margin:0; padding:25px 0 10px 20px;}

h4
{ color:#00527F; font-size:1.2em; font-weight: bold; margin:0; padding: 25px 0 10px 0px;}

p
{ color:#222222; font-family:Times,Verdana,serif; font-size:1.1em; line-height:120%; margin:0; padding:10px 10px 10px 20px; }

ul
{ list-style: none; margin:0; padding: 10px 10px 10px 20px; }

ul li
{ margin: 0 0 0 -20px; padding:0 0 20px 20px; }

ul li p
{ padding: 2px 0; }

ul > li > p:first-child
{ font-weight: bold; font-family:Tahoma,Arial,sans-serif; font-size: 1em; }

pre
{ background:#ddd; font-size: 1em; font-family:monospace; margin:5px; padding:10px; overflow: auto; width: 95%; border: 1px dotted; }

code
{ font-family:monospace; font-weight:bold; font-size: 1em; }

a
{ color: #00759F; font-size: 1.05em; text-decoration: none; }

a:hover
{ color: #00759F; text-decoration: underline; cursor: pointer; }

a:visited
{ color: #003D5F; }

.parameter-name { display:block; }

table.documentation-list td { width: 10%; }

table { border-collapse: collapse; margin: 10px; width: 95%; }

tbody td, thead th { margin: 0; padding: 5; border: 1px solid #000; }

h1 a { color: #fff }

ul.topnav li { display: inline; margin: 5px; }


text:
__author__ = 'luckydonald'
import logging
logger = logging.getLogger(__name__)

#sender.py
class NoResponse(Exception):
	pass

#sender.py
class ConnectionError(NoResponse):
	pass

#sender.py
class UnknownFunction(Exception):
	pass

#result_parser.py/sender.py
class IllegalResponseException(Exception):
	pass

#argument_types.py
class ArgumentParseError(Exception):
	pass


text:
print 'first num ?',
age = raw_input()
print 'second num ?',
height = raw_input()

print "So you're %r old , %r tall ."%(age,
height)


text:
factors = [999, 999]

palindrome = 0

begin
  factors[1] = 999

  begin
    number = factors.reduce(&:*).to_s
    palindrome = number.to_i if number == number.reverse && number.to_i > palindrome

    factors[1] -= 1
  end while factors[1] > 99

  factors[0] -= 1
end while factors[0] > 99

puts palindrome


text:
var __extends = this.__extends || function (d, b) {
    for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
    function __() { this.constructor = d; }
    __.prototype = b.prototype;
    d.prototype = new __();
};
var Parrot;
(function (Parrot) {
    (function (Renderers) {
        var DocTypeRenderer = (function (_super) {
            __extends(DocTypeRenderer, _super);
            function DocTypeRenderer() {
                _super.apply(this, arguments);
                this.DefaultChildTag = "div";
                this.Elements = ["doctype"];
            }
            DocTypeRenderer.prototype.Render = function (statement, host, model, rendererProvider) {
                var value = "html";

                if (statement.Parameters.length > 0) {
                    var localModel = this.GetLocalModelValue(host, statement, model);
                    value = localModel;
                }

                return "<!DOCTYPE " + value + ">";
            };

            DocTypeRenderer.prototype.AfterRender = function (statement, host, model, rendererProvider) {
            };
            return DocTypeRenderer;
        })(Parrot.Renderers.BaseRenderer);
        Renderers.DocTypeRenderer = DocTypeRenderer;
    })(Parrot.Renderers || (Parrot.Renderers = {}));
    var Renderers = Parrot.Renderers;
})(Parrot || (Parrot = {}));
//# sourceMappingURL=docTypeRenderer.js.map


text:
const {spawnLightningCluster} = require('ln-docker-daemons');
const {test} = require('@alexbosworth/tap');

const {createCluster} = require('./../macros');
const {createInvoice} = require('./../../');
const {delay} = require('./../macros');
const {getChannels} = require('./../../');
const {getRouteToDestination} = require('./../../');
const {openChannel} = require('./../../');
const {pay} = require('./../../');
const {setupChannel} = require('./../macros');
const {waitForChannel} = require('./../macros');
const {waitForPendingChannel} = require('./../macros');

const channelCapacityTokens = 1e6;
const confirmationCount = 20;
const defaultFee = 1e3;
const size = 2;
const tokens = 1e3;

// Rebalancing channels should result in balanced channels
test('Rebalance', async ({end, equal}) => {
  const {kill, nodes} = await spawnLightningCluster({size});

  const [control, target] = nodes;

  const {generate, lnd} = control;

  await setupChannel({generate, lnd, give: 1e5, to: target});

  await setupChannel({
    generate: target.generate,
    give: 1e5,
    lnd: target.lnd,
    to: control,
  });

  const invoice = await createInvoice({lnd, tokens});

  const [inChannelId] = (await getChannels({lnd})).channels.map(({id}) => id);

  const {route} = await getRouteToDestination({
    lnd,
    tokens,
    destination: control.id,
    outgoing_channel: inChannelId,
    payment: invoice.payment,
    total_mtokens: !!invoice.payment ? invoice.mtokens : undefined,
  });

  const selfPay = await pay({lnd, path: {id: invoice.id, routes: [route]}});

  equal(selfPay.secret, invoice.secret, 'Payment made to self');

  await kill({});

  return end();
});


text:
SYNONYM

#### According to
The Catalogue of Life, 3rd January 2011

#### Published in
Garden 46:547. 1894

#### Original name
null

### Remarks
null


text:
"Files-related handler"

import os
from cannelloni.settings import BASE_DIR

def files_handler(request, variable):
    "Handle file upload"
    target = os.path.join(BASE_DIR, 'files', str(variable.uuid))
    with open(target, 'wb+') as destination:
        for chunk in request.FILES[str(variable.uuid)].chunks():
            destination.write(chunk)
    variable.value = target


text:
struct DefinedInBImpl {
  void f();
  struct Inner {};
  friend void FoundByADL(DefinedInBImpl);
};


text:
namespace content {

// Tests can derive from this to implement special test-specific resources.
// It's assumed that a test will only need one mock resource, so it can
// static_cast to get its own implementation.
class MockResource : public ppapi::Resource {
 public:
  MockResource(PP_Instance instance)
      : Resource(ppapi::OBJECT_IS_IMPL, instance) {}

 private:
  ~MockResource() override {}
};

}  // namespace content

#endif  // CONTENT_RENDERER_PEPPER_MOCK_RESOURCE_H_


text:
namespace Voxels
{

/// Interface for implementing a surface that can be voxelized in a
/// Voxel grid. Wgile the grid is being generated it will call the
/// GetSurface method that must feed it the distance between the surface and the
/// points in space it's called for.
class VOXELS_API VoxelSurface
{
public:
	virtual ~VoxelSurface() {}

	/// Called when generating a Voxel grid for this surface.
	/// You must provide linear distances to the supplied points.
	/// @param xStart starting coordinate on X axis
	/// @param xEnd ending coordinate on X axis
	/// @param xStep setp on the X axis
	/// @param yStart starting coordinate on Y axis
	/// @param yEnd ending coordinate on Y axis
	/// @param yStep setp on the Y axis
	/// @param zStart starting coordinate on Z axis
	/// @param zEnd ending coordinate on Z axis
	/// @param zStep setp on the Z axis
	/// @param output the distance values you need to fill.
	/// @param materialid the material id values you need to fill.
	/// Can be nullptr - in that case ignore the parameter.
	/// @param blend the blend values you need to fill.
	/// Can be nullptr - in that case ignore the parameter.
	/// values should be filled on the X, Y and Z axis in that order
	virtual void GetSurface(float xStart, float xEnd, float xStep,
								float yStart, float yEnd, float yStep,
								float zStart, float zEnd, float zStep,
								float* output,
								unsigned char* materialid,
								unsigned char* blend) = 0;
};

}


text:
layout: post
title: "Dynamo: Amazon's Highly Available Key value Store"
date: 2015-03-09
comments: True
categories: "big-data"
description: "Dynamo is a highly available key-value storage system built by Amazon. It sacrifices a consistency under certain failure scenarios, makes extensive use of object versioning and application-assisted conflict resolution in a manner that provides a novel interface for developers to use."
---

Dynamo is a highly available key-value storage system built by Amazon. It sacrifices a consistency under certain failure scenarios, makes extensive use of object versioning and application-assisted conflict resolution in a manner that provides a novel interface for developers to use.

<!--more-->

<hr class="soft"/>

### Motivation For building up Dynamo
* To provide a distributed key-value storage system with good scalability, high availability, and also satisfy their Service Level Agreement, which guarantees a majority of requests can be done in a certain time bound.

### System Assumption and Requirements
* Query Model
  * => They assume that only simple Read / Write operation to data item identified by a uniquely key.

* Crucial Efficiency requirement for low latency
  * => In the 99.9% of the distribution, there should be really tight response time

* Weak ACID
  * => For the purpose of higher available, relaxed Consistency + no isolation guarantee is acceptable

### Three Major Design Decisions

1. Sacrifice Strong Consistency for High availability.
2. Resolve replication conflict during read instead of write.
  * => "Always writable" System
3. Decentralized Server Architecture
  * => "Peer to peer" Fashioned System

### General Architecture Design

* Date Partition - Suit the incremental scalability requirements
  * Use Consistent Hash to partition keys and organize its position, treat output ranges as a circular "ring".
    * So that there is no need to "rehash" all the keys when the system is scaled up
  * Use Virtual Nodes to make physical nodes have multiple virtual nodes.
    1. Spread data evenly across the system.
    2. Balance the load.
  * Replicated Keys in N nodes. Keep a "Preference list" for keys, which is the server that responsible

* Data Versioning
  * => keep a list of \<node, counter\>(vector clock) for data merge in read
    * For causual results, merge the records with the latest result
    * For non-causual results, keep conflicts and sent results to client

* Read / Write Execution : "Sloppy Quorum" to keep replica consistency
  * Requires a number R/W of nodes must participate in a successful read/write operation.(R + W > N, R or W < N)
    * Majority vote mechanism for select successful operation
  * All read/write operations are performed on first N healthy nodes from the preference list
    * So it is "sloppy", rather than strict

* To handle Temporary Node/Network Failure: __Hinted Handoff Mechanism__
  * Basic Idea
      * Select another node temporary replacing the failure node and receive replicas.
      * Those replicas will be labeled with a hint, and stored in a separate local database that is scanned periodically.
      * Send replicas back when node failure is resolved
  * => to keep "always writable"

* Permanent Failure Handling
  * To handle the Replica synchronization problem with Permanent Failure ( such as replica lost ) :
    * Use __Merkle Tree__, which is a tree that keeps individual key hash value as leaves and keeps the hash value of children node in parent node
    * => to __efficiently__ detect inconsistency and minimize transferred data

* To achieve Node Failure detection
  * Use gossip-style protocol to perform decentralized failure detection and to propagate membership changes information
    * => to get eventual consistency.

### References
[**[1]**](http://www.allthingsdistributed.com/files/amazon-dynamo-sosp2007.pdf)Dynamo: Amazon's Highly Available Key-value Store


text:
@implementation MGDataCell

- (instancetype)initWithTitle:(NSString *)title
{
    if (self = [super init]) {
        _isExpanded = NO;
        _title = [title copy];
    }

    return self;
}

@end


text:
this.hamvturkey = this.hamvturkey || {};
(function(){

	function Game(skipSong){
		this.init(skipSong);
	}
	Game.prototype = {
			//
			stage:null,
			bg:null,
			turkey:null,
			goal:null,
			ham:null,
			loader:null,
			sound:null,
			crosshairs:null,
			scoreboard:null,
			container:null,
			score:0,
			shots:10,
			saves:0,
			skipSong:false,
			isTouch:false,
			shotLanded:true,
			imgPath:'assets/img/',
			audioPath:'assets/sound/',
			init:function(isTouch,skipSong){
				//
				if(!skipSong){
					this.skipSong = isTouch;
				}
				this.isTouch = isTouch;
				this.loadPreloaderAssets();
			},
			onShot:function(event){
				console.log(event);
				if(this.shots<=0) return;
				if(!this.shotLanded) return;
				this.shotLanded = false;

				var stageX = event.stageX;
				var stageY = event.stageY;
				if(this.crosshairs){
					stageX = this.crosshairs.x;
					stageY = this.crosshairs.y;
				}
				var p = this.container.addChildAt(new hamvturkey.Puck(this.loader.getResult('puck').src),this.container.getChildIndex(this.turkey));
				p.x = 300;
				p.y = 345;
				p.on('hit',createjs.proxy(this.onPuckContact,this));
				//to be called after shot animation
				this.turkey.shoot(
					function(){
						p.fire(stageX,stageY);
					}
				);
				//console.log(this.loader.getResult("sfx_shoot"));
				this.sound.playSFX(hamvturkey.SoundManager.SHOOT);
			},
			onPuckContact: function (event){
				//console.log('shoot!');
				var puck = event.currentTarget;
				var pt = this.goal.globalToLocal(puck.x,puck.y);
				var arr = this.goal.getObjectsUnderPoint(pt.x,pt.y);
				//console.log('puckpoint:',pt);
				//console.log('hamx:',this.ham.x);
				//if(this.ham.hitTest(pt.x,pt.y)){
				var h = this.ham.getBounds();
				if((pt.x>h.x) && (pt.x < h.x+h.width) && (pt.y > h.y) && (pt.y<h.y+h.height) ){

					//alert('save by the ham');
					this.ham.save();
					this.sound.impact();
					puck.drop(270);
					this.sound.announcer.save();
					this.saves++;
					this.scoreboard.saves.transition(this.saves);
					this.scoreboard.messages.announceSave();

				}else {
					switch(arr[0]){
						case this.goal.right_crossbar:
							//alert('wide right');
							puck.deflectRight();
							this.sound.boing();
							this.scoreboard.messages.showMessage(["Wide right","wide",'right'],2000);

						break;
						case this.goal.left_crossbar:
							//alert('wide left');
							this.sound.boing();
							puck.deflectLeft();
							this.scoreboard.messages.showMessage(["Wide left!",'wide left!',"you",'just','missed...'],3000);

						break;
						case this.goal.top_crossbar:
							//alert('off the crossbar');
							this.sound.boing();
							puck.deflectUp();
							this.scoreboard.messages.showMessage(["Awwww....","too high",'aim lower!'],2000);

						break;
						default:
							//alert('goal!!!!');
							this.goal.addChildAt(puck,this.goal.getChildIndex(this.ham)-1);
							puck.x -=this.goal.x;
							puck.y-=this.goal.y;
							puck.drop(50);
							this.score++;
							this.scoreboard.goals.transition(this.score);
							this.sound.announcer.scores();
							this.scoreboard.messages.showMessage(["Turkey","Scores","!!!!","Yeah Buddy!!!"],3000);
							if(this.score>3)this.ham.speedUp();
						break;
					}
				}
				this.shotLanded = true;
					this.shots--;
					this.scoreboard.shots.transition(this.shots);
				if(this.shots<=0){
					this.gameOver();
				}

			},
			gameOver:function(){
				var messages = ['Game over','game','over'];
				switch(this.score){
					case 10:
						messages = ['10 out of 10!','perfect game!','congrats!!!'];
					break;
					case 5:
						messages = ['5 out of 10?',"It's a draw...",'Try the fish?','Try again.'];
					break;
					case 0:
						messages.push('you missed...');
						messages.push('every shot...')
						messages.push('...');
						messages.push(':(');
						messages.push(':( :( :( :(');
						messages.push(':(');
						messages.push('Are you...');
						messages.push('A vegetarian?');
						messages.push(':(');

					break;
					default:
						if(this.score>5){
							messages.push('Turkey Wins!!!');
							messages.push('Turkey!');
							messages.push('Wins!');
							messages.push(this.score+' out of 10...');
							messages.push('not bad...');
							messages.push('but can you...')
							messages.push('do better?');
							messages.push('try again.');
						}else {
							messages.push(this.score+' out of 10?')
							if(this.saves>5){
								messages.push(this.saves+' saves...')
								messages.push("that's...");
								messages.push('one');
								messages.push('tough');
								messages.push('ham!');
							}

							messages.push('try again.');
							this.ham.dance();
						}
					break;
				}
				this.scoreboard.shots.transition('0');
				if(this.crosshairs) createjs.Tween.get(this.crosshairs).to({alpha:0},500);
				this.ham.stopMoving();
				this.scoreboard.messages.showMessage(messages,messages.length*1500,createjs.proxy(this.resetGame,this));
				this.sound.playSFX(hamvturkey.SoundManager.GAMEOVER);
			},
			resetGame:function(){
				this.shots = 10;
				this.score = 0;
				this.saves = 0;
				var messages = ["Fire","when",'ready...'];
				this.scoreboard.messages.showMessage(messages,messages.length*1000);
				this.scoreboard.shots.transition(this.shots,200,0);
				this.scoreboard.saves.transition(this.saves,200,0);
				this.scoreboard.goals.transition(this.score,200,0);
				if(this.crosshairs) createjs.Tween.get(this.crosshairs).to({alpha:1},500);
				this.ham.startMoving();
				var i;
				for(i=0;i<this.container.children.length;i++){
					if(this.container.children[i] instanceof hamvturkey.Puck){
						this.container.children[i].disappear();
					}
				}
				this.goal.clearPucks();
			},
			loadPreloaderAssets:function(){
				var imgPath = 'assets/img/';
				manifest= [
					{src:imgPath+"sprite_preloader_bg.jpg",id:hamvturkey.Preloader.BG},
					{src:imgPath+"sprite_thermometer_bg.png",id:hamvturkey.Preloader.THERMOMETER},
					{src:imgPath+"sprite_thermometer_arrow.png",id:hamvturkey.Preloader.ARROW}
				];

				this.loader = new createjs.LoadQueue(false);
				this.loader.addEventListener("complete", createjs.proxy(this.onPreloaderAssetsLoaded,this));
				this.loader.loadManifest(manifest);
			},
			onPreloaderAssetsLoaded:function(event){
				this.initStage();
				this.preloader = this.stage.addChild(new hamvturkey.Preloader(this.loader));
				this.preloader.intro(createjs.proxy(this.loadAssets,this));

			//	this.loadAssets();
			},
			loadAssets:function(){
				//view-source:http://localhost/EaselJs/examples/SpriteSheet.html
				//compress pngs: https://tinypng.com/
				//modify sounds: audacity
				manifest = [
					{src:this.imgPath+"sprite_bg.jpg",id:"bg"},
					{src:this.imgPath+"sprite_turkey.png",id:"turkey"},
					{src:this.imgPath+"sprite_ham.png",id:"ham"},
					{src:this.imgPath+"sprite_digits_trans.png",id:"digits"},
					{src:this.imgPath+"sprite_scoreboard.png",id:"scoreboard"},
					{src:this.imgPath+"sprite_message_bg.png",id:"messageboard"},
					{src:this.imgPath+"crosshairs.png",id:"crosshairs"},
					{src:this.imgPath+"sprite_fruitcake.png",id:"puck"},
					{src:this.imgPath+"sprite_gameon_bg.jpg",id:"gameon"}
				];
				this.loader = new createjs.LoadQueue(false);
				this.sound = new hamvturkey.SoundManager(this.loader);
				if(createjs.Sound.initializeDefaultPlugins()){
					//add audio to manifest
					this.sound.soundEnabled = true;
					manifest.push(
						{id:hamvturkey.SoundManager.SCORES, src:this.audioPath+'scores.mp3|'+this.audioPath+'scores.ogg'},
						{id:hamvturkey.SoundManager.SAVE, src:this.audioPath+'save.mp3|'+this.audioPath+'save.ogg'},
						{id:hamvturkey.SoundManager.SHOOT, src:this.audioPath+'shoot.mp3|'+this.audioPath+'shoot.ogg'},
						{id:hamvturkey.SoundManager.BOING_1, src:this.audioPath+'boing.mp3|'+this.audioPath+'boing.ogg'},
						{id:hamvturkey.SoundManager.BOING_2, src:this.audioPath+'boing2.mp3|'+this.audioPath+'boing2.ogg'},
						{id:hamvturkey.SoundManager.BOING_3, src:this.audioPath+'boing3.mp3|'+this.audioPath+'boing3.ogg'},
						{id:hamvturkey.SoundManager.GAMEOVER, src:this.audioPath+'gameover.mp3|'+this.audioPath+'gameover.ogg'},
						{id:hamvturkey.SoundManager.IMPACT, src:this.audioPath+'impact.mp3|'+this.audioPath+'impact.ogg'}
					);
					if(!this.skipSong){
						manifest.push({id:hamvturkey.SoundManager.THEME, src:this.audioPath+'turkeyvsham.mp3|'+this.audioPath+'turkeyvsham.ogg'});
					}
					createjs.Sound.registerPlugin(createjs.HTMLAudioPlugin);  // need this so it doesn't default to Web Audio
					this.loader.installPlugin(createjs.Sound);
				}
				this.loader.addEventListener('progress', createjs.proxy(this.preloader.onLoadProgress,this.preloader));
				this.loader.addEventListener("complete", createjs.proxy(this.preloader.onLoadComplete,this.preloader));
				this.preloader.addEventListener("complete", createjs.proxy(this.onAssetsLoaded,this));

				this.loader.loadManifest(manifest);
			},
			onAssetsLoaded:function(event){
				//console.log(event);
				//TODO wrap this all in a "click/touch to play" event
				this.container = this.stage.addChildAt(new createjs.Container(),0);
				this.container.setBounds(0,0,550,367);
				this.bg = this.container.addChild(new createjs.Bitmap(this.loader.getResult("bg")));
				this.turkey = this.container.addChild(new hamvturkey.Turkey());
				this.turkey.y = 120;
				this.turkey.x = 290;
				this.turkey.scaleX = this.turkey.scaleY = .9;
				this.turkey.buildSprite(this.loader.getResult("turkey"));
				this.goal = this.container.addChild(new hamvturkey.Goal(83,60,250,200,7,.01));
				this.ham =  this.goal.addChild(new hamvturkey.Ham(85));
				this.ham.buildSprite(this.loader.getResult("ham"));
				this.scoreboard = this.container.addChild(new hamvturkey.Scoreboard(this.loader.getResult("scoreboard"),this.loader.getResult("digits"),this.loader.getResult("messageboard")));
				createjs.Tween.get(this).wait(500).call(createjs.proxy(this.circleWipe,this));
			},
			circleWipe:function(){
				var bounds = this.container.getBounds();
				this.stage.removeChild(this.preloader);
				this.gameon = this.stage.addChildAt(new createjs.Bitmap(this.loader.getResult('gameon')),0);
				this.circlemask = new createjs.Shape();
				this.circlemask.graphics.beginFill('red').drawCircle(0,0,bounds.width);
				this.circlemask.x = bounds.width*.5;
				this.circlemask.y = bounds.height*.5;
				this.circlemask.cache(bounds.x,bounds.y,bounds.width,bounds.height);
				this.container.cache(bounds.x,bounds.y,bounds.width,bounds.height);
				this.container.mask = this.circlemask;
				this.circlemask.scaleX = this.circlemask.scaleY = 0;
				createjs.Tween.get(this.circlemask).wait(1000).to({scaleX:1,scaleY:1},900).call(createjs.proxy(this.playSong,this));
			},
			playSong:function(){
				this.container.uncache();
				this.container.mask = null;
				this.stage.removeChild(this.gameon);
				if(this.sound.soundEnabled && !this.skipSong){
					this.turkey.startMoving();
					this.ham.dance();
					var game = this, tHam, tTurkey;
					var turkey_delay = 4500, count = 0;
					console.log(count);
					var lis = game.ham.on(hamvturkey.Ham.DANCE_COMPLETE,function(){
						count++;
						if(count==2){
							//
						}
						if(count<=4){
							game.turkey.crouch();
							createjs.Tween.get(game.turkey).wait(1500).call(createjs.proxy(game.turkey.startMoving, game.turkey));
							createjs.Tween.get(game.ham).wait(1500).call(createjs.proxy(game.ham.dance, game.ham));
						}else if(count == 5){
							game.ham.dance();
						}else {
							game.ham.off(lis);
							game.turkey.crouch();
						}
					});
					var song = this.sound.playSong(hamvturkey.SoundManager.THEME);
					song.addEventListener('complete',function(){
						game.startGame();
					});
				}else {
					this.startGame();
				}
			},
			startGame:function(){
				this.scoreboard.lower(createjs.proxy(function(){
					this.scoreboard.shots.transition(this.shots,200,0	);
					this.scoreboard.saves.transition(this.saves,200,500);
					this.scoreboard.goals.transition(this.score,200,500);

				},this));
				this.ham.startMoving();
				this.turkey.startMoving();
				this.container.on('click',createjs.proxy(this.onShot,this));
				this.ham.mousEnabled = false;
				this.stage.enableMouseOver(10);
				if(!this.isTouch){
					this.crosshairs = this.container.addChild(new createjs.Bitmap(this.loader.getResult('crosshairs')));
					this.crosshairs.regX = this.crosshairs.regY = 12.5;
				}
				var action = "Click";
				if(this.isTouch) action = "Tap";
				var messages = [];
				for(var i = 0;i<10;i++) messages.push(action+' goal','to shoot');
				this.scoreboard.messages.showMessage(messages,messages.length*2000);

			},
			initStage:function(){
				this.stage = new createjs.Stage("gameCanvas");
				createjs.Ticker.setFPS(60);
				createjs.Ticker.addEventListener("tick", createjs.proxy(this.tick,this));
			},
			tick:function(event){
				this.updateCursor();
				if(this.ham) this.ham.move();
				this.stage.update();
			},
			updateCursor:function(){
				if(!this.crosshairs) return;
				var goal = this.goal.getBounds();
				var mouseX = this.stage.mouseX;
				var mouseY = this.stage.mouseY;
				goal.width-=6;
				if(mouseX<goal.x){
					this.crosshairs.x = goal.x;
				}else if(mouseX>goal.x+goal.width){
					this.crosshairs.x = goal.x+goal.width;
				}else{
					this.crosshairs.x = mouseX;
				}
				if(mouseY<goal.y){
					this.crosshairs.y = goal.y;
				}else if(mouseY>goal.y+goal.height){
					this.crosshairs.y = goal.y+goal.height;
				}else{
					this.crosshairs.y = mouseY;
				}
				this.crosshairs.rotation++;
			}

	}
	hamvturkey.Game = Game;
}());