text:
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
source "${KUBE_ROOT}/cluster/gce/${KUBE_CONFIG_FILE-"config-default.sh"}"
source "${KUBE_ROOT}/cluster/common.sh"
source "${KUBE_ROOT}/cluster/lib/util.sh"

if [[ "${NODE_OS_DISTRIBUTION}" == "debian" || "${NODE_OS_DISTRIBUTION}" == "container-linux" || "${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then
  source "${KUBE_ROOT}/cluster/gce/${NODE_OS_DISTRIBUTION}/node-helper.sh"
else
  echo "Cannot operate on cluster using node os distro: ${NODE_OS_DISTRIBUTION}" >&2
  exit 1
fi

if [[ "${MASTER_OS_DISTRIBUTION}" == "container-linux" || "${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" ]]; then
  source "${KUBE_ROOT}/cluster/gce/${MASTER_OS_DISTRIBUTION}/master-helper.sh"
else
  echo "Cannot operate on cluster using master os distro: ${MASTER_OS_DISTRIBUTION}" >&2
  exit 1
fi

if [[ "${MASTER_OS_DISTRIBUTION}" == "gci" ]]; then
  # If the master image is not set, we use the latest GCI image.
  # Otherwise, we respect whatever is set by the user.
  MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-${GCI_VERSION}}
  MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-google-containers}
elif [[ "${MASTER_OS_DISTRIBUTION}" == "debian" ]]; then
  MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-${CVM_VERSION}}
  MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-google-containers}
fi

# Sets node image based on the specified os distro. Currently this function only
# supports gci and debian.
function set-node-image() {
  if [[ "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then
    # If the node image is not set, we use the latest GCI image.
    # Otherwise, we respect whatever is set by the user.
    NODE_IMAGE=${KUBE_GCE_NODE_IMAGE:-${GCI_VERSION}}
    NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-google-containers}
  elif [[ "${NODE_OS_DISTRIBUTION}" == "debian" ]]; then
    NODE_IMAGE=${KUBE_GCE_NODE_IMAGE:-${CVM_VERSION}}
    NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-google-containers}
  fi
}

set-node-image

# Verfiy cluster autoscaler configuration.
if [[ "${ENABLE_CLUSTER_AUTOSCALER}" == "true" ]]; then
  if [ -z $AUTOSCALER_MIN_NODES ]; then
    echo "AUTOSCALER_MIN_NODES not set."
    exit 1
  fi
  if [ -z $AUTOSCALER_MAX_NODES ]; then
    echo "AUTOSCALER_MAX_NODES not set."
    exit 1
  fi
fi

NODE_INSTANCE_PREFIX="${INSTANCE_PREFIX}-minion"
NODE_TAGS="${NODE_TAG}"

ALLOCATE_NODE_CIDRS=true

KUBE_PROMPT_FOR_UPDATE=${KUBE_PROMPT_FOR_UPDATE:-"n"}
# How long (in seconds) to wait for cluster initialization.
KUBE_CLUSTER_INITIALIZATION_TIMEOUT=${KUBE_CLUSTER_INITIALIZATION_TIMEOUT:-300}

function join_csv() {
  local IFS=','; echo "$*";
}

# This function returns the first string before the comma
function split_csv() {
  echo "$*" | cut -d',' -f1
}

# Verify prereqs
function verify-prereqs() {
  local cmd
  for cmd in gcloud gsutil; do
    if ! which "${cmd}" >/dev/null; then
      local resp="n"
      if [[ "${KUBE_PROMPT_FOR_UPDATE}" == "y" ]]; then
        echo "Can't find ${cmd} in PATH.  Do you wish to install the Google Cloud SDK? [Y/n]"
        read resp
      fi
      if [[ "${resp}" != "n" && "${resp}" != "N" ]]; then
        curl https://sdk.cloud.google.com | bash
      fi
      if ! which "${cmd}" >/dev/null; then
        echo "Can't find ${cmd} in PATH, please fix and retry. The Google Cloud " >&2
        echo "SDK can be downloaded from https://cloud.google.com/sdk/." >&2
        exit 1
      fi
    fi
  done
  update-or-verify-gcloud
}

# Use the gcloud defaults to find the project.  If it is already set in the
# environment then go with that.
#
# Vars set:
#   PROJECT
#   PROJECT_REPORTED
function detect-project() {
  if [[ -z "${PROJECT-}" ]]; then
    PROJECT=$(gcloud config list project --format 'value(core.project)')
  fi

  if [[ -z "${PROJECT-}" ]]; then
    echo "Could not detect Google Cloud Platform project.  Set the default project using " >&2
    echo "'gcloud config set project <PROJECT>'" >&2
    exit 1
  fi
  if [[ -z "${PROJECT_REPORTED-}" ]]; then
    echo "Project: ${PROJECT}" >&2
    echo "Zone: ${ZONE}" >&2
    PROJECT_REPORTED=true
  fi
}

# Copy a release tar and its accompanying hash.
function copy-to-staging() {
  local -r staging_path=$1
  local -r gs_url=$2
  local -r tar=$3
  local -r hash=$4

  echo "${hash}" > "${tar}.sha1"
  gsutil -m -q -h "Cache-Control:private, max-age=0" cp "${tar}" "${tar}.sha1" "${staging_path}"
  gsutil -m acl ch -g all:R "${gs_url}" "${gs_url}.sha1" >/dev/null 2>&1
  echo "+++ $(basename ${tar}) uploaded (sha1 = ${hash})"
}

# Given the cluster zone, return the list of regional GCS release
# bucket suffixes for the release in preference order. GCS doesn't
# give us an API for this, so we hardcode it.
#
# Assumed vars:
#   RELEASE_REGION_FALLBACK
#   REGIONAL_KUBE_ADDONS
#   ZONE
# Vars set:
#   PREFERRED_REGION
#   KUBE_ADDON_REGISTRY
function set-preferred-region() {
  case ${ZONE} in
    asia-*)
      PREFERRED_REGION=("asia" "us" "eu")
      ;;
    europe-*)
      PREFERRED_REGION=("eu" "us" "asia")
      ;;
    *)
      PREFERRED_REGION=("us" "eu" "asia")
      ;;
  esac
  local -r preferred="${PREFERRED_REGION[0]}"

  if [[ "${RELEASE_REGION_FALLBACK}" != "true" ]]; then
    PREFERRED_REGION=( "${preferred}" )
  fi

  # If we're using regional GCR, and we're outside the US, go to the
  # regional registry. The gcr.io/google_containers registry is
  # appropriate for US (for now).
  if [[ "${REGIONAL_KUBE_ADDONS}" == "true" ]] && [[ "${preferred}" != "us" ]]; then
    KUBE_ADDON_REGISTRY="${preferred}.gcr.io/google_containers"
  else
    KUBE_ADDON_REGISTRY="gcr.io/google_containers"
  fi

  if [[ "${ENABLE_DOCKER_REGISTRY_CACHE:-}" == "true" ]]; then
    DOCKER_REGISTRY_MIRROR_URL="https://${preferred}-mirror.gcr.io"
  fi
}

# Take the local tar files and upload them to Google Storage.  They will then be
# downloaded by the master as part of the start up script for the master.
#
# Assumed vars:
#   PROJECT
#   SERVER_BINARY_TAR
#   SALT_TAR
#   KUBE_MANIFESTS_TAR
#   ZONE
# Vars set:
#   SERVER_BINARY_TAR_URL
#   SERVER_BINARY_TAR_HASH
#   SALT_TAR_URL
#   SALT_TAR_HASH
#   KUBE_MANIFESTS_TAR_URL
#   KUBE_MANIFESTS_TAR_HASH
function upload-server-tars() {
  SERVER_BINARY_TAR_URL=
  SERVER_BINARY_TAR_HASH=
  SALT_TAR_URL=
  SALT_TAR_HASH=
  KUBE_MANIFESTS_TAR_URL=
  KUBE_MANIFESTS_TAR_HASH=

  local project_hash
  if which md5 > /dev/null 2>&1; then
    project_hash=$(md5 -q -s "$PROJECT")
  else
    project_hash=$(echo -n "$PROJECT" | md5sum | awk '{ print $1 }')
  fi

  # This requires 1 million projects before the probability of collision is 50%
  # that's probably good enough for now :P
  project_hash=${project_hash:0:10}

  set-preferred-region

  SERVER_BINARY_TAR_HASH=$(sha1sum-file "${SERVER_BINARY_TAR}")
  SALT_TAR_HASH=$(sha1sum-file "${SALT_TAR}")
  if [[ -n "${KUBE_MANIFESTS_TAR:-}" ]]; then
    KUBE_MANIFESTS_TAR_HASH=$(sha1sum-file "${KUBE_MANIFESTS_TAR}")
  fi

  local server_binary_tar_urls=()
  local salt_tar_urls=()
  local kube_manifest_tar_urls=()

  for region in "${PREFERRED_REGION[@]}"; do
    suffix="-${region}"
    if [[ "${suffix}" == "-us" ]]; then
      suffix=""
    fi
    local staging_bucket="gs://kubernetes-staging-${project_hash}${suffix}"

    # Ensure the buckets are created
    if ! gsutil ls "${staging_bucket}" >/dev/null; then
      echo "Creating ${staging_bucket}"
      gsutil mb -l "${region}" "${staging_bucket}"
    fi

    local staging_path="${staging_bucket}/${INSTANCE_PREFIX}-devel"

    echo "+++ Staging server tars to Google Storage: ${staging_path}"
    local server_binary_gs_url="${staging_path}/${SERVER_BINARY_TAR##*/}"
    local salt_gs_url="${staging_path}/${SALT_TAR##*/}"
    copy-to-staging "${staging_path}" "${server_binary_gs_url}" "${SERVER_BINARY_TAR}" "${SERVER_BINARY_TAR_HASH}"
    copy-to-staging "${staging_path}" "${salt_gs_url}" "${SALT_TAR}" "${SALT_TAR_HASH}"

    # Convert from gs:// URL to an https:// URL
    server_binary_tar_urls+=("${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}")
    salt_tar_urls+=("${salt_gs_url/gs:\/\//https://storage.googleapis.com/}")
    if [[ -n "${KUBE_MANIFESTS_TAR:-}" ]]; then
      local kube_manifests_gs_url="${staging_path}/${KUBE_MANIFESTS_TAR##*/}"
      copy-to-staging "${staging_path}" "${kube_manifests_gs_url}" "${KUBE_MANIFESTS_TAR}" "${KUBE_MANIFESTS_TAR_HASH}"
      # Convert from gs:// URL to an https:// URL
      kube_manifests_tar_urls+=("${kube_manifests_gs_url/gs:\/\//https://storage.googleapis.com/}")
    fi
  done

  SERVER_BINARY_TAR_URL=$(join_csv "${server_binary_tar_urls[@]}")
  SALT_TAR_URL=$(join_csv "${salt_tar_urls[@]}")
  if [[ -n "${KUBE_MANIFESTS_TAR:-}" ]]; then
    KUBE_MANIFESTS_TAR_URL=$(join_csv "${kube_manifests_tar_urls[@]}")
  fi
}

# Detect minions created in the minion group
#
# Assumed vars:
#   NODE_INSTANCE_PREFIX
# Vars set:
#   NODE_NAMES
#   INSTANCE_GROUPS
function detect-node-names() {
  detect-project
  INSTANCE_GROUPS=()
  INSTANCE_GROUPS+=($(gcloud compute instance-groups managed list \
    --zones "${ZONE}" --project "${PROJECT}" \
    --regexp "${NODE_INSTANCE_PREFIX}-.+" \
    --format='value(instanceGroup)' || true))
  NODE_NAMES=()
  if [[ -n "${INSTANCE_GROUPS[@]:-}" ]]; then
    for group in "${INSTANCE_GROUPS[@]}"; do
      NODE_NAMES+=($(gcloud compute instance-groups managed list-instances \
        "${group}" --zone "${ZONE}" --project "${PROJECT}" \
        --format='value(instance)'))
    done
  fi
  echo "INSTANCE_GROUPS=${INSTANCE_GROUPS[*]:-}" >&2
  echo "NODE_NAMES=${NODE_NAMES[*]:-}" >&2
}

# Detect the information about the minions
#
# Assumed vars:
#   ZONE
# Vars set:
#   NODE_NAMES
#   KUBE_NODE_IP_ADDRESSES (array)
function detect-nodes() {
  detect-project
  detect-node-names
  KUBE_NODE_IP_ADDRESSES=()
  for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
    local node_ip=$(gcloud compute instances describe --project "${PROJECT}" --zone "${ZONE}" \
      "${NODE_NAMES[$i]}" --format='value(networkInterfaces[0].accessConfigs[0].natIP)')
    if [[ -z "${node_ip-}" ]] ; then
      echo "Did not find ${NODE_NAMES[$i]}" >&2
    else
      echo "Found ${NODE_NAMES[$i]} at ${node_ip}"
      KUBE_NODE_IP_ADDRESSES+=("${node_ip}")
    fi
  done
  if [[ -z "${KUBE_NODE_IP_ADDRESSES-}" ]]; then
    echo "Could not detect Kubernetes minion nodes.  Make sure you've launched a cluster with 'kube-up.sh'" >&2
    exit 1
  fi
}

# Detect the IP for the master
#
# Assumed vars:
#   MASTER_NAME
#   ZONE
#   REGION
# Vars set:
#   KUBE_MASTER
#   KUBE_MASTER_IP
function detect-master() {
  detect-project
  KUBE_MASTER=${MASTER_NAME}
  echo "Trying to find master named '${MASTER_NAME}'" >&2
  if [[ -z "${KUBE_MASTER_IP-}" ]]; then
    local master_address_name="${MASTER_NAME}-ip"
    echo "Looking for address '${master_address_name}'" >&2
    KUBE_MASTER_IP=$(gcloud compute addresses describe "${master_address_name}" \
      --project "${PROJECT}" --region "${REGION}" -q --format='value(address)')
  fi
  if [[ -z "${KUBE_MASTER_IP-}" ]]; then
    echo "Could not detect Kubernetes master node.  Make sure you've launched a cluster with 'kube-up.sh'" >&2
    exit 1
  fi
  echo "Using master: $KUBE_MASTER (external IP: $KUBE_MASTER_IP)" >&2
}

# Reads kube-env metadata from master
#
# Assumed vars:
#   KUBE_MASTER
#   PROJECT
#   ZONE
function get-master-env() {
  # TODO(zmerlynn): Make this more reliable with retries.
  gcloud compute --project ${PROJECT} ssh --zone ${ZONE} ${KUBE_MASTER} --command \
    "curl --fail --silent -H 'Metadata-Flavor: Google' \
      'http://metadata/computeMetadata/v1/instance/attributes/kube-env'" 2>/dev/null
  gcloud compute --project ${PROJECT} ssh --zone ${ZONE} ${KUBE_MASTER} --command \
    "curl --fail --silent -H 'Metadata-Flavor: Google' \
      'http://metadata/computeMetadata/v1/instance/attributes/kube-master-certs'" 2>/dev/null
}

# Robustly try to create a static ip.
# $1: The name of the ip to create
# $2: The name of the region to create the ip in.
function create-static-ip() {
  detect-project
  local attempt=0
  local REGION="$2"
  while true; do
    if gcloud compute addresses create "$1" \
      --project "${PROJECT}" \
      --region "${REGION}" -q > /dev/null; then
      # successful operation - wait until it's visible
      start="$(date +%s)"
      while true; do
        now="$(date +%s)"
        # Timeout set to 15 minutes
        if [ $((now - start)) -gt 900 ]; then
          echo "Timeout while waiting for master IP visibility"
          exit 2
        fi
        if gcloud compute addresses describe "$1" --project "${PROJECT}" --region "${REGION}" >/dev/null 2>&1; then
          break
        fi
        echo "Master IP not visible yet. Waiting..."
        sleep 5
      done
      break
    fi

    if gcloud compute addresses describe "$1" \
      --project "${PROJECT}" \
      --region "${REGION}" >/dev/null 2>&1; then
      # it exists - postcondition satisfied
      break
    fi

    if (( attempt > 4 )); then
      echo -e "${color_red}Failed to create static ip $1 ${color_norm}" >&2
      exit 2
    fi
    attempt=$(($attempt+1))
    echo -e "${color_yellow}Attempt $attempt failed to create static ip $1. Retrying.${color_norm}" >&2
    sleep $(($attempt * 5))
  done
}

# Robustly try to create a firewall rule.
# $1: The name of firewall rule.
# $2: IP ranges.
# $3: Target tags for this firewall rule.
function create-firewall-rule() {
  detect-project
  local attempt=0
  while true; do
    if ! gcloud compute firewall-rules create "$1" \
      --project "${PROJECT}" \
      --network "${NETWORK}" \
      --source-ranges "$2" \
      --target-tags "$3" \
      --allow tcp,udp,icmp,esp,ah,sctp; then
      if (( attempt > 4 )); then
        echo -e "${color_red}Failed to create firewall rule $1 ${color_norm}" >&2
        exit 2
      fi
      echo -e "${color_yellow}Attempt $(($attempt+1)) failed to create firewall rule $1. Retrying.${color_norm}" >&2
      attempt=$(($attempt+1))
      sleep $(($attempt * 5))
    else
        break
    fi
  done
}

# $1: version (required)
function get-template-name-from-version() {
  # trim template name to pass gce name validation
  echo "${NODE_INSTANCE_PREFIX}-template-${1}" | cut -c 1-63 | sed 's/[\.\+]/-/g;s/-*$//g'
}

# Robustly try to create an instance template.
# $1: The name of the instance template.
# $2: The scopes flag.
# $3 and others: Metadata entries (must all be from a file).
function create-node-template() {
  detect-project
  local template_name="$1"

  # First, ensure the template doesn't exist.
  # TODO(zmerlynn): To make this really robust, we need to parse the output and
  #                 add retries. Just relying on a non-zero exit code doesn't
  #                 distinguish an ephemeral failed call from a "not-exists".
  if gcloud compute instance-templates describe "$template_name" --project "${PROJECT}" &>/dev/null; then
    echo "Instance template ${1} already exists; deleting." >&2
    if ! gcloud compute instance-templates delete "$template_name" --project "${PROJECT}" &>/dev/null; then
      echo -e "${color_yellow}Failed to delete existing instance template${color_norm}" >&2
      exit 2
    fi
  fi

  local attempt=1
  local preemptible_minions=""
  if [[ "${PREEMPTIBLE_NODE}" == "true" ]]; then
    preemptible_minions="--preemptible --maintenance-policy TERMINATE"
  fi
  local local_ssds=""
  if [ ! -z ${NODE_LOCAL_SSDS+x} ]; then
      for i in $(seq ${NODE_LOCAL_SSDS}); do
          local_ssds="$local_ssds--local-ssd=interface=SCSI "
      done
  fi
  while true; do
    echo "Attempt ${attempt} to create ${1}" >&2
    if ! gcloud compute instance-templates create "$template_name" \
      --project "${PROJECT}" \
      --machine-type "${NODE_SIZE}" \
      --boot-disk-type "${NODE_DISK_TYPE}" \
      --boot-disk-size "${NODE_DISK_SIZE}" \
      --image-project="${NODE_IMAGE_PROJECT}" \
      --image "${NODE_IMAGE}" \
      --tags "${NODE_TAG}" \
      --network "${NETWORK}" \
      ${local_ssds} \
      ${preemptible_minions} \
      $2 \
      --can-ip-forward \
      --metadata-from-file $(echo ${@:3} | tr ' ' ',') >&2; then
        if (( attempt > 5 )); then
          echo -e "${color_red}Failed to create instance template $template_name ${color_norm}" >&2
          exit 2
        fi
        echo -e "${color_yellow}Attempt ${attempt} failed to create instance template $template_name. Retrying.${color_norm}" >&2
        attempt=$(($attempt+1))
        sleep $(($attempt * 5))

        # In case the previous attempt failed with something like a
        # Backend Error and left the entry laying around, delete it
        # before we try again.
        gcloud compute instance-templates delete "$template_name" --project "${PROJECT}" &>/dev/null || true
    else
        break
    fi
  done
}

# Robustly try to add metadata on an instance.
# $1: The name of the instance.
# $2...$n: The metadata key=value pairs to add.
function add-instance-metadata() {
  local -r instance=$1
  shift 1
  local -r kvs=( "$@" )
  detect-project
  local attempt=0
  while true; do
    if ! gcloud compute instances add-metadata "${instance}" \
      --project "${PROJECT}" \
      --zone "${ZONE}" \
      --metadata "${kvs[@]}"; then
        if (( attempt > 5 )); then
          echo -e "${color_red}Failed to add instance metadata in ${instance} ${color_norm}" >&2
          exit 2
        fi
        echo -e "${color_yellow}Attempt $(($attempt+1)) failed to add metadata in ${instance}. Retrying.${color_norm}" >&2
        attempt=$(($attempt+1))
        sleep $((5 * $attempt))
    else
        break
    fi
  done
}

# Robustly try to add metadata on an instance, from a file.
# $1: The name of the instance.
# $2...$n: The metadata key=file pairs to add.
function add-instance-metadata-from-file() {
  local -r instance=$1
  shift 1
  local -r kvs=( "$@" )
  detect-project
  local attempt=0
  while true; do
    echo "${kvs[@]}"
    if ! gcloud compute instances add-metadata "${instance}" \
      --project "${PROJECT}" \
      --zone "${ZONE}" \
      --metadata-from-file "$(join_csv ${kvs[@]})"; then
        if (( attempt > 5 )); then
          echo -e "${color_red}Failed to add instance metadata in ${instance} ${color_norm}" >&2
          exit 2
        fi
        echo -e "${color_yellow}Attempt $(($attempt+1)) failed to add metadata in ${instance}. Retrying.${color_norm}" >&2
        attempt=$(($attempt+1))
        sleep $(($attempt * 5))
    else
        break
    fi
  done
}

# Instantiate a kubernetes cluster
#
# Assumed vars
#   KUBE_ROOT
#   <Various vars set in config file>
function kube-up() {
  ensure-temp-dir
  detect-project

  load-or-gen-kube-basicauth
  load-or-gen-kube-bearertoken

  # Make sure we have the tar files staged on Google Storage
  find-release-tars
  upload-server-tars

  # ensure that environmental variables specifying number of migs to create
  set_num_migs

  if [[ ${KUBE_USE_EXISTING_MASTER:-} == "true" ]]; then
    detect-master
    parse-master-env
    create-nodes
  elif [[ ${KUBE_REPLICATE_EXISTING_MASTER:-} == "true" ]]; then
    if  [[ "${MASTER_OS_DISTRIBUTION}" != "gci" && "${MASTER_OS_DISTRIBUTION}" != "debian" ]]; then
      echo "Master replication supported only for gci and debian"
      return 1
    fi
    create-loadbalancer
    # If replication of master fails, we need to ensure that the replica is removed from etcd clusters.
    if ! replicate-master; then
      remove-replica-from-etcd 2379 || true
      remove-replica-from-etcd 4002 || true
    fi
  else
    check-existing
    create-network
    write-cluster-name
    create-autoscaler-config
    create-master
    create-nodes-firewall
    create-nodes-template
    create-nodes
    check-cluster
  fi
}

function check-existing() {
  local running_in_terminal=false
  # May be false if tty is not allocated (for example with ssh -T).
  if [ -t 1 ]; then
    running_in_terminal=true
  fi

  if [[ ${running_in_terminal} == "true" || ${KUBE_UP_AUTOMATIC_CLEANUP} == "true" ]]; then
    if ! check-resources; then
      local run_kube_down="n"
      echo "${KUBE_RESOURCE_FOUND} found." >&2
      # Get user input only if running in terminal.
      if [[ ${running_in_terminal} == "true" && ${KUBE_UP_AUTOMATIC_CLEANUP} == "false" ]]; then
        read -p "Would you like to shut down the old cluster (call kube-down)? [y/N] " run_kube_down
      fi
      if [[ ${run_kube_down} == "y" || ${run_kube_down} == "Y" || ${KUBE_UP_AUTOMATIC_CLEANUP} == "true" ]]; then
        echo "... calling kube-down" >&2
        kube-down
      fi
    fi
  fi
}

function create-network() {
  if ! gcloud compute networks --project "${PROJECT}" describe "${NETWORK}" &>/dev/null; then
    echo "Creating new network: ${NETWORK}"
    # The network needs to be created synchronously or we have a race. The
    # firewalls can be added concurrent with instance creation.
    gcloud compute networks create --project "${PROJECT}" "${NETWORK}" --range "10.240.0.0/16"
  fi

  if ! gcloud compute firewall-rules --project "${PROJECT}" describe "${CLUSTER_NAME}-default-internal-master" &>/dev/null; then
    gcloud compute firewall-rules create "${CLUSTER_NAME}-default-internal-master" \
      --project "${PROJECT}" \
      --network "${NETWORK}" \
      --source-ranges "10.0.0.0/8" \
      --allow "tcp:1-2379,tcp:2382-65535,udp:1-65535,icmp" \
      --target-tags "${MASTER_TAG}"&
  fi

  if ! gcloud compute firewall-rules --project "${PROJECT}" describe "${CLUSTER_NAME}-default-internal-node" &>/dev/null; then
    gcloud compute firewall-rules create "${CLUSTER_NAME}-default-internal-node" \
      --project "${PROJECT}" \
      --network "${NETWORK}" \
      --source-ranges "10.0.0.0/8" \
      --allow "tcp:1-65535,udp:1-65535,icmp" \
      --target-tags "${NODE_TAG}"&
  fi

  if ! gcloud compute firewall-rules describe --project "${PROJECT}" "${NETWORK}-default-ssh" &>/dev/null; then
    gcloud compute firewall-rules create "${NETWORK}-default-ssh" \
      --project "${PROJECT}" \
      --network "${NETWORK}" \
      --source-ranges "0.0.0.0/0" \
      --allow "tcp:22" &
  fi
}

function delete-firewall-rules() {
  for fw in $@; do
    if [[ -n $(gcloud compute firewall-rules --project "${PROJECT}" describe "${fw}" --format='value(name)' 2>/dev/null || true) ]]; then
      gcloud compute firewall-rules delete --project "${PROJECT}" --quiet "${fw}" &
    fi
  done
  kube::util::wait-for-jobs || {
    echo -e "${color_red}Failed to delete firewall rules.${color_norm}" >&2
  }
}

function delete-network() {
  if [[ -n $(gcloud compute networks --project "${PROJECT}" describe "${NETWORK}" --format='value(name)' 2>/dev/null || true) ]]; then
    if ! gcloud compute networks delete --project "${PROJECT}" --quiet "${NETWORK}"; then
      echo "Failed to delete network '${NETWORK}'. Listing firewall-rules:"
      gcloud compute firewall-rules --project "${PROJECT}" list --filter="network=${NETWORK}"
      return 1
    fi
  fi
}

# Assumes:
#   NUM_NODES
# Sets:
#   MASTER_ROOT_DISK_SIZE
function get-master-root-disk-size() {
  if [[ "${NUM_NODES}" -le "1000" ]]; then
    export MASTER_ROOT_DISK_SIZE="20"
  else
    export MASTER_ROOT_DISK_SIZE="50"
  fi
}

# Assumes:
#   NUM_NODES
# Sets:
#   MASTER_DISK_SIZE
function get-master-disk-size() {
  if [[ "${NUM_NODES}" -le "1000" ]]; then
    export MASTER_DISK_SIZE="20GB"
  else
    export MASTER_DISK_SIZE="100GB"
  fi
}

# Generates SSL certificates for etcd cluster. Uses cfssl program.
#
# Assumed vars:
#   KUBE_TEMP: temporary directory
#
# Args:
#  $1: host name
#  $2: CA certificate
#  $3: CA key
#
# If CA cert/key is empty, the function will also generate certs for CA.
#
# Vars set:
#   ETCD_CA_KEY_BASE64
#   ETCD_CA_CERT_BASE64
#   ETCD_PEER_KEY_BASE64
#   ETCD_PEER_CERT_BASE64
#
function create-etcd-certs {
  local host=${1}
  local ca_cert=${2:-}
  local ca_key=${3:-}

  GEN_ETCD_CA_CERT="${ca_cert}" GEN_ETCD_CA_KEY="${ca_key}" \
    generate-etcd-cert "${KUBE_TEMP}/cfssl" "${host}" "peer" "peer"

  pushd "${KUBE_TEMP}/cfssl"
  ETCD_CA_KEY_BASE64=$(cat "ca-key.pem" | base64 | tr -d '\r\n')
  ETCD_CA_CERT_BASE64=$(cat "ca.pem" | gzip | base64 | tr -d '\r\n')
  ETCD_PEER_KEY_BASE64=$(cat "peer-key.pem" | base64 | tr -d '\r\n')
  ETCD_PEER_CERT_BASE64=$(cat "peer.pem" | gzip | base64 | tr -d '\r\n')
  popd
}

function create-master() {
  echo "Starting master and configuring firewalls"
  gcloud compute firewall-rules create "${MASTER_NAME}-https" \
    --project "${PROJECT}" \
    --network "${NETWORK}" \
    --target-tags "${MASTER_TAG}" \
    --allow tcp:443 &

  # We have to make sure the disk is created before creating the master VM, so
  # run this in the foreground.
  get-master-disk-size
  gcloud compute disks create "${MASTER_NAME}-pd" \
    --project "${PROJECT}" \
    --zone "${ZONE}" \
    --type "${MASTER_DISK_TYPE}" \
    --size "${MASTER_DISK_SIZE}"

  # Create disk for cluster registry if enabled
  if [[ "${ENABLE_CLUSTER_REGISTRY}" == true && -n "${CLUSTER_REGISTRY_DISK}" ]]; then
    gcloud compute disks create "${CLUSTER_REGISTRY_DISK}" \
      --project "${PROJECT}" \
      --zone "${ZONE}" \
      --type "${CLUSTER_REGISTRY_DISK_TYPE_GCE}" \
      --size "${CLUSTER_REGISTRY_DISK_SIZE}" &
  fi

  # Create rule for accessing and securing etcd servers.
  if ! gcloud compute firewall-rules --project "${PROJECT}" describe "${MASTER_NAME}-etcd" &>/dev/null; then
    gcloud compute firewall-rules create "${MASTER_NAME}-etcd" \
      --project "${PROJECT}" \
      --network "${NETWORK}" \
      --source-tags "${MASTER_TAG}" \
      --allow "tcp:2380,tcp:2381" \
      --target-tags "${MASTER_TAG}" &
  fi

  # Generate a bearer token for this cluster. We push this separately
  # from the other cluster variables so that the client (this
  # computer) can forget it later. This should disappear with
  # http://issue.k8s.io/3168
  KUBELET_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
  KUBE_PROXY_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
  if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
    NODE_PROBLEM_DETECTOR_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
  fi

  # Reserve the master's IP so that it can later be transferred to another VM
  # without disrupting the kubelets.
  create-static-ip "${MASTER_NAME}-ip" "${REGION}"
  MASTER_RESERVED_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \
    --project "${PROJECT}" --region "${REGION}" -q --format='value(address)')

  if [[ "${REGISTER_MASTER_KUBELET:-}" == "true" ]]; then
    KUBELET_APISERVER="${MASTER_RESERVED_IP}"
  fi

  KUBERNETES_MASTER_NAME="${MASTER_RESERVED_IP}"
  MASTER_ADVERTISE_ADDRESS="${MASTER_RESERVED_IP}"

  create-certs "${MASTER_RESERVED_IP}"
  create-etcd-certs ${MASTER_NAME}

  # Sets MASTER_ROOT_DISK_SIZE that is used by create-master-instance
  get-master-root-disk-size

  create-master-instance "${MASTER_RESERVED_IP}" &
}

# Adds master replica to etcd cluster.
#
# Assumed vars:
#   REPLICA_NAME
#   PROJECT
#   EXISTING_MASTER_NAME
#   EXISTING_MASTER_ZONE
#
# $1: etcd client port
# $2: etcd internal port
# returns the result of ssh command which adds replica
function add-replica-to-etcd() {
  local -r client_port="${1}"
  local -r internal_port="${2}"
  gcloud compute ssh "${EXISTING_MASTER_NAME}" \
    --project "${PROJECT}" \
    --zone "${EXISTING_MASTER_ZONE}" \
    --command \
      "curl localhost:${client_port}/v2/members -XPOST -H \"Content-Type: application/json\" -d '{\"peerURLs\":[\"https://${REPLICA_NAME}:${internal_port}\"]}' -s"
  return $?
}

# Sets EXISTING_MASTER_NAME and EXISTING_MASTER_ZONE variables.
#
# Assumed vars:
#   PROJECT
#
# NOTE: Must be in sync with get-replica-name-regexp
function set-existing-master() {
  local existing_master=$(gcloud compute instances list \
    --project "${PROJECT}" \
    --regexp "$(get-replica-name-regexp)" \
    --format "value(name,zone)" | head -n1)
  EXISTING_MASTER_NAME="$(echo "${existing_master}" | cut -f1)"
  EXISTING_MASTER_ZONE="$(echo "${existing_master}" | cut -f2)"
}

function replicate-master() {
  set-replica-name
  set-existing-master

  echo "Experimental: replicating existing master ${EXISTING_MASTER_ZONE}/${EXISTING_MASTER_NAME} as ${ZONE}/${REPLICA_NAME}"

  # Before we do anything else, we should configure etcd to expect more replicas.
  if ! add-replica-to-etcd 2379 2380; then
    echo "Failed to add master replica to etcd cluster."
    return 1
  fi
  if ! add-replica-to-etcd 4002 2381; then
    echo "Failed to add master replica to etcd events cluster."
    return 1
  fi

  # We have to make sure the disk is created before creating the master VM, so
  # run this in the foreground.
  get-master-disk-size
  gcloud compute disks create "${REPLICA_NAME}-pd" \
    --project "${PROJECT}" \
    --zone "${ZONE}" \
    --type "${MASTER_DISK_TYPE}" \
    --size "${MASTER_DISK_SIZE}"

  # Sets MASTER_ROOT_DISK_SIZE that is used by create-master-instance
  get-master-root-disk-size

  local existing_master_replicas="$(get-all-replica-names)"
  replicate-master-instance "${EXISTING_MASTER_ZONE}" "${EXISTING_MASTER_NAME}" "${existing_master_replicas}"

  # Add new replica to the load balancer.
  gcloud compute target-pools add-instances "${MASTER_NAME}" \
    --project "${PROJECT}" \
    --zone "${ZONE}" \
    --instances "${REPLICA_NAME}"
}

# Detaches old and ataches new external IP to a VM.
#
# Arguments:
#   $1 - VM name
#   $2 - VM zone
#   $3 - external static IP; if empty will use an ephemeral IP address.
function attach-external-ip() {
  local NAME=${1}
  local ZONE=${2}
  local IP_ADDR=${3:-}
  local ACCESS_CONFIG_NAME=$(gcloud compute instances describe "${NAME}" \
    --project "${PROJECT}" --zone "${ZONE}" \
    --format="value(networkInterfaces[0].accessConfigs[0].name)")
  gcloud compute instances delete-access-config "${NAME}" \
    --project "${PROJECT}" --zone "${ZONE}" \
    --access-config-name "${ACCESS_CONFIG_NAME}"
  if [[ -z ${IP_ADDR} ]]; then
    gcloud compute instances add-access-config "${NAME}" \
      --project "${PROJECT}" --zone "${ZONE}" \
      --access-config-name "${ACCESS_CONFIG_NAME}"
  else
    gcloud compute instances add-access-config "${NAME}" \
      --project "${PROJECT}" --zone "${ZONE}" \
      --access-config-name "${ACCESS_CONFIG_NAME}" \
      --address "${IP_ADDR}"
  fi
}

# Creates load balancer in front of apiserver if it doesn't exists already. Assumes there's only one
# existing master replica.
#
# Assumes:
#   PROJECT
#   MASTER_NAME
#   ZONE
#   REGION
function create-loadbalancer() {
  detect-master

  # Step 0: Return early if LB is already configured.
  if gcloud compute forwarding-rules describe ${MASTER_NAME} \
    --project "${PROJECT}" --region ${REGION} > /dev/null 2>&1; then
    echo "Load balancer already exists"
    return
  fi

  local EXISTING_MASTER_NAME="$(get-all-replica-names)"
  local EXISTING_MASTER_ZONE=$(gcloud compute instances list "${EXISTING_MASTER_NAME}" \
    --project "${PROJECT}" --format="value(zone)")

  echo "Creating load balancer in front of an already existing master in ${EXISTING_MASTER_ZONE}"

  # Step 1: Detach master IP address and attach ephemeral address to the existing master
  attach-external-ip "${EXISTING_MASTER_NAME}" "${EXISTING_MASTER_ZONE}"

  # Step 2: Create target pool.
  gcloud compute target-pools create "${MASTER_NAME}" --project "${PROJECT}" --region "${REGION}"
  # TODO: We should also add master instances with suffixes
  gcloud compute target-pools add-instances "${MASTER_NAME}" --instances "${EXISTING_MASTER_NAME}" --project "${PROJECT}" --zone "${EXISTING_MASTER_ZONE}"

  # Step 3: Create forwarding rule.
  # TODO: This step can take up to 20 min. We need to speed this up...
  gcloud compute forwarding-rules create ${MASTER_NAME} \
    --project "${PROJECT}" --region ${REGION} \
    --target-pool ${MASTER_NAME} --address=${KUBE_MASTER_IP} --ports=443

  echo -n "Waiting for the load balancer configuration to propagate..."
  local counter=0
  until $(curl -k -m1 https://${KUBE_MASTER_IP} &> /dev/null); do
    counter=$((counter+1))
    echo -n .
    if [[ ${counter} -ge 1800 ]]; then
      echo -e "${color_red}TIMEOUT${color_norm}" >&2
      echo -e "${color_red}Load balancer failed to initialize within ${counter} seconds.${color_norm}" >&2
      exit 2
    fi
  done
  echo "DONE"
}

function create-nodes-firewall() {
  # Create a single firewall rule for all minions.
  create-firewall-rule "${NODE_TAG}-all" "${CLUSTER_IP_RANGE}" "${NODE_TAG}" &

  # Report logging choice (if any).
  if [[ "${ENABLE_NODE_LOGGING-}" == "true" ]]; then
    echo "+++ Logging using Fluentd to ${LOGGING_DESTINATION:-unknown}"
  fi

  # Wait for last batch of jobs
  kube::util::wait-for-jobs || {
    echo -e "${color_red}Some commands failed.${color_norm}" >&2
  }
}

function create-nodes-template() {
  echo "Creating minions."

  # TODO(zmerlynn): Refactor setting scope flags.
  local scope_flags=
  if [ -n "${NODE_SCOPES}" ]; then
    scope_flags="--scopes ${NODE_SCOPES}"
  else
    scope_flags="--no-scopes"
  fi

  write-node-env

  local template_name="${NODE_INSTANCE_PREFIX}-template"

  create-node-instance-template $template_name
}

# Assumes:
# - MAX_INSTANCES_PER_MIG
# - NUM_NODES
# exports:
# - NUM_MIGS
function set_num_migs() {
  local defaulted_max_instances_per_mig=${MAX_INSTANCES_PER_MIG:-1000}

  if [[ ${defaulted_max_instances_per_mig} -le "0" ]]; then
    echo "MAX_INSTANCES_PER_MIG cannot be negative. Assuming default 1000"
    defaulted_max_instances_per_mig=1000
  fi
  export NUM_MIGS=$(((${NUM_NODES} + ${defaulted_max_instances_per_mig} - 1) / ${defaulted_max_instances_per_mig}))
}

# Assumes:
# - NUM_MIGS
# - NODE_INSTANCE_PREFIX
# - NUM_NODES
# - PROJECT
# - ZONE
function create-nodes() {
  local template_name="${NODE_INSTANCE_PREFIX}-template"

  local instances_left=${NUM_NODES}

  #TODO: parallelize this loop to speed up the process
  for ((i=1; i<=${NUM_MIGS}; i++)); do
    local group_name="${NODE_INSTANCE_PREFIX}-group-$i"
    if [[ $i == ${NUM_MIGS} ]]; then
      # TODO: We don't add a suffix for the last group to keep backward compatibility when there's only one MIG.
      # We should change it at some point, but note #18545 when changing this.
      group_name="${NODE_INSTANCE_PREFIX}-group"
    fi
    # Spread the remaining number of nodes evenly
    this_mig_size=$((${instances_left} / (${NUM_MIGS}-${i}+1)))
    instances_left=$((instances_left-${this_mig_size}))

    gcloud compute instance-groups managed \
        create "${group_name}" \
        --project "${PROJECT}" \
        --zone "${ZONE}" \
        --base-instance-name "${group_name}" \
        --size "${this_mig_size}" \
        --template "$template_name" || true;
    gcloud compute instance-groups managed wait-until-stable \
        "${group_name}" \
        --zone "${ZONE}" \
        --project "${PROJECT}" || true;
  done
}

# Assumes:
# - NUM_MIGS
# - NODE_INSTANCE_PREFIX
# - PROJECT
# - ZONE
# - AUTOSCALER_MAX_NODES
# - AUTOSCALER_MIN_NODES
# Exports
# - AUTOSCALER_MIG_CONFIG
function create-cluster-autoscaler-mig-config() {

  # Each MIG must have at least one node, so the min number of nodes
  # must be greater or equal to the number of migs.
  if [[ ${AUTOSCALER_MIN_NODES} < ${NUM_MIGS} ]]; then
    echo "AUTOSCALER_MIN_NODES must be greater or equal ${NUM_MIGS}"
    exit 2
  fi

  # Each MIG must have at least one node, so the min number of nodes
  # must be greater or equal to the number of migs.
  if [[ ${AUTOSCALER_MAX_NODES} < ${NUM_MIGS} ]]; then
    echo "AUTOSCALER_MAX_NODES must be greater or equal ${NUM_MIGS}"
    exit 2
  fi

  # The code assumes that the migs were created with create-nodes
  # function which tries to evenly spread nodes across the migs.
  AUTOSCALER_MIG_CONFIG=""

  local left_min=${AUTOSCALER_MIN_NODES}
  local left_max=${AUTOSCALER_MAX_NODES}

  for ((i=1; i<=${NUM_MIGS}; i++)); do
    local group_name="${NODE_INSTANCE_PREFIX}-group-$i"
    if [[ $i == ${NUM_MIGS} ]]; then
      # TODO: We don't add a suffix for the last group to keep backward compatibility when there's only one MIG.
      # We should change it at some point, but note #18545 when changing this.
      group_name="${NODE_INSTANCE_PREFIX}-group"
    fi

    this_mig_min=$((${left_min}/(${NUM_MIGS}-${i}+1)))
    this_mig_max=$((${left_max}/(${NUM_MIGS}-${i}+1)))
    left_min=$((left_min-$this_mig_min))
    left_max=$((left_max-$this_mig_max))

    local mig_url="https://www.googleapis.com/compute/v1/projects/${PROJECT}/zones/${ZONE}/instanceGroups/${group_name}"
    AUTOSCALER_MIG_CONFIG="${AUTOSCALER_MIG_CONFIG} --nodes=${this_mig_min}:${this_mig_max}:${mig_url}"
  done

  AUTOSCALER_MIG_CONFIG="${AUTOSCALER_MIG_CONFIG} --scale-down-enabled=${AUTOSCALER_ENABLE_SCALE_DOWN}"
}

# Assumes:
# - NUM_MIGS
# - NODE_INSTANCE_PREFIX
# - PROJECT
# - ZONE
# - ENABLE_CLUSTER_AUTOSCALER
# - AUTOSCALER_MAX_NODES
# - AUTOSCALER_MIN_NODES
function create-autoscaler-config() {
  # Create autoscaler for nodes configuration if requested
  if [[ "${ENABLE_CLUSTER_AUTOSCALER}" == "true" ]]; then
    create-cluster-autoscaler-mig-config
    echo "Using autoscaler config: ${AUTOSCALER_MIG_CONFIG}"
  fi
}

function check-cluster() {
  detect-node-names
  detect-master

  echo "Waiting up to ${KUBE_CLUSTER_INITIALIZATION_TIMEOUT} seconds for cluster initialization."
  echo
  echo "  This will continually check to see if the API for kubernetes is reachable."
  echo "  This may time out if there was some uncaught error during start up."
  echo

  # curl in mavericks is borked.
  secure=""
  if which sw_vers >& /dev/null; then
    if [[ $(sw_vers | grep ProductVersion | awk '{print $2}') = "10.9."* ]]; then
      secure="--insecure"
    fi
  fi

  local start_time=$(date +%s)
  until curl --cacert "${CERT_DIR}/pki/ca.crt" \
          -H "Authorization: Bearer ${KUBE_BEARER_TOKEN}" \
          ${secure} \
          --max-time 5 --fail --output /dev/null --silent \
          "https://${KUBE_MASTER_IP}/api/v1/pods"; do
      local elapsed=$(($(date +%s) - ${start_time}))
      if [[ ${elapsed} -gt ${KUBE_CLUSTER_INITIALIZATION_TIMEOUT} ]]; then
          echo -e "${color_red}Cluster failed to initialize within ${KUBE_CLUSTER_INITIALIZATION_TIMEOUT} seconds.${color_norm}" >&2
          exit 2
      fi
      printf "."
      sleep 2
  done

  echo "Kubernetes cluster created."

  export KUBE_CERT="${CERT_DIR}/pki/issued/kubecfg.crt"
  export KUBE_KEY="${CERT_DIR}/pki/private/kubecfg.key"
  export CA_CERT="${CERT_DIR}/pki/ca.crt"
  export CONTEXT="${PROJECT}_${INSTANCE_PREFIX}"
  (
   umask 077

   # Update the user's kubeconfig to include credentials for this apiserver.
   create-kubeconfig

   create-kubeconfig-for-federation
  )

  # ensures KUBECONFIG is set
  get-kubeconfig-basicauth
  echo
  echo -e "${color_green}Kubernetes cluster is running.  The master is running at:"
  echo
  echo -e "${color_yellow}  https://${KUBE_MASTER_IP}"
  echo
  echo -e "${color_green}The user name and password to use is located in ${KUBECONFIG}.${color_norm}"
  echo

}

# Removes master replica from etcd cluster.
#
# Assumed vars:
#   REPLICA_NAME
#   PROJECT
#   EXISTING_MASTER_NAME
#   EXISTING_MASTER_ZONE
#
# $1: etcd client port
# returns the result of ssh command which removes replica
function remove-replica-from-etcd() {
  local -r port="${1}"
  [[ -n "${EXISTING_MASTER_NAME}" ]] || return
  gcloud compute ssh "${EXISTING_MASTER_NAME}" \
    --project "${PROJECT}" \
    --zone "${EXISTING_MASTER_ZONE}" \
    --command \
    "curl -s localhost:${port}/v2/members/\$(curl -s localhost:${port}/v2/members -XGET | sed 's/{\\\"id/\n/g' | grep ${REPLICA_NAME}\\\" | cut -f 3 -d \\\") -XDELETE -L 2>/dev/null"
  local -r res=$?
  echo "Removing etcd replica, name: ${REPLICA_NAME}, port: ${port}, result: ${res}"
  return "${res}"
}

# Delete a kubernetes cluster. This is called from test-teardown.
#
# Assumed vars:
#   MASTER_NAME
#   NODE_INSTANCE_PREFIX
#   ZONE
# This function tears down cluster resources 10 at a time to avoid issuing too many
# API calls and exceeding API quota. It is important to bring down the instances before bringing
# down the firewall rules and routes.
function kube-down() {
  local -r batch=200

  detect-project
  detect-node-names # For INSTANCE_GROUPS

  echo "Bringing down cluster"
  set +e  # Do not stop on error

  if [[ "${KUBE_DELETE_NODES:-}" != "false" ]]; then
    # Get the name of the managed instance group template before we delete the
    # managed instance group. (The name of the managed instance group template may
    # change during a cluster upgrade.)
    local templates=$(get-template "${PROJECT}")

    for group in ${INSTANCE_GROUPS[@]:-}; do
      if gcloud compute instance-groups managed describe "${group}" --project "${PROJECT}" --zone "${ZONE}" &>/dev/null; then
        gcloud compute instance-groups managed delete \
          --project "${PROJECT}" \
          --quiet \
          --zone "${ZONE}" \
          "${group}" &
      fi
    done

    # Wait for last batch of jobs
    kube::util::wait-for-jobs || {
      echo -e "Failed to delete instance group(s)." >&2
    }

    for template in ${templates[@]:-}; do
      if gcloud compute instance-templates describe --project "${PROJECT}" "${template}" &>/dev/null; then
        gcloud compute instance-templates delete \
          --project "${PROJECT}" \
          --quiet \
          "${template}"
      fi
    done
  fi

  local -r REPLICA_NAME="${KUBE_REPLICA_NAME:-$(get-replica-name)}"

  set-existing-master

  # Un-register the master replica from etcd and events etcd.
  remove-replica-from-etcd 2379
  remove-replica-from-etcd 4002

  # Delete the master replica (if it exists).
  if gcloud compute instances describe "${REPLICA_NAME}" --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then
    # If there is a load balancer in front of apiservers we need to first update its configuration.
    if gcloud compute target-pools describe "${MASTER_NAME}" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then
      gcloud compute target-pools remove-instances "${MASTER_NAME}" \
        --project "${PROJECT}" \
        --zone "${ZONE}" \
        --instances "${REPLICA_NAME}"
    fi
    # Now we can safely delete the VM.
    gcloud compute instances delete \
      --project "${PROJECT}" \
      --quiet \
      --delete-disks all \
      --zone "${ZONE}" \
      "${REPLICA_NAME}"
  fi

  # Delete the master replica pd (possibly leaked by kube-up if master create failed).
  # TODO(jszczepkowski): remove also possibly leaked replicas' pds
  local -r replica_pd="${REPLICA_NAME:-${MASTER_NAME}}-pd"
  if gcloud compute disks describe "${replica_pd}" --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then
    gcloud compute disks delete \
      --project "${PROJECT}" \
      --quiet \
      --zone "${ZONE}" \
      "${replica_pd}"
  fi

  # Delete disk for cluster registry if enabled
  if [[ "${ENABLE_CLUSTER_REGISTRY}" == true && -n "${CLUSTER_REGISTRY_DISK}" ]]; then
    if gcloud compute disks describe "${CLUSTER_REGISTRY_DISK}" --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then
      gcloud compute disks delete \
        --project "${PROJECT}" \
        --quiet \
        --zone "${ZONE}" \
        "${CLUSTER_REGISTRY_DISK}"
    fi
  fi

  # Check if this are any remaining master replicas.
  local REMAINING_MASTER_COUNT=$(gcloud compute instances list \
    --project "${PROJECT}" \
    --regexp "$(get-replica-name-regexp)" \
    --format "value(zone)" | wc -l)

  # In the replicated scenario, if there's only a single master left, we should also delete load balancer in front of it.
  if [[ "${REMAINING_MASTER_COUNT}" -eq 1 ]]; then
    if gcloud compute forwarding-rules describe "${MASTER_NAME}" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then
      detect-master
      local REMAINING_REPLICA_NAME="$(get-all-replica-names)"
      local REMAINING_REPLICA_ZONE=$(gcloud compute instances list "${REMAINING_REPLICA_NAME}" \
        --project "${PROJECT}" --format="value(zone)")
      gcloud compute forwarding-rules delete \
        --project "${PROJECT}" \
        --region "${REGION}" \
        --quiet \
        "${MASTER_NAME}"
      attach-external-ip "${REMAINING_REPLICA_NAME}" "${REMAINING_REPLICA_ZONE}" "${KUBE_MASTER_IP}"
      gcloud compute target-pools delete \
        --project "${PROJECT}" \
        --region "${REGION}" \
        --quiet \
        "${MASTER_NAME}"
    fi
  fi

  # If there are no more remaining master replicas, we should delete all remaining network resources.
  if [[ "${REMAINING_MASTER_COUNT}" -eq 0 ]]; then
    # Delete firewall rule for the master, etcd servers, and nodes.
    delete-firewall-rules "${MASTER_NAME}-https" "${MASTER_NAME}-etcd" "${NODE_TAG}-all"
    # Delete the master's reserved IP
    if gcloud compute addresses describe "${MASTER_NAME}-ip" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then
      gcloud compute addresses delete \
        --project "${PROJECT}" \
        --region "${REGION}" \
        --quiet \
        "${MASTER_NAME}-ip"
    fi
  fi

  if [[ "${KUBE_DELETE_NODES:-}" != "false" ]]; then
    # Find out what minions are running.
    local -a minions
    minions=( $(gcloud compute instances list \
                  --project "${PROJECT}" --zones "${ZONE}" \
                  --regexp "${NODE_INSTANCE_PREFIX}-.+" \
                  --format='value(name)') )
    # If any minions are running, delete them in batches.
    while (( "${#minions[@]}" > 0 )); do
      echo Deleting nodes "${minions[*]::${batch}}"
      gcloud compute instances delete \
        --project "${PROJECT}" \
        --quiet \
        --delete-disks boot \
        --zone "${ZONE}" \
        "${minions[@]::${batch}}"
      minions=( "${minions[@]:${batch}}" )
    done
  fi

  # If there are no more remaining master replicas: delete routes, pd for influxdb and update kubeconfig
  if [[ "${REMAINING_MASTER_COUNT}" -eq 0 ]]; then
    # Delete routes.
    local -a routes
    # Clean up all routes w/ names like "<cluster-name>-<node-GUID>"
    # e.g. "kubernetes-12345678-90ab-cdef-1234-567890abcdef". The name is
    # determined by the node controller on the master.
    # Note that this is currently a noop, as synchronously deleting the node MIG
    # first allows the master to cleanup routes itself.
    local TRUNCATED_PREFIX="${INSTANCE_PREFIX:0:26}"
    routes=( $(gcloud compute routes list --project "${PROJECT}" \
      --regexp "${TRUNCATED_PREFIX}-.{8}-.{4}-.{4}-.{4}-.{12}"  \
      --format='value(name)') )
    while (( "${#routes[@]}" > 0 )); do
      echo Deleting routes "${routes[*]::${batch}}"
      gcloud compute routes delete \
        --project "${PROJECT}" \
        --quiet \
        "${routes[@]::${batch}}"
      routes=( "${routes[@]:${batch}}" )
    done

    # Delete persistent disk for influx-db.
    if gcloud compute disks describe "${INSTANCE_PREFIX}"-influxdb-pd --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then
      gcloud compute disks delete \
        --project "${PROJECT}" \
        --quiet \
        --zone "${ZONE}" \
        "${INSTANCE_PREFIX}"-influxdb-pd
    fi

    # Delete all remaining firewall rules and network.
    delete-firewall-rules \
      "${CLUSTER_NAME}-default-internal-master" \
      "${CLUSTER_NAME}-default-internal-node" \
      "${NETWORK}-default-ssh" \
      "${NETWORK}-default-internal"  # Pre-1.5 clusters
    if [[ "${KUBE_DELETE_NETWORK}" == "true" ]]; then
      delete-network || true  # might fail if there are leaked firewall rules
    fi

    # If there are no more remaining master replicas, we should update kubeconfig.
    export CONTEXT="${PROJECT}_${INSTANCE_PREFIX}"
    clear-kubeconfig
  else
  # If some master replicas remain: cluster has been changed, we need to re-validate it.
    echo "... calling validate-cluster" >&2
    # Override errexit
    (validate-cluster) && validate_result="$?" || validate_result="$?"

    # We have two different failure modes from validate cluster:
    # - 1: fatal error - cluster won't be working correctly
    # - 2: weak error - something went wrong, but cluster probably will be working correctly
    # We just print an error message in case 2).
    if [[ "${validate_result}" -eq 1 ]]; then
      exit 1
    elif [[ "${validate_result}" -eq 2 ]]; then
      echo "...ignoring non-fatal errors in validate-cluster" >&2
    fi
  fi
  set -e
}

# Prints name of one of the master replicas in the current zone. It will be either
# just MASTER_NAME or MASTER_NAME with a suffix for a replica (see get-replica-name-regexp).
#
# Assumed vars:
#   PROJECT
#   ZONE
#   MASTER_NAME
#
# NOTE: Must be in sync with get-replica-name-regexp and set-replica-name.
function get-replica-name() {
  echo $(gcloud compute instances list \
    --project "${PROJECT}" \
    --zones "${ZONE}" \
    --regexp "$(get-replica-name-regexp)" \
    --format "value(name)" | head -n1)
}

# Prints comma-separated names of all of the master replicas in all zones.
#
# Assumed vars:
#   PROJECT
#   MASTER_NAME
#
# NOTE: Must be in sync with get-replica-name-regexp and set-replica-name.
function get-all-replica-names() {
  echo $(gcloud compute instances list \
    --project "${PROJECT}" \
    --regexp "$(get-replica-name-regexp)" \
    --format "value(name)" | tr "\n" "," | sed 's/,$//')
}

# Prints the number of all of the master replicas in all zones.
#
# Assumed vars:
#   MASTER_NAME
function get-master-replicas-count() {
  detect-project
  local num_masters=$(gcloud compute instances list \
    --project "${PROJECT}" \
    --regexp "$(get-replica-name-regexp)" \
    --format "value(zone)" | wc -l)
  echo -n "${num_masters}"
}

# Prints regexp for full master machine name. In a cluster with replicated master,
# VM names may either be MASTER_NAME or MASTER_NAME with a suffix for a replica.
function get-replica-name-regexp() {
  echo "${MASTER_NAME}(-...)?"
}

# Sets REPLICA_NAME to a unique name for a master replica that will match
# expected regexp (see get-replica-name-regexp).
#
# Assumed vars:
#   PROJECT
#   ZONE
#   MASTER_NAME
#
# Sets:
#   REPLICA_NAME
function set-replica-name() {
  local instances=$(gcloud compute instances list \
    --project "${PROJECT}" \
    --regexp "$(get-replica-name-regexp)" \
    --format "value(name)")

  suffix=""
  while echo "${instances}" | grep "${suffix}" &>/dev/null; do
    suffix="$(date | md5sum | head -c3)"
  done
  REPLICA_NAME="${MASTER_NAME}-${suffix}"
}

# Gets the instance template for given NODE_INSTANCE_PREFIX. It echos the template name so that the function
# output can be used.
# Assumed vars:
#   NODE_INSTANCE_PREFIX
#
# $1: project
function get-template() {
  gcloud compute instance-templates list -r "${NODE_INSTANCE_PREFIX}-template(-(${KUBE_RELEASE_VERSION_DASHED_REGEX}|${KUBE_CI_VERSION_DASHED_REGEX}))?" \
    --project="${1}" --format='value(name)'
}

# Checks if there are any present resources related kubernetes cluster.
#
# Assumed vars:
#   MASTER_NAME
#   NODE_INSTANCE_PREFIX
#   ZONE
#   REGION
# Vars set:
#   KUBE_RESOURCE_FOUND
function check-resources() {
  detect-project
  detect-node-names

  echo "Looking for already existing resources"
  KUBE_RESOURCE_FOUND=""

  if [[ -n "${INSTANCE_GROUPS[@]:-}" ]]; then
    KUBE_RESOURCE_FOUND="Managed instance groups ${INSTANCE_GROUPS[@]}"
    return 1
  fi

  if gcloud compute instance-templates describe --project "${PROJECT}" "${NODE_INSTANCE_PREFIX}-template" &>/dev/null; then
    KUBE_RESOURCE_FOUND="Instance template ${NODE_INSTANCE_PREFIX}-template"
    return 1
  fi

  if gcloud compute instances describe --project "${PROJECT}" "${MASTER_NAME}" --zone "${ZONE}" &>/dev/null; then
    KUBE_RESOURCE_FOUND="Kubernetes master ${MASTER_NAME}"
    return 1
  fi

  if gcloud compute disks describe --project "${PROJECT}" "${MASTER_NAME}"-pd --zone "${ZONE}" &>/dev/null; then
    KUBE_RESOURCE_FOUND="Persistent disk ${MASTER_NAME}-pd"
    return 1
  fi

  if gcloud compute disks describe --project "${PROJECT}" "${CLUSTER_REGISTRY_DISK}" --zone "${ZONE}" &>/dev/null; then
    KUBE_RESOURCE_FOUND="Persistent disk ${CLUSTER_REGISTRY_DISK}"
    return 1
  fi

  # Find out what minions are running.
  local -a minions
  minions=( $(gcloud compute instances list \
                --project "${PROJECT}" --zones "${ZONE}" \
                --regexp "${NODE_INSTANCE_PREFIX}-.+" \
                --format='value(name)') )
  if (( "${#minions[@]}" > 0 )); then
    KUBE_RESOURCE_FOUND="${#minions[@]} matching matching ${NODE_INSTANCE_PREFIX}-.+"
    return 1
  fi

  if gcloud compute firewall-rules describe --project "${PROJECT}" "${MASTER_NAME}-https" &>/dev/null; then
    KUBE_RESOURCE_FOUND="Firewall rules for ${MASTER_NAME}-https"
    return 1
  fi

  if gcloud compute firewall-rules describe --project "${PROJECT}" "${NODE_TAG}-all" &>/dev/null; then
    KUBE_RESOURCE_FOUND="Firewall rules for ${MASTER_NAME}-all"
    return 1
  fi

  local -a routes
  routes=( $(gcloud compute routes list --project "${PROJECT}" \
    --regexp "${INSTANCE_PREFIX}-minion-.{4}" --format='value(name)') )
  if (( "${#routes[@]}" > 0 )); then
    KUBE_RESOURCE_FOUND="${#routes[@]} routes matching ${INSTANCE_PREFIX}-minion-.{4}"
    return 1
  fi

  if gcloud compute addresses describe --project "${PROJECT}" "${MASTER_NAME}-ip" --region "${REGION}" &>/dev/null; then
    KUBE_RESOURCE_FOUND="Master's reserved IP"
    return 1
  fi

  # No resources found.
  return 0
}

# Prepare to push new binaries to kubernetes cluster
#  $1 - whether prepare push to node
function prepare-push() {
  local node="${1-}"
  #TODO(dawnchen): figure out how to upgrade a Container Linux node
  if [[ "${node}" == "true" && "${NODE_OS_DISTRIBUTION}" != "debian" ]]; then
    echo "Updating nodes in a kubernetes cluster with ${NODE_OS_DISTRIBUTION} is not supported yet." >&2
    exit 1
  fi
  if [[ "${node}" != "true" && "${MASTER_OS_DISTRIBUTION}" != "debian" ]]; then
    echo "Updating the master in a kubernetes cluster with ${MASTER_OS_DISTRIBUTION} is not supported yet." >&2
    exit 1
  fi

  OUTPUT=${KUBE_ROOT}/_output/logs
  mkdir -p ${OUTPUT}

  ensure-temp-dir
  detect-project
  detect-master
  detect-node-names
  get-kubeconfig-basicauth
  get-kubeconfig-bearertoken

  # Make sure we have the tar files staged on Google Storage
  tars_from_version

  # Prepare node env vars and update MIG template
  if [[ "${node}" == "true" ]]; then
    write-node-env

    # TODO(zmerlynn): Refactor setting scope flags.
    local scope_flags=
    if [ -n "${NODE_SCOPES}" ]; then
      scope_flags="--scopes ${NODE_SCOPES}"
    else
      scope_flags="--no-scopes"
    fi

    # Ugly hack: Since it is not possible to delete instance-template that is currently
    # being used, create a temp one, then delete the old one and recreate it once again.
    local tmp_template_name="${NODE_INSTANCE_PREFIX}-template-tmp"
    create-node-instance-template $tmp_template_name

    local template_name="${NODE_INSTANCE_PREFIX}-template"
    for group in ${INSTANCE_GROUPS[@]:-}; do
      gcloud compute instance-groups managed \
        set-instance-template "${group}" \
        --template "$tmp_template_name" \
        --zone "${ZONE}" \
        --project "${PROJECT}" || true;
    done

    gcloud compute instance-templates delete \
      --project "${PROJECT}" \
      --quiet \
      "$template_name" || true

    create-node-instance-template "$template_name"

    for group in ${INSTANCE_GROUPS[@]:-}; do
      gcloud compute instance-groups managed \
        set-instance-template "${group}" \
        --template "$template_name" \
        --zone "${ZONE}" \
        --project "${PROJECT}" || true;
    done

    gcloud compute instance-templates delete \
      --project "${PROJECT}" \
      --quiet \
      "$tmp_template_name" || true
  fi
}

# Push binaries to kubernetes master
function push-master() {
  echo "Updating master metadata ..."
  write-master-env
  prepare-startup-script
  add-instance-metadata-from-file "${KUBE_MASTER}" "kube-env=${KUBE_TEMP}/master-kube-env.yaml" "startup-script=${KUBE_TEMP}/configure-vm.sh"

  echo "Pushing to master (log at ${OUTPUT}/push-${KUBE_MASTER}.log) ..."
  cat ${KUBE_TEMP}/configure-vm.sh | gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone "${ZONE}" "${KUBE_MASTER}" --command "sudo bash -s -- --push" &> ${OUTPUT}/push-"${KUBE_MASTER}".log
}

# Push binaries to kubernetes node
function push-node() {
  node=${1}

  echo "Updating node ${node} metadata... "
  prepare-startup-script
  add-instance-metadata-from-file "${node}" "kube-env=${KUBE_TEMP}/node-kube-env.yaml" "startup-script=${KUBE_TEMP}/configure-vm.sh"

  echo "Start upgrading node ${node} (log at ${OUTPUT}/push-${node}.log) ..."
  cat ${KUBE_TEMP}/configure-vm.sh | gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone "${ZONE}" "${node}" --command "sudo bash -s -- --push" &> ${OUTPUT}/push-"${node}".log
}

# Push binaries to kubernetes cluster
function kube-push() {
  # Disable this until it's fixed.
  # See https://github.com/kubernetes/kubernetes/issues/17397
  echo "./cluster/kube-push.sh is currently not supported in GCE."
  echo "Please use ./cluster/gce/upgrade.sh."
  exit 1

  prepare-push true

  push-master

  for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
    push-node "${NODE_NAMES[$i]}" &
  done

  kube::util::wait-for-jobs || {
    echo -e "${color_red}Some commands failed.${color_norm}" >&2
  }

  # TODO(zmerlynn): Re-create instance-template with the new
  # node-kube-env. This isn't important until the node-ip-range issue
  # is solved (because that's blocking automatic dynamic nodes from
  # working). The node-kube-env has to be composed with the KUBELET_TOKEN
  # and KUBE_PROXY_TOKEN.  Ideally we would have
  # http://issue.k8s.io/3168
  # implemented before then, though, so avoiding this mess until then.

  echo
  echo "Kubernetes cluster is running.  The master is running at:"
  echo
  echo "  https://${KUBE_MASTER_IP}"
  echo
  echo "The user name and password to use is located in ~/.kube/config"
  echo
}

# -----------------------------------------------------------------------------
# Cluster specific test helpers used from hack/e2e.go

# Execute prior to running tests to build a release if required for env.
#
# Assumed Vars:
#   KUBE_ROOT
function test-build-release() {
  # Make a release
  "${KUBE_ROOT}/build/release.sh"
}

# Execute prior to running tests to initialize required structure. This is
# called from hack/e2e.go only when running -up.
#
# Assumed vars:
#   Variables from config.sh
function test-setup() {
  # Detect the project into $PROJECT if it isn't set
  detect-project

  if [[ ${MULTIZONE:-} == "true" && -n ${E2E_ZONES:-} ]]; then
    for KUBE_GCE_ZONE in ${E2E_ZONES}
    do
      KUBE_GCE_ZONE="${KUBE_GCE_ZONE}" KUBE_USE_EXISTING_MASTER="${KUBE_USE_EXISTING_MASTER:-}" "${KUBE_ROOT}/cluster/kube-up.sh"
      KUBE_USE_EXISTING_MASTER="true" # For subsequent zones we use the existing master
    done
  else
    "${KUBE_ROOT}/cluster/kube-up.sh"
  fi

  # Open up port 80 & 8080 so common containers on minions can be reached
  # TODO(roberthbailey): Remove this once we are no longer relying on hostPorts.
  local start=`date +%s`
  gcloud compute firewall-rules create \
    --project "${PROJECT}" \
    --target-tags "${NODE_TAG}" \
    --allow tcp:80,tcp:8080 \
    --network "${NETWORK}" \
    "${NODE_TAG}-${INSTANCE_PREFIX}-http-alt" 2> /dev/null || true
  # As there is no simple way to wait longer for this operation we need to manually
  # wait some additional time (20 minutes altogether).
  while ! gcloud compute firewall-rules describe --project "${PROJECT}" "${NODE_TAG}-${INSTANCE_PREFIX}-http-alt" 2> /dev/null; do
    if [[ $(($start + 1200)) -lt `date +%s` ]]; then
      echo -e "${color_red}Failed to create firewall ${NODE_TAG}-${INSTANCE_PREFIX}-http-alt in ${PROJECT}" >&2
      exit 1
    fi
    sleep 5
  done

  # Open up the NodePort range
  # TODO(justinsb): Move to main setup, if we decide whether we want to do this by default.
  start=`date +%s`
  gcloud compute firewall-rules create \
    --project "${PROJECT}" \
    --target-tags "${NODE_TAG}" \
    --allow tcp:30000-32767,udp:30000-32767 \
    --network "${NETWORK}" \
    "${NODE_TAG}-${INSTANCE_PREFIX}-nodeports" 2> /dev/null || true
  # As there is no simple way to wait longer for this operation we need to manually
  # wait some additional time (20 minutes altogether).
  while ! gcloud compute firewall-rules describe --project "${PROJECT}" "${NODE_TAG}-${INSTANCE_PREFIX}-nodeports" 2> /dev/null; do
    if [[ $(($start + 1200)) -lt `date +%s` ]]; then
      echo -e "${color_red}Failed to create firewall ${NODE_TAG}-${INSTANCE_PREFIX}-nodeports in ${PROJECT}" >&2
      exit 1
    fi
    sleep 5
  done
}

# Execute after running tests to perform any required clean-up. This is called
# from hack/e2e.go
function test-teardown() {
  detect-project
  echo "Shutting down test cluster in background."
  delete-firewall-rules \
    "${NODE_TAG}-${INSTANCE_PREFIX}-http-alt" \
    "${NODE_TAG}-${INSTANCE_PREFIX}-nodeports"
  if [[ ${MULTIZONE:-} == "true" && -n ${E2E_ZONES:-} ]]; then
      local zones=( ${E2E_ZONES} )
      # tear them down in reverse order, finally tearing down the master too.
      for ((zone_num=${#zones[@]}-1; zone_num>0; zone_num--))
      do
	  KUBE_GCE_ZONE="${zones[zone_num]}" KUBE_USE_EXISTING_MASTER="true" "${KUBE_ROOT}/cluster/kube-down.sh"
      done
      KUBE_GCE_ZONE="${zones[0]}" KUBE_USE_EXISTING_MASTER="false" "${KUBE_ROOT}/cluster/kube-down.sh"
  else
      "${KUBE_ROOT}/cluster/kube-down.sh"
  fi
}

# SSH to a node by name ($1) and run a command ($2).
function ssh-to-node() {
  local node="$1"
  local cmd="$2"
  # Loop until we can successfully ssh into the box
  for try in {1..5}; do
    if gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --ssh-flag="-o ConnectTimeout=30" --project "${PROJECT}" --zone="${ZONE}" "${node}" --command "echo test > /dev/null"; then
      break
    fi
    sleep 5
  done
  # Then actually try the command.
  gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --ssh-flag="-o ConnectTimeout=30" --project "${PROJECT}" --zone="${ZONE}" "${node}" --command "${cmd}"
}

# Perform preparations required to run e2e tests
function prepare-e2e() {
  detect-project
}

# Writes configure-vm.sh to a temporary location with comments stripped. GCE
# limits the size of metadata fields to 32K, and stripping comments is the
# easiest way to buy us a little more room.
function prepare-startup-script() {
  # Find a standard sed instance (and ensure that the command works as expected on a Mac).
  SED=sed
  if which gsed &>/dev/null; then
    SED=gsed
  fi
  if ! ($SED --version 2>&1 | grep -q GNU); then
    echo "!!! GNU sed is required.  If on OS X, use 'brew install gnu-sed'."
    exit 1
  fi
  $SED '/^\s*#\([^!].*\)*$/ d' ${KUBE_ROOT}/cluster/gce/configure-vm.sh > ${KUBE_TEMP}/configure-vm.sh
}


"It is a real privilege being invited into a young person's life and it is inspirational seeing them develop and flourish despite what they are going through medically. Doing this job has taught me how precious life truly is."
Caroline has a Masters Module in Teenage Cancer Care, an Adavanced Certificate in Cognitive Behavioural Therapy, a NOCN Substance Misuse Practitioner Award, an Advanced Certificate in Person Centred Counselling and a BSc (hons) in Psychology.
I provide emotional support to young people who choose to have their treatment closer to home and not at the Teenage Cancer Trust Units.
I enjoy travelling to different parts of the North West and taking the service to the young people in their homes and in their communities. It is a real privilege being invited into a young person's life and it is inspirational seeing them develop and flourish despite what they are going through medically. Doing this job has taught me how precious life truly is.
Teenage Cancer Trust helps young people live "normally" after being diagnosed with cancer. They have a team of professionals who understand what they are going through and understand what is important to them as a young person.


text:
#include "tensorflow/core/kernels/cwise_ops_common.h"

namespace tensorflow {
REGISTER5(BinaryOp, CPU, "FloorDiv", functor::safe_floor_div, uint8, uint16,
          int16, int32, int64);
REGISTER3(BinaryOp, CPU, "FloorDiv", functor::floor_div_real, float,
          Eigen::half, double);

#if GOOGLE_CUDA
REGISTER4(BinaryOp, GPU, "FloorDiv", functor::floor_div, uint8, uint16, int16,
          int64);
REGISTER3(BinaryOp, GPU, "FloorDiv", functor::floor_div_real, float,
          Eigen::half, double);
#endif

#if GOOGLE_CUDA
// A special GPU kernel for int32.
// TODO(b/25387198): Also enable int32 in device memory. This kernel
// registration requires all int32 inputs and outputs to be in host memory.
REGISTER_KERNEL_BUILDER(Name("FloorDiv")
                            .Device(DEVICE_GPU)
                            .HostMemory("x")
                            .HostMemory("y")
                            .HostMemory("z")
                            .TypeConstraint<int32>("T"),
                        BinaryOp<CPUDevice, functor::safe_floor_div<int32>>);
#endif

#ifdef TENSORFLOW_USE_SYCL
REGISTER_KERNEL_BUILDER(Name("FloorDiv")
                            .Device(DEVICE_SYCL)
                            .HostMemory("x")
                            .HostMemory("y")
                            .HostMemory("z")
                            .TypeConstraint<int32>("T"),
                        BinaryOp<CPUDevice, functor::safe_floor_div<int32>>);
#endif  // TENSORFLOW_USE_SYCL
}  // namespace tensorflow

redpajama_set_name: RedPajamaGithub


Wrapping a boat is typically significantly cheaper than a spray painting job, often as little as one third of the price of a quality paint job. Individual prices will depend on factors such as the size of the area of the boat you want wrapped along with the design itself and the finish etc. There is also the added benefit of the time it takes to do it, wrapping can be done in as little as one day while painting can take weeks meaning much more time off the water – costly if it's a commercial vessel.
Maintaining a wrapped boat is a lot easier than a painted one. No buffing or polishing is required with a vinyl wrapped boat, just a simple wash with water and soap will keep your wrap pristine. Additionally, any chips or holes which may appear in the wrap, should you knock off something in the water, are very simple to patch up and can be done with ease and without needing to dry dock the boat.
While many marine paints contain chemicals, which can be toxic to marine life, our vinyl wraps for boats are eco-friendly with no solvents or chemicals used in their production. Given our vast and beautiful array of marine life in Australian waters, this is a massive benefit of wraps compared to painting boats.
Another huge advantage of wrapping your boat is the protection the wrap offers to the factory paintwork underneath. Our vinyl wrap protects your vessel from fading and damage from the harsh glare of the Australian sun and constantly being in the water. We even offer clear wraps just to preserve your factory paint to increase the re-sale value of your boat down the track.
There are practically limitless options when it comes to customising your boat. Your boat will turn heads from the ramp onto the water with a custom design unique to your vessel. Why not proudly show off your company branding on your boat if it's a commercial vessel? Or be the envy of fellow seamen with a black matte finish inspired by Mediterranean yachts of the rich and famous? Along with infinite designs and colours we also offer a variety of different textures including matte, gloss, satin and metallic finishes. Whatever the grand idea you have for your boat, Auto Artisan can make it a reality.
To discuss a vinyl wrap for your boat, call Auto Artisan or drop into our Sydney workshop today for a no-obligation quote.


text:
COVID Outbreaks at the Southern Border Cannot Be Ignored
Lawmakers are increasingly raising concerns about unvaccinated undocumented migrants being relocated to detention centers in their states.
Patrice Onwuka	| July 29, 2021
As the Delta variant of the coronavirus continues to spread across the country, little attention has been paid to the public health issues at the U.S. southern border, where the spread of the disease among illegal migrants is a real concern and it could be moving to other states.
Fixing the surge of migrants attempting to cross the southern border has been one of the most significant challenges facing recent administrations. The Trump administration was blasted for installing strict policies that stemmed migrant flows at various times over the past four years. The Biden administration has reversed many of those policies leading to a surge of people attempting to cross into the U.S. from Central American countries and others countries around the world.
As a result, border apprehensions surpassed 1 million this fiscal year, a level not seen since 2006. Each month that President Biden has been in office, apprehensions have risen, but not enough attention has been paid to the physical health of migrants entering our nation.
We cannot say that Illegal immigrants are driving the spread of COVID-19, but some officials are beginning to think they are contributing to it.
Infections spreading among illegal migrants under U.S. care
As the New York Times reported at the start of July, coronavirus cases are surging at the migrant detention centers along the southern border. Last week, Fox News reported that 135 detainees in the Rio Grande Valley (RGV) Sector tested positive for COVID-19 in the first two weeks of July alone, an increase of 900% compared to the previous 14 months.
Yet, these conditions have been downplayed (perhaps intentionally). A new set of whistleblower claims about the conditions at Fort Bliss, a child detention center in Texas, suggest that the administration wants us to believe that there's nothing really bad going on despite evidence to the contrary:
Covid was widespread among children and eventually spread to many employees. Hundreds of children contracted Covid-19 in overcrowded conditions. Adequate masks were not consistently provided to children, nor was their use consistently enforced," the whistleblowers, Arthur Pearlstein and Lauren Reinhold, said in a federal whistleblower complaint filed Wednesday.
But at the end of their service, they said, federal detailees were regularly given written instructions from HHS public affairs that told them, "when asked, to make everything sound positive about the Fort Bliss experience and to play down anything negative."
How awful. Sickness aside, any parent should be saddened by the reported ill-treatment of children under the care of the Health and Human Services. Overcrowded facilities house little ones who face lice outbreaks, a lack of clean underwear, and mistreatment at the hands of those who are supposed to care for them.
This is the problem that the Biden administration created in choosing to accept unaccompanied children brought to the southern border rather than turning them away.
Title 42 may stay a little longer
In one tentative sign of good news, the Biden administration has indicated it may not end Title 42, a public health order related to the pandemic, to the chagrin of illegal immigrant advocates. Title 42 was installed by the Trump administration and empowered border authorities to immediately turn away migrants to the northern and southern border on public health grounds instead of detaining and processing them. This was a common-sense approach to stop the spread of the coronavirus from untested and potentially sick migrants as we deal with the national public health crisis within our borders.
The order faces a legal challenge from the American Civil Liberties Union, which is reportedly negotiating with the White House on whether or how to move forward, and vocal opposition from leftist groups that want illegal migrants to come freely into our country.
The Biden administration hastily reversed other Trump-era policies but kept this one in place, knowing that in lifting the order, they would unleash a flood of potentially sick migrants. However, the administration planned a phased lifting of Title 42 sometime this month (July 2021) in response to opponents of the regulation.
Although the White House has backed down on plans, President Biden could change his mind tomorrow and end Title 42. If he does so, it would likely be calamitous as all of those migrants who have been turned away would now be allowed in and people around the world would be incentivized to come to the southern border. If Border Patrol agents are overwhelmed now, they would overrun.
Infected migrants spreading out across the country
Lawmakers are increasingly raising concerns about unvaccinated undocumented migrants being relocated to detention centers in their states, potentially carrying the virus with them. Iowa Governor Kim Reynolds said recently "As they're trying to impact additional restrictions on Americans, we've got people coming across the border that haven't been vaccinated, and so that's also something we need to continue to look at."
Public health experts say it's not surprising that migrants could be contributing to the spread of the virus in different states. As the New York Post reports, they are not tested for coronavirus until after they arrive at a facility and then placed in isolation. There's ample opportunity for them to be exposed or to infect someone else on the bus ride to the facility.
When they are relocated to areas with low vaccination rates, it spells a recipe for trouble. Sharon Dolovich, director of the Covid Behind Bars Data Project at UCLA explained:
"You have people coming in and out of the facility, into communities where incomplete vaccination allows these variants to flourish, and then you bring them inside the facilities, and that variant will spread.
"What you're describing is the combination of insufficient vaccination plus the evolution of the virus, and that is really scary."
Where is the Biden administration on this issue?
Vice President Kamala Harris recently took a trip to the southern border for what turned out to be little more than a photo-op. She visited an area that was not a hot spot for border crossings and she hardly engaged local officials, federal border agents, and local communities on the issues bubbling up there including health concerns of migrants and the impact they could have on spreading coronavirus. This is not a political issue. It's a matter of public health and one that the White House cannot ignore.
Travel Bans are Still in Place
May Davis	| July 22, 2021
Kamala Harris's Visit to the Southern Border Missed the Mark
Patrice Onwuka	| June 29, 2021
Ninth Circuit Allows Accused Male Student to Sue UCLA for Sex Discrimination
Jennifer C. Braceras	| Blog
Supreme Court Deems OSHA Vaccine Mandate Unlawful, and Rightly So
May Davis	| Blog
40-Year-High Inflation Rocks American Households
Patrice Onwuka	| Blog
Forget Social Distancing—We Need to Focus on Social Connection
Kristin Shapiro	| Blog

redpajama_set_name: RedPajamaCommonCrawl


Please note that the Maple Hill South route starts at Exit 62 on Thursday and Friday.
It starts at Exit 64 on Saturday and Monday.
Welcome to the University of Arkansas.
We hope you find this map helpful as you navigate on campus.
Please use the directions provided to drive to your on-campus residence.
Or check out our Move-in app on the Apple App Store. You can find it through the link provided or search Housing Move-in to find it in the store.
After unloading, we recommend that parents travel to the nearest parking deck to continue helping their student settle in to campus. The closest parking garage to your residence hall is noted on the map. You may not park overnight in the parking deck.
Students are encouraged to park in Lot 56 near MLK Blvd. after unloading. Students with Resident Reserved parking permits may begin parking in those lots mid-afternoon Monday.
Please see the map of the north and south loops.


text:
<?xml version='1.0' encoding='utf-8'?>
<manifest android:hardwareAccelerated="true" android:versionCode="10207" android:versionName="1.2.7" package="com.adondeir.adic" xmlns:android="http://schemas.android.com/apk/res/android">
    <supports-screens android:anyDensity="true" android:largeScreens="true" android:normalScreens="true" android:resizeable="true" android:smallScreens="true" android:xlargeScreens="true" />
    <application android:hardwareAccelerated="true" android:icon="@mipmap/icon" android:label="@string/app_name" android:supportsRtl="true">
        <activity android:configChanges="orientation|keyboardHidden|keyboard|screenSize|locale" android:label="@string/activity_name" android:launchMode="singleTop" android:name="MainActivity" android:screenOrientation="portrait" android:theme="@android:style/Theme.DeviceDefault.NoActionBar" android:windowSoftInputMode="adjustResize">
            <intent-filter android:label="@string/launcher_name">
                <action android:name="android.intent.action.MAIN" />
                <category android:name="android.intent.category.LAUNCHER" />
            </intent-filter>
        </activity>
        <receiver android:enabled="true" android:name="com.google.android.gms.analytics.AnalyticsReceiver">
            <intent-filter>
                <action android:name="com.google.android.gms.analytics.ANALYTICS_DISPATCH" />
            </intent-filter>
        </receiver>
        <service android:enabled="true" android:exported="false" android:name="com.google.android.gms.analytics.AnalyticsService" />
        <receiver android:enabled="true" android:exported="true" android:name="com.google.android.gms.analytics.CampaignTrackingReceiver">
            <intent-filter>
                <action android:name="com.android.vending.INSTALL_REFERRER" />
            </intent-filter>
        </receiver>
        <service android:enabled="true" android:exported="false" android:name="com.google.android.gms.analytics.CampaignTrackingService" />
    </application>
    <uses-sdk android:minSdkVersion="14" android:targetSdkVersion="25" />
    <uses-permission android:name="android.permission.ACCESS_COARSE_LOCATION" />
    <uses-permission android:name="android.permission.ACCESS_FINE_LOCATION" />
    <uses-feature android:name="android.hardware.location.gps" />
    <uses-permission android:name="android.permission.INTERNET" />
    <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
</manifest>


text:
Thorbjørn Berntsen (born 13 April 1935) is a former Norwegian politician representing the Labour Party. He was Minister of the Environment from
1990 to 1997. He was also a member of the Storting for Oslo from 1977 to 1997.

References

1935 births
Living people
Ministers of Climate and the Environment of Norway
Members of the Storting
Labour Party (Norway) politicians
20th-century Norwegian politicians

redpajama_set_name: RedPajamaWikipedia


text:
Belt's Beer Garden
Belt's Beer Garden: Un-American!
There's beer made outside of America??!
By Dave Drury May 20, 2016, 5:00pm EDT
Share All sharing options for: Belt's Beer Garden: Un-American!
This week, I give America a break. I've got two foreign breweries - one I'm sure you've heard of - to spotlight...because good beer isn't exclusive to the US.
First up, from our friendly neighbors to the North, Driftwood's Fat Tug.
Driftwood Fat Tug
I had never heard of Driftwood before. But when I saw this cool looking bottle - the dark blue matte label mixed with the shiny silver lettering - it really caught my eye. What can I say; I'm a sucker for a cool bottle. And, at $8.99 for the 22-ounce bomber, the beer was reasonably priced so I had to try it.
Driftwood, I later came to find out, is from our neighbors to the north. They call Victoria in British Columbia, Canada home. The brewery, which is right on the edge of Canada/Washington border, has been making beers since 2008 and has grown vastly since their inception. I'm not too sure where all in the United States they distribute though, so you'll just have to keep an eye out for them.
When I popped open the cap and poured Driftwood's only year-round IPA, Fat Tug, the contents were a light copper, slightly amber color with over two fingers of fluffy white head that has some pretty solid retention time to it.
Inside the 22-ounces I poured are five different types of hops (Amarillo, Cascade, Centennial, Citra, and Columbus) and three types of malt (Carapils, Crystal and Pilsner). And, I've got to say, it smells wonderful. The aroma consisted of loads of juicy tropical fruit – honeydew melon, grapefruit, mango – with a tiny hop presence and a bready malt backing.
I was really happy that the flavor mirrored the aroma almost spot on. Up front there is some mild carbonation that is followed by the fruity hops; mostly it's the melon and mango that show up here. The tastes then turn towards the malts on the backend, ending with a solid bready, yet slightly caramel, flavor. The beer has a slightly creamier mouthfeel to it than many IPAs and is pretty dry on the backend as well.
Fair warning though: the 80+ IBU (not sure on the actual amount, I've seen anywhere from 80-100) this beer has leaves a lingering bitterness on the aftertaste that some people probably wouldn't like.
The 7% booze that's inside the bottle is hidden really well behind the hops and malts. However, as the bomber got emptier, I could feel the alcohol start to warm my chest as I drank it down. I was surprised at how moderate the lacing was as the beer disappeared. There were little cloud formations along the side of my glass but, for the most part, there really wasn't much lacing at all.
Overall, this might be my favorite Canadian beer so far. Like I said, I had never heard of Driftwood before but now I see I might be missing out on some really great brews. Fat Tug has great tropical flavors, a nice balance between hops and malt, and a solid ABV to make it worth you while and your money. 8.5/10
Next up, this company is quite well known and headquartered in Copenhagen, Denmark. Evil Twin is back with their Mango Femme Fatale.
Evil Twin began in 2010 and is a ghost brewery, they make beers using others brewing equipment instead of having their own brewery. For example, this beer - Mango Femme Fatale - was made using Westbrook Brewing Co's equipment in Mount Pleasant, South Carolina. The 22-ounce bomber will run you around $10.99, which is about average for Evil Twin beers.
Mango Femme Fatale is an American IPA brewed with mangos and a single hop – the Kohatu hop.
When poured, this IPA is a burnt orange color with more head than it knows what to do with. There were easily more than three fingers of foam on my initial pour. Luckily the head dissipates rather quickly and, within like two minutes, there is just a little bit of airy white bubbles left atop the beer.
There is an interesting aroma with this one. It has quite a yeasty, spicy aroma – it smells way more like a farmhouse than an IPA. There's also plenty of fruit (mango, passion fruit) with a backing of pine and pale malts.
The taste followed the aroma rather closely. There is an initial burst of carbonation (it's pretty damn carbonated) that fizzes out into tropical fruit flavor before the pines and malts kick in and grab a hold of the taste. I was surprised that there was not as much mango as I thought there would be. It was mostly pineapple and passion fruit with just a subtle touch of the mango.
Also, like the aroma indicated, this beer tasted much more like a saison than an IPA. There was a spicy yeast quality with every sip that I'm used to getting only with farmhouse ales.
At the end there is a slightly bitter aftertaste that lingers for a little while. The fact that Mango Femme Fatale ends on the dry side adds to that bitterness. The beer is a tad higher than average in alcohol content, at 7% ABV, and is hidden really well in the taste.
Overall, this is not one of my favorite from them. The mango is a little too hidden and, with all the other flavors and spices going on, the beer just didn't live up to the rest of their arsenal. Maybe I just don't like the Kohatu hops…I'm not sure. But I do know that Evil Twin is usually a very solid brand. This one just falls a tad short. 7/10
2023 Tropical Bowl Post-Game Recap and Analysis 10 former MAC stars participated in a contest under dreary conditions in Orlando this past weekend in front of an audience of pro scouts.
Four MAC players invited to 2023 NFLPA Collegiate Bowl The Pasadena-based bowl game continues to draw talent from across the country, with four MAC stars set to participate.


Emily Shealy is a professional Dressage trainer and rider, specializing in the classical "German" training scale that emphasizes clear, correct, and consistent communication between horse and rider. Emily's training techniques are fair and straightforward, with the horse's wellbeing in the forefront of all work. She utilizes groundwork, long lines, and lunge lessons as regular tools of her work, in addition to routine under-saddle training.
Emily grew up with horses and the Dressage community as she worked her way up the training scale and competition levels. She worked as a full time working student for professional Dressage trainers for over 15 years, as well as putting herself through training and school as a professional braider and groom for nationally & internationally recognized riders for many years prior to stepping out on her own as a professional trainer.
She has had the privilege to work with many national and internationally known Dressage masters and trainers throughout her own training, and as a result, has gained extensive training and show experience with a variety of horses and ponies at many different levels of competition.
Emily's students have been very successful in their goals while under her tutelage, whether it has been to show, or achieve personal training 'bests'. A few of her students' accolades include, but are not limited to, Century Club membership, USDF medals, Pony Club ratings & awards for excellence, and regional championship awards.
As a professionally certified massage therapist and saddle fitter with Shealy Sporthorses, Emily is also able to bring a unique, physiological point of view to her students and horses in training, offering a complete wellness and training programme.
Shealy Sporthorses Dressage is currently accepting students looking to achieve a higher level of communication and clarity in their work with their own horses.


Definition - What does Triplex Pump mean?
A Triplex Pump is a kind of a pump that uses 3 plungers to reduce pulsation of the single reciprocating plunger pumps. The ripple graph or pump ripple of the pump transducer can be smoothed by adding pulsation dampener. High quality plunger seals are required for the dynamic relation of plunger and high pressure fluids. A plunger pump with a large number of plungers has the benefit of the increased or better flow without any pulsation dampener. The only drawback is the increase in crankshaft load and moving parts.
A Triplex Pump is used to move fluids such as slurry. The pump displaces the fluids with the help of physical or mechanical movements. The pump consists of one kind of pump with various cylinders. These are most common type of configuration of pumps that can be used in well servicing and drilling operations. The pumps used in the well service operations are generally capable of handling a vast range of fluid types which include abrasive fluids, corrosive fluids and slurries which contains large particulates. These are the positive displacement reciprocating pumps that consist of three plungers.


You may recognize this one from the promos for Space Dandy on Toonami a few years back.
LOL, this song totally applies right now.
I saw him live in Toronto a couple weeks ago and loved it. Recommend checking him out if he's coming by your area.
Right now I'm listening to the Global DJ Broadcast with Markus Schulz on DI.FM's trance Channel.
I love me some Magic Sword, man. I hope they have a new EP or album in the works. Be nice to have some new material. Legend was closing in on two years ago and only had three songs.


Celine Dion fragrance and perfume list.
Celine Dion fragrances: by Coty, Inc. Celine Dion is a French-Canadian singer. She launched her first celebrity perfume, Celine Dion Parfum, in 2003.
Celine Dion, (born 30 Mar 1968 in Charlemagne, Quebec, Canada) is a French-Canadian singer. Emerging as a teen star, and gaining international recognition in the 1980s, Celine released her first English-language album, Unison, in 1990. She has won multiple Grammy awards, including four in 1999 for My Heart Will Go On, famed in the James Cameron movie, Titanic. At this point, Celine announced she was taking a break to start a family with manager-producer, Rene Angelil. Dion returned to the music industry in 2002, signing with Caesars Palace, Las Vegas, to perform nightly shows at the Colosseum. After nearly 5 years in Las Vegas, Celine returned to the pop music scene and released her Taking Changes album in 2007. In 2003 Celine Dion launched her first of several celebrity perfumes, Celine Dion Parfum, with Coty, Inc.


Check Out All the 2014 FIFI Award Winners—Did Your Favorite Perfume Make the Cut?
The Oscars of the fragrance industry, the FIFI Awards (a.k.a. the Fragrance Foundation Awards), took place last night in what you would imagine is a very nice-smelling Lincoln Center. Check out the fragrances that were honored and see if your favorite is now an award winner.


Combine extreme audio quality and freedom of movement with this wireless SteelSeries Siberia gaming headset. It provides the potential for unlimited play with the help of a pair of hot-swap lithium-ion batteries, and it provides Dolby 7.1 surround sound for professional-level audio. This SteelSeries Siberia gaming headset reduces the risk of interference and lag with its frequency hopping technology.


Double Two shirts are perfect for a smart look or for office wear. With sizes up to a 5XL, a comfortable fit should be easy to find. We stock a wide range of colours and styles, so there is something for all occasions. We also have Ben Sherman and Voeut shirts.


text:
HomePodcast EpsisodesMouse Kingdoms - Weekly EpisodeMK 011: Magic Kingdom - Tomorrowland
Podcast EpsisodesMouse Kingdoms - Weekly EpisodeWDW
MK 011: Magic Kingdom – Tomorrowland
On this episode of Mouse Kingdoms, we discuss More details about Galaxy's Edge ahead of the Disneyland opening, the Skyliner is unwrapped, Epcot Food and Wine information, musical lineup, and more!
Please support the show if you can by going to https://www.mousekingdoms.com/support/.
If you want some Mouse Kingdoms Swag, you can pick some up at https://www.mousekingdoms.com/store.
Book your travel through ConciEARS and get your first year of Touring Plans for FREE! Just mention that you heard about ConciEARS from Mouse Kingdoms Weekly at booking!
This week a lot of the Skyliner gondolas were unwrapped on the line. Each of the gondolas have a different theme ranging from classics like Peter Pan, to attractions like the Haunted Mansion, to the new Mickey animated shorts. The artwork is made to look like the characters are interacting with the gondolas. – http://blogmickey.com/2019/05/all-disney-skyliner-character-gondola-cabins-unwrapped-at-disneys-hollywood-studios/
As Disneyland prepares to debut Galaxy's Edge at the end of the month, Disney has been releasing more and more details on the new land. This week they released the full menus for each restaurant. Some of the options include a Roasted Endorian Tip-yip Salad, which is marinated chicken served with mixed greens, roasted seasonal vegetables, quinoa, pumpkin seeds, and green curry ranch, and Ronto Wrap, which is roasted pork and grilled sausage with peppercorn sauce and tangy slaw wrapped in pita. There are a lot of alcoholic and non-alcoholic drinks as well as a variety of snacks and desserts. For the full menus, plus pictures, check out the link in our show notes. – https://wdwnt.com/2019/05/full-menus-released-for-all-dining-locations-in-galaxys-edge/
Guests will have four themes to choose from when they are building their custom lightsabers in Galaxy's Edge. The four paths that can be chosen are Peace and Justice, which offers simple designs of the Republic era, Power and Control, which have sharp and strong designs of the Sith, Elemental Nature, which use natural parts like wood from a Brylark tree, or a tooth from a Rancor, or Protection and Defense, which is more ancient and mysterious. Five different kyber crystals, which power the sabers, are available to choose from. Colors include purple, green, blue, yellow, and white. The kyber crystal will be placed into a holocron where Jedi or Sith will speak wisdom to the guest. 10 legacy lightsabers will also be available to purchase. – https://wdwnt.com/2019/05/photos-in-depth-look-at-lightsaber-hilts-kyber-crystals-and-holocrons-available-at-savis-workshop-in-star-wars-galaxys-edge/
The new Space Restaurant coming to Future World in Epcot is scheduled to open later this year. The new table service location, being constructed next to Mission: Space, will make guests feel like they are traveling through space while dining. The effect is created with a huge 250 foot long digital screen that wraps around the dining room. – https://wdwnt.com/2019/05/late-2019-opening-announced-for-upcoming-space-themed-restaurant-at-epcot/
More details were released this week about the Food and Wine Festival coming to Epcot from August 29th through November 23rd. There will be 30 global marketplace booths, the Eat to the Beat concert series returning three times each night, the Disney du Jour Dance party is expanding to seven days a week, the Sunday Brunch with the Chef series will run every Sunday through November 17, Party for the Senses, :Mix It, Make It, Celebrate It" hands on workshops on select Mondays and Thursdays, and more. – https://www.wdwmagic.com/events/international-food-and-wine-festival/news/09may2019-more-details-announced-for-the-2019-epcot-food-and-wine-festival.htm
The lineup has been announced for the Eat to the Beat concert series. Nine new acts have been announced for this year, including some first timers like Sawyer Brown, Lauren Daigle, and Kris Allen. Returning acts include Sugar Ray, Hanson, and Boyz II Men. For a complete list and dates, check out the link in our show notes. Shows are Sunday through Thursday at 5:30, 6:45, and 8pm, and Fridays and Saturdays at 6:30, 7:45, and 9pm. – https://www.wdwmagic.com/events/international-food-and-wine-festival/news/09may2019-eat-to-the-beat-concert-series-line-up-for-the-2019-epcot-international-food-and-wine-festival.htm
The Disney Parks are getting more and more reliant on technology. This week, Disney rolled out online reservations for the Bibbity Bobbidi Boutique, The Pirates League, and the Harmony Barber Shop. The online system will allow guests to search for available dates and is similar to the online dining reservation system. Reservations can be made 180 days in advance. – http://blogmickey.com/2019/05/bibbidi-bobbidi-boutique-and-more-experiences-now-bookable-online/
Blog Mickey reported on a rumor that wireless internet access could be coming to the bus system. The new system could be coming to the buses as early as later this year. This would be a nice feature for guests with limited or no cellular signal, especially international guests, to use the My Disney Experience app to plan their day. – http://blogmickey.com/2019/05/rumor-wi-fi-to-be-added-to-buses-at-walt-disney-world-resort/
The first phase of the Tron coaster coming to Tomorrowland in the Magic Kingdom is coming to a close. On May 18th, the Tomorrowland Speedway will reopen with the reconfigured track, which made room for parts of the Tron coaster that should be opening in 2021. The ride vehicles don't look to have been converted to electric like some were hoping, and instead still using gas engines. – https://wdwnt.com/2018/11/tomorrowland-speedway-to-reopen-may-18-2019/
Mickey's Backyard BBQ has been demolished to make way for Fort Wilderness changes. The first water park on property has been completely demolished to make way for the upcoming Disney Vacation Club property called Reflections – A Disney Lakeside Lodge. The Tri-Circle-D ranch, which houses the horses used on Main Street, will also be relocated for upcoming construction soon. The area known as Clementine's Beach will become DVC bungalows. – http://blogmickey.com/2019/05/mickeys-backyard-bbq-demolished-as-fort-wilderness-prepares-for-expansive-changes/
The next time you take a stroll through the Bayou into Pirates of the Caribbean, you may notice that the waterfall effect that had Davy Jones and Blackbeard is now gone. The section of the attraction has returned to the "Dead Men Tell No Tales" audio track that was in place before the change. – https://twitter.com/WDWNT/status/1126880732394209281/photo/1
Discussion Topic:
Magic Kingdom Tomorrowland
Astro Orbiter
Buzz Lightyear's Space Ranger Spin
Monster's, Inc. Laugh Floor
Tomorrowland Speedway
Tomorrowland Transit Authority People Mover
Walt Disney's Carousel of Progress
Tron Lightcycle Power Run
https://dts.podtrac.com/redirect.mp3/pinecast.com/listen/f7c6fdb0-28b4-4415-955e-c22fc95ce09e.mp3
Podcast (mousekingdoms): Play in new window | Download
Previous articleMK 010: Phantom Manor Returns
Next articleDLW 081: New Flex Annual Pass
Mouse Kingdoms - Weekly Episode
MK 014: Epcot Center
MK 014: Trip Planning
MK 013: Magic Kingdom – Main Street, U.S.A.