text:
#!/bin/bash
# Copyright 2018 Mirantis
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
set -o errtrace

if [ $(uname) = Darwin ]; then
  readlinkf(){ perl -MCwd -e 'print Cwd::abs_path shift' "$1";}
else
  readlinkf(){ readlink -f "$1"; }
fi
DIND_ROOT="$(cd $(dirname "$(readlinkf "${BASH_SOURCE}")"); pwd)"

docker_info_output="$(docker info)"

RUN_ON_BTRFS_ANYWAY="${RUN_ON_BTRFS_ANYWAY:-}"
if [[ ! ${RUN_ON_BTRFS_ANYWAY} ]] && echo "$docker_info_output"| grep -q '^ *Storage Driver: btrfs'; then
  echo "ERROR: Docker is using btrfs storage driver which is unsupported by kubeadm-dind-cluster" >&2
  echo "Please refer to the documentation for more info." >&2
  echo "Set RUN_ON_BTRFS_ANYWAY to non-empty string to continue anyway." >&2
  exit 1
fi

# In case of linuxkit / moby linux, -v will not work so we can't
# mount /lib/modules and /boot. Also we'll be using localhost
# to access the apiserver.
using_linuxkit=
if ! echo "$docker_info_output"|grep -s '^ *Operating System: .*Docker for Windows' > /dev/null 2>&1 ; then
    if echo "$docker_info_output"|grep -s '^ *Kernel Version: .*-moby$' >/dev/null 2>&1 ||
         echo "$docker_info_output"|grep -s '^ *Kernel Version: .*-linuxkit' > /dev/null 2>&1 ; then
        using_linuxkit=1
    fi
fi

# Determine when using Linux and docker daemon running locally
using_local_linuxdocker=
if [[ $(uname) == Linux && -z ${DOCKER_HOST:-} ]]; then
    using_local_linuxdocker=1
fi

EMBEDDED_CONFIG=y;DOWNLOAD_KUBECTL=y;DIND_K8S_VERSION=v1.12;DIND_IMAGE_DIGEST=sha256:17079e95510a65171b408b1200f4e25fb086e616118bfb0ac3308175afaf357b;DIND_COMMIT=62f5a9277678777b63ae55d144bd2f99feb7c824

# dind::localhost provides the local host IP based on the address family used for service subnet.
function dind::localhost() {
  if [[ ${SERVICE_NET_MODE} = "ipv6" ]]; then
    echo '[::1]'
  else
    echo '127.0.0.1'
  fi
}

# dind::family-for indicates whether the CIDR or IP is for an IPv6 or IPv4 family.
function dind::family-for {
    local addr=$1
    if [[ "$addr" = *":"* ]]; then
        echo "ipv6"
    else
        echo "ipv4"
    fi
}

# dind::cluster-suffix builds a suffix used for resources, based on the DIND_LABEL.
function dind::cluster-suffix {
  if [ "$DIND_LABEL" != "$DEFAULT_DIND_LABEL" ]; then
    echo "-${DIND_LABEL}"
  else
    echo ''
  fi
}

function dind::net-name {
  echo "kubeadm-dind-net$( dind::cluster-suffix )"
}

# dind::add-cluster will inject the cluster ID to the IP address. For IPv4, it is
# assumed that the IP is a /24 with the third part of the address available for cluster ID.
# For IPv6, it is assumed that there is enough space for the cluster to be added, and the
# cluster ID will be added to the 16 bits before the double colon. For example:
#
# 10.192.0.0/24 => 10.192.5.0/24
# fd00:77:20::/64 => fd00:77:20:5::/64
#
# This function is intended to be used for management networks.
#
# TODO: Validate that there is enough space for cluster ID.
# TODO: For IPv6 could get fancy and handle case where cluster ID is placed in upper 8 bits of hextet
# TODO: Consider if want to do /16 for IPv4 management subnet.
#
function dind::add-cluster {
  local cidr=$1
  local ip_mode=$2

  if [[ ${ip_mode} = "ipv4" ]]; then
      echo ${cidr} | sed "s/^\([0-9]*\.[0-9]*\.\).*\/24$/\1${CLUSTER_ID}.0\/24/"
  else  # IPv6
      echo ${cidr} | sed "s/^\(.*\)\(\:\:\/[0-9]*\)$/\1:${CLUSTER_ID}\2/"
  fi
}

# dind::get-and-validate-cidrs takes a list of CIDRs and validates them based on the ip
# mode, returning them. For IPv4 only and IPv6 only modes, only one CIDR is expected. For
# dual stack, two CIDRS are expected. It verifies that the CIDRs are the right family and
# will use the provided defaults, when CIDRs are missing. For dual-stack, the IPv4 address
# will be first.
#
# For the management network, the cluster ID will be injected into the CIDR. Also, if no
# MGMT_CIDRS value is specified, but the legacy DIND_SUBNET/DIND_SUBNET_SIZE is provided,
# that will be used for the (first) CIDR.
#
# NOTE: It is expected that the CIDR size is /24 for IPv4 management networks.
#
# For pod CIDRs, the size will be increased by 8, to leave room for the node ID to be
# injected into the address.
#
# NOTE: For IPv4, the pod size is expected to be /16 -> /24 in usage.
#
function dind::get-and-validate-cidrs {
  IFS=', ' read -r -a cidrs <<< "$1"
  IFS=', ' read -r -a defaults <<< "$2"
  local is_mgmt=$3
  case ${IP_MODE} in
    ipv4)
      case ${#cidrs[@]} in
        0)
          cidrs[0]="${defaults[0]}"
          ;;
        1)
          ;;
        *)
          echo "ERROR! More than one CIDR provided '$1'"
          exit 1
          ;;
      esac
      if [[ $( dind::family-for "${cidrs[0]}" ) != "ipv4" ]]; then
        echo "ERROR! CIDR must be IPv4 value"
        exit 1
      fi
      if [[ ${is_mgmt} = true ]]; then
        cidrs[0]="$( dind::add-cluster "${cidrs[0]}" "${IP_MODE}" )"
      fi
      ;;

    ipv6)
      case ${#cidrs[@]} in
        0)
          cidrs[0]="${defaults[0]}"
          ;;
        1)
          ;;
        *)
          echo "ERROR! More than one CIDR provided '$1'"
          exit 1
          ;;
      esac
      if [[ $( dind::family-for "${cidrs[0]}" ) != "ipv6" ]]; then
        echo "ERROR! CIDR must be IPv6 value"
        exit 1
      fi
      if [[ ${is_mgmt} = true ]]; then
        cidrs[0]="$( dind::add-cluster "${cidrs[0]}" "${IP_MODE}" )"
      fi
      ;;

    dual-stack)
      case ${#cidrs[@]} in
        0)
          cidrs[0]="${defaults[0]}"
          cidrs[1]="${defaults[1]}"
          ;;
        1)
          if [[ $( dind::family-for "${cidrs[0]}" ) = "ipv6" ]]; then
            cidrs[1]=${cidrs[0]}
            cidrs[0]="${defaults[0]}"  # Assuming first default is a V4 address
          else
            cidrs[1]="${defaults[1]}"
          fi
          ;;
        2)
          # Force ordering to have V4 address first
          if [[ $( dind::family-for "${cidrs[0]}" ) = "ipv6" ]]; then
            local temp=${cidrs[0]}
            cidrs[0]=${cidrs[1]}
            cidrs[1]=${temp}
          fi
          ;;
        *)
          echo "ERROR! More than two CIDRs provided '$1'"
          exit 1
          ;;
      esac
      local have_v4=""
      local have_v6=""
      for cidr in ${cidrs[@]}; do
        if [[ $( dind::family-for "${cidr}" ) = "ipv6" ]]; then
          have_v6=1
        else
          have_v4=1
        fi
      done
      if [[ -z ${have_v4} ]]; then
        echo "ERROR! Missing IPv4 CIDR in '$1'"
        exit 1
      fi
      if [[ -z ${have_v6} ]]; then
        echo "ERROR! Missing IPv6 CIDR in '$1'"
        exit 1
      fi
      if [[ ${is_mgmt} = true ]]; then
        cidrs[0]="$( dind::add-cluster "${cidrs[0]}" "${IP_MODE}" )"
        cidrs[1]="$( dind::add-cluster "${cidrs[1]}" "${IP_MODE}" )"
      fi
      ;;
  esac
  echo "${cidrs[@]}"
}

# dind::make-ip-from-cidr  strips off the slash and size, and appends the
# interface part to the prefix to form an IP. For IPv4, it strips off the
# fourth part of the prefix, so that it can be replaced. It assumes that the
# resulting prefix will be of sufficient size. It also will use hex for the
# appended part for IPv6, and decimal for IPv4.
#
# fd00:20::/64 -> fd00:20::a
# 10.96.0.0/12 -> 10.96.0.10
#
function dind::make-ip-from-cidr {
  prefix="$(echo $1 | sed 's,/.*,,')"
  if [[ $( dind::family-for ${prefix} ) == "ipv4" ]]; then
    printf "%s%d" $( echo ${prefix} | sed 's/0$//' ) $2
  else
    printf "%s%x" ${prefix} $2
  fi
}

# dind::add-cluster-id-and-validate-nat64-prefix will modify the IPv4 mapping
# subnet prefix, by adding the cluster ID (default 0) to the second octet.
# It will produce an error, if the prefix is not in the 10.0.0.0/8 or
# 172.16.0.0/12 private networks.
function dind::add-cluster-id-and-validate-nat64-prefix {
  local parts
  IFS="." read -a parts <<<${NAT64_V4_SUBNET_PREFIX}
  if [[ ${#parts[@]} -ne 2 ]]; then
    echo "ERROR! NAT64_V4_SUBNET_PREFIX must be two octets (have '${NAT64_V4_SUBNET_PREFIX}')"
    exit 1
  fi
  (( parts[1]+=${CLUSTER_ID} ))
  NAT64_V4_SUBNET_PREFIX="${parts[0]}.${parts[1]}"
  echo "Added cluster ID offset (${CLUSTER_ID}) to NAT64_V4_SUBNET_PREFIX giving prefix '${NAT64_V4_SUBNET_PREFIX}'"
  if [[ ${parts[0]} -eq 10 ]]; then
    if [[ ${parts[1]} > 253 ]]; then
      echo "ERROR! NAT64_V4_SUBNET_PREFIX is too large for 10.0.0.0/8 private net"
      exit 1
    fi
  elif [[ ${parts[0]} -eq 172 ]]; then
    if [[ ${parts[1]} -lt 16 || ${parts[1]} -gt 31 ]]; then
      echo "ERROR! NAT64_V4_SUBNET_PREFIX is outside of range for 172.16.0.0/12 private net"
      exit 1
    fi
  else
      echo "ERROR! NAT64_V4_SUBNET_PREFIX is not in 10.0.0.0/8 or 172.16.0.0/12 private networks"
      exit 1
  fi
  echo "Using NAT64 V4 mapping network prefix: ${NAT64_V4_SUBNET_PREFIX}"
}

# START OF PROCESSING...

IP_MODE="${IP_MODE:-ipv4}"  # ipv4, ipv6, dual-stack
# FUTURE: Once dual-stack support is released, check K8s version, and reject for older versions.
if [[ ! ${EMBEDDED_CONFIG:-} ]]; then
  source "${DIND_ROOT}/config.sh"
fi

# Multicluster support
# Users can specify a cluster ID number from 1..254, represented as a string.
# This will be used to form resource names "cluster-#", and will be used in the
# management subnet to give unique networks for each cluster. If the cluster ID
# is not specified, or zero, it will be considered a single cluster or the first
# in the multi-cluster. This is the recommended usage.
#
# For legacy support, the user can specify DIND_LABEL, which will be used in the
# resource names. If a cluster ID is specified (a hybrid case, where people are
# using the new method, but want custom names), the resourse name will have the
# suffix "-#" with the cluster ID. If no cluster ID is specified (for backward
# compatibility), then the resource name will be just the DIND_LABEL, and a pseudo-
# random number from 1..13 will be generated for the cluster ID to be used in
# management network. The range is limited, because, in IPv6 mode, the cluster ID
# is used in the NAT64 V4 subnet prefix, which must be in a private network.
# The default is 172.18, so the cluster ID cannot be larger than 13 to guarantee
# a valid value.
#
# To get around that limitation, you can set the cluster ID, in addition to the
# DIND_LABEL, and optionally, change the NAT64_V4_SUBNET_PREFIX value.
#
DEFAULT_DIND_LABEL='mirantis.kubeadm_dind_cluster_runtime'
if [[ -z ${DIND_LABEL+x} ]]; then  # No legacy DIND_LABEL set
  if [[ -z ${CLUSTER_ID+x} ]]; then  # No cluster ID set
    DIND_LABEL=${DEFAULT_DIND_LABEL}  # Single cluster mode
    CLUSTER_ID="0"
  else  # Have cluster ID
    if [[ ${CLUSTER_ID} = "0" ]]; then
      DIND_LABEL=${DEFAULT_DIND_LABEL}  # Single cluster mode or first cluster of multi-cluster
    else
      DIND_LABEL="cluster-${CLUSTER_ID}"  # Multi-cluster
    fi
  fi
else  # Legacy DIND_LABEL set for multi-cluster
  if [[ -z ${CLUSTER_ID+x} ]]; then  # No cluster ID set, make one from 1..13, but don't use in resource names
    CLUSTER_ID="$(( ($RANDOM % 12) + 1 ))"
  else
    if [[ ${CLUSTER_ID} = "0" ]]; then
      CLUSTER_ID="$(( ($RANDOM % 12) + 1 ))"  # Force a pseudo-random cluster for additional legacy cluster
    else
      DIND_LABEL="${DIND_LABEL}-${CLUSTER_ID}"
    fi
  fi
fi

CNI_PLUGIN="${CNI_PLUGIN:-bridge}"
GCE_HOSTED="${GCE_HOSTED:-}"
DIND_ALLOW_AAAA_USE="${DIND_ALLOW_AAAA_USE:-}"  # Default is to use DNS64 always for IPv6 mode
KUBE_ROUTER_VERSION="${KUBE_ROUTER_VERSION:-v0.2.0}"

# Use legacy DIND_SUBNET/DIND_SUBNET_SIZE, only if MGMT_CIDRS is not set.
legacy_mgmt_cidr=""
if [[ ${DIND_SUBNET:-} && ${DIND_SUBNET_SIZE:-} ]]; then
  legacy_mgmt_cidr="${DIND_SUBNET}/${DIND_SUBNET_SIZE}"
fi

if [[ ${IP_MODE} = "dual-stack" ]]; then
  mgmt_net_defaults="10.192.0.0/24, fd00:20::/64"

  KUBE_RSYNC_ADDR="${KUBE_RSYNC_ADDR:-::1}"
  SERVICE_CIDR="${SERVICE_CIDR:-fd00:30::/110}"  # Will default to IPv6 service net family

  pod_net_defaults="10.244.0.0/16, fd00:40::/72"

  USE_HAIRPIN="${USE_HAIRPIN:-true}"  # Default is to use hairpin for dual-stack
  DIND_ALLOW_AAAA_USE=true  # Forced, so can access external hosts via IPv6
  if [[ ${DIND_ALLOW_AAAA_USE} && ${GCE_HOSTED} ]]; then
    echo "ERROR! GCE does not support use of IPv6 for external addresses - aborting."
    exit 1
  fi
elif [[ ${IP_MODE} = "ipv6" ]]; then
  mgmt_net_defaults="fd00:20::/64"

  KUBE_RSYNC_ADDR="${KUBE_RSYNC_ADDR:-::1}"
  SERVICE_CIDR="${SERVICE_CIDR:-fd00:30::/110}"

  pod_net_defaults="fd00:40::/72"

  USE_HAIRPIN="${USE_HAIRPIN:-true}"  # Default is to use hairpin for IPv6
  if [[ ${DIND_ALLOW_AAAA_USE} && ${GCE_HOSTED} ]]; then
    echo "ERROR! GCE does not support use of IPv6 for external addresses - aborting."
    exit 1
  fi
else  # IPv4 mode
  mgmt_net_defaults="10.192.0.0/24"

  KUBE_RSYNC_ADDR="${KUBE_RSYNC_ADDR:-127.0.0.1}"
  SERVICE_CIDR="${SERVICE_CIDR:-10.96.0.0/12}"

  pod_net_defaults="10.244.0.0/16"

  USE_HAIRPIN="${USE_HAIRPIN:-false}"  # Disabled for IPv4, as issue with Virtlet networking
  if [[ ${DIND_ALLOW_AAAA_USE} ]]; then
    echo "WARNING! The DIND_ALLOW_AAAA_USE option is for IPv6 mode - ignoring setting."
    DIND_ALLOW_AAAA_USE=
  fi
  if [[ ${CNI_PLUGIN} = "calico" || ${CNI_PLUGIN} = "calico-kdd" ]]; then
    pod_net_defaults="192.168.0.0/16"
  fi
fi

IFS=' ' read -r -a mgmt_net_cidrs <<<$( dind::get-and-validate-cidrs "${MGMT_CIDRS:-${legacy_mgmt_cidr}}" "${mgmt_net_defaults[@]}" true )

REMOTE_DNS64_V4SERVER="${REMOTE_DNS64_V4SERVER:-8.8.8.8}"
if [[ ${IP_MODE} == "ipv6" ]]; then
  # Uses local DNS64 container
  dns_server="$( dind::make-ip-from-cidr ${mgmt_net_cidrs[0]} 0x100 )"
  DNS64_PREFIX="${DNS64_PREFIX:-fd00:10:64:ff9b::}"
  DNS64_PREFIX_SIZE="${DNS64_PREFIX_SIZE:-96}"
  DNS64_PREFIX_CIDR="${DNS64_PREFIX}/${DNS64_PREFIX_SIZE}"

  LOCAL_NAT64_SERVER="$( dind::make-ip-from-cidr ${mgmt_net_cidrs[0]} 0x200 )"
  NAT64_V4_SUBNET_PREFIX="${NAT64_V4_SUBNET_PREFIX:-172.18}"
  dind::add-cluster-id-and-validate-nat64-prefix
else
  dns_server="${REMOTE_DNS64_V4SERVER}"
fi

SERVICE_NET_MODE="$( dind::family-for ${SERVICE_CIDR} )"
DNS_SVC_IP="$( dind::make-ip-from-cidr ${SERVICE_CIDR} 10 )"

ETCD_HOST="${ETCD_HOST:-$( dind::localhost )}"

IFS=' ' read -r -a pod_net_cidrs <<<$( dind::get-and-validate-cidrs "${POD_NETWORK_CIDR:-}" "${pod_net_defaults[@]}" false )

declare -a pod_prefixes
declare -a pod_sizes
# Extract the prefix and size from the provided pod CIDR(s), based on the IP mode of each. The
# size will be increased by 8, to make room for the node ID to be added to the prefix later.
# Bridge and PTP plugins can process IPv4 and IPv6 pod CIDRs, other plugins must be IPv4 only.
for pod_cidr in "${pod_net_cidrs[@]}"; do
  if [[ $( dind::family-for "${pod_cidr}" ) = "ipv4" ]]; then
    actual_size=$( echo ${pod_cidr} | sed 's,.*/,,' )
    if [[ ${actual_size} -ne 16 ]]; then
        echo "ERROR! For IPv4 CIDRs, the size must be /16. Have '${pod_cidr}'"
        exit 1
    fi
    pod_sizes+=( 24 )
    pod_prefixes+=( "$(echo ${pod_cidr} | sed 's/^\([0-9]*\.[0-9]*\.\).*/\1/')" )
  else  # IPv6
    if [[ ${CNI_PLUGIN} != "bridge" && ${CNI_PLUGIN} != "ptp" ]]; then
      echo "ERROR! IPv6 pod networks are only supported by bridge and PTP CNI plugins"
      exit 1
    fi
    # There are several cases to address. First, is normal split of prefix and size:
    #   fd00:10:20:30::/64  --->  fd00:10:20:30:  /72
    #
    # Second, is when the prefix needs to be padded, so that node ID can be added later:
    #   fd00:10::/64  --->  fd00:10:0:0:  /72
    #
    # Third, is when the low order part of the address, must be removed for the prefix,
    # as the node ID will be placed in the lower byte:
    #   fd00:10:20:30:4000::/72  --->  fd00:10:20:30:40  /80
    #
    # We will attempt to check for three error cases. One is when the address part is
    # way too big for the size specified:
    #   fd00:10:20:30:40::/48  --->  fd00:10:20:  /56 desired, but conflict with 30:40:
    #
    # Another is when the address part, once trimmed for the size, would loose info:
    #   fd00:10:20:1234::/56  --->  fd00:10:20:12  /64, but lost 34:, which conflicts
    #
    # Lastly, again, trimming would leave high byte in hextet, conflicting with
    # the node ID:
    #   fd00:10:20:30:1200::/64  --->  fd00:10:20:30:12  /72, but 12 conflicts
    #
    # Note: later, the node ID will be appended to the prefix generated.
    #
    cluster_size="$(echo ${pod_cidr} | sed 's,.*::/,,')"
    pod_sizes+=( $((${cluster_size}+8)) )

    pod_prefix="$(echo ${pod_cidr} | sed 's,::/.*,:,')"
    num_colons="$(grep -o ":" <<< "${pod_prefix}" | wc -l)"
    need_zero_pads=$((${cluster_size}/16))

    if [[ ${num_colons} -gt $((need_zero_pads + 1)) ]]; then
        echo "ERROR! Address part of CIDR (${pod_prefix}) is too large for /${cluster_size}"
        exit 1
    fi
    if [[ ${num_colons} -gt ${need_zero_pads} ]]; then
      # Will be replacing lowest byte with node ID, so pull off lower byte and colon
        if [[ ${pod_prefix: -3} != "00:" ]]; then   # last byte is not zero
          echo "ERROR! Cannot trim address part of CIDR (${pod_prefix}) to fit in /${cluster_size}"
          exit 1
        fi
        pod_prefix=${pod_prefix::-3}
        if [[ $(( ${cluster_size} % 16 )) -eq 0 && $( ${pod_prefix: -1} ) != ":" ]]; then  # should not be upper byte for this size CIDR
          echo "ERROR! Trimmed address part of CIDR (${pod_prefix}) is still too large for /${cluster_size}"
          exit 1
        fi
    fi
    # Add in zeros to pad 16 bits at a time, up to the padding needed, which is
    # need_zero_pads - num_colons.
    while [ ${num_colons} -lt ${need_zero_pads} ]; do
        pod_prefix+="0:"
      ((num_colons++))
    done
    pod_prefixes+=( "${pod_prefix}" )
  fi
done

DIND_IMAGE_BASE="${DIND_IMAGE_BASE:-mirantis/kubeadm-dind-cluster}"
if [[ ${DIND_COMMIT:-} ]]; then
  if [[ ${DIND_COMMIT} = current ]]; then
    DIND_COMMIT="$(cd "${DIND_ROOT}"; git rev-parse HEAD)"
  fi
  DIND_K8S_VERSION="${DIND_K8S_VERSION:-v1.13}"
  DIND_IMAGE="${DIND_IMAGE_BASE}:${DIND_COMMIT}-${DIND_K8S_VERSION}"
else
  DIND_IMAGE="${DIND_IMAGE:-${DIND_IMAGE_BASE}:local}"
fi
if [[ ${DIND_IMAGE_DIGEST:-} ]]; then
  DIND_IMAGE="${DIND_IMAGE}@${DIND_IMAGE_DIGEST}"
fi

BUILD_KUBEADM="${BUILD_KUBEADM:-}"
BUILD_HYPERKUBE="${BUILD_HYPERKUBE:-}"
if [[ ! -z ${DIND_K8S_BIN_DIR:-} ]]; then
  BUILD_KUBEADM=""
  BUILD_HYPERKUBE=""
fi
KUBEADM_SOURCE="${KUBEADM_SOURCE-}"
HYPERKUBE_SOURCE="${HYPERKUBE_SOURCE-}"
NUM_NODES=${NUM_NODES:-2}
EXTRA_PORTS="${EXTRA_PORTS:-}"
KUBECTL_DIR="${KUBECTL_DIR:-${HOME}/.kubeadm-dind-cluster}"
SKIP_SNAPSHOT="${SKIP_SNAPSHOT:-}"
E2E_REPORT_DIR="${E2E_REPORT_DIR:-}"
DIND_NO_PARALLEL_E2E="${DIND_NO_PARALLEL_E2E:-}"
DNS_SERVICE="${DNS_SERVICE:-coredns}"
DIND_STORAGE_DRIVER="${DIND_STORAGE_DRIVER:-overlay2}"

DIND_CA_CERT_URL="${DIND_CA_CERT_URL:-}"
DIND_PROPAGATE_HTTP_PROXY="${DIND_PROPAGATE_HTTP_PROXY:-}"
DIND_HTTP_PROXY="${DIND_HTTP_PROXY:-}"
DIND_HTTPS_PROXY="${DIND_HTTPS_PROXY:-}"
DIND_NO_PROXY="${DIND_NO_PROXY:-}"

DIND_DAEMON_JSON_FILE="${DIND_DAEMON_JSON_FILE:-/etc/docker/daemon.json}"  # can be set to /dev/null
DIND_REGISTRY_MIRROR="${DIND_REGISTRY_MIRROR:-}"  # plain string format
DIND_INSECURE_REGISTRIES="${DIND_INSECURE_REGISTRIES:-}"  # json list format
# comma-separated custom network(s) for cluster nodes to join
DIND_CUSTOM_NETWORKS="${DIND_CUSTOM_NETWORKS:-}"

SKIP_DASHBOARD="${SKIP_DASHBOARD:-}"

# you can set special value 'none' not to set any FEATURE_GATES / KUBELET_FEATURE_GATES.
FEATURE_GATES="${FEATURE_GATES:-none}"
KUBELET_FEATURE_GATES="${KUBELET_FEATURE_GATES:-DynamicKubeletConfig=true}"

ENABLE_CEPH="${ENABLE_CEPH:-}"

DIND_CRI="${DIND_CRI:-docker}"
case "${DIND_CRI}" in
  docker)
    CRI_SOCKET=/var/run/dockershim.sock
    ;;
  containerd)
    CRI_SOCKET=/var/run/containerd/containerd.sock
    ;;
  *)
    echo >&2 "Bad DIND_CRI. Please specify 'docker' or 'containerd'"
    ;;
esac

# TODO: Test multi-cluster for IPv6, before enabling
if [[ "${DIND_LABEL}" != "${DEFAULT_DIND_LABEL}"  && "${IP_MODE}" == 'dual-stack' ]]; then
    echo "Multiple parallel clusters currently not supported for dual-stack mode" >&2
    exit 1
fi

# not configurable for now, would need to setup context for kubectl _inside_ the cluster
readonly INTERNAL_APISERVER_PORT=8080

function dind::need-source {
  if [[ ! -f cluster/kubectl.sh ]]; then
    echo "$0 must be called from the Kubernetes repository root directory" 1>&2
    exit 1
  fi
}

build_tools_dir="build"
use_k8s_source=y
if [[ ! ${BUILD_KUBEADM} && ! ${BUILD_HYPERKUBE} ]]; then
  use_k8s_source=
fi
if [[ ${use_k8s_source} ]]; then
  dind::need-source
  kubectl=cluster/kubectl.sh
  if [[ ! -f ${build_tools_dir}/common.sh ]]; then
    build_tools_dir="build-tools"
  fi
else
  if [[ ! ${DOWNLOAD_KUBECTL:-} ]] && ! hash kubectl 2>/dev/null; then
    echo "You need kubectl binary in your PATH to use prebuilt DIND image" 1>&2
    exit 1
  fi
  kubectl=kubectl
fi

function dind::retry {
  # based on retry function in hack/jenkins/ scripts in k8s source
  for i in {1..10}; do
    "$@" && return 0 || sleep ${i}
  done
  "$@"
}

busybox_image="busybox:1.30.1"
e2e_base_image="golang:1.12.4"
sys_volume_args=()
build_volume_args=()

function dind::set-build-volume-args {
  if [ ${#build_volume_args[@]} -gt 0 ]; then
    return 0
  fi
  build_container_name=
  if [ -n "${KUBEADM_DIND_LOCAL:-}" ]; then
    build_volume_args=(-v "$PWD:/go/src/k8s.io/kubernetes")
  else
    build_container_name="$(KUBE_ROOT=${PWD} ETCD_HOST=${ETCD_HOST} &&
                            . ${build_tools_dir}/common.sh &&
                            kube::build::verify_prereqs >&2 &&
                            echo "${KUBE_DATA_CONTAINER_NAME:-${KUBE_BUILD_DATA_CONTAINER_NAME}}")"
    build_volume_args=(--volumes-from "${build_container_name}")
  fi
}

function dind::volume-exists {
  local name="$1"
  if docker volume inspect "${name}" >& /dev/null; then
    return 0
  fi
  return 1
}

function dind::create-volume {
  local name="$1"
  docker volume create --label "${DIND_LABEL}" --name "${name}" >/dev/null
}

# We mount /boot and /lib/modules into the container
# below to in case some of the workloads need them.
# This includes virtlet, for instance. Also this may be
# useful in future if we want DIND nodes to pass
# preflight checks.
# Unfortunately we can't do this when using Mac Docker
# (unless a remote docker daemon on Linux is used)
# NB: there's no /boot on recent Mac dockers
function dind::prepare-sys-mounts {
  if [[ ! ${using_linuxkit} ]]; then
    sys_volume_args=()
    if [[ -d /boot ]]; then
      sys_volume_args+=(-v /boot:/boot)
    fi
    if [[ -d /lib/modules ]]; then
      sys_volume_args+=(-v /lib/modules:/lib/modules)
    fi
    return 0
  fi
  local dind_sys_vol_name
  dind_sys_vol_name="kubeadm-dind-sys$( dind::cluster-suffix )"
  if ! dind::volume-exists "$dind_sys_vol_name"; then
    dind::step "Saving a copy of docker host's /lib/modules"
    dind::create-volume "$dind_sys_vol_name"
    # Use a dirty nsenter trick to fool Docker on Mac and grab system
    # /lib/modules into sys.tar file on kubeadm-dind-sys volume.
    local nsenter="nsenter --mount=/proc/1/ns/mnt --"
    docker run \
           --rm \
           --privileged \
           -v "$dind_sys_vol_name":/dest \
           --pid=host \
           "${busybox_image}" \
           /bin/sh -c \
           "if ${nsenter} test -d /lib/modules; then ${nsenter} tar -C / -c lib/modules >/dest/sys.tar; fi"
  fi
  sys_volume_args=(-v "$dind_sys_vol_name":/dind-sys)
}

tmp_containers=()

function dind::cleanup {
  if [ ${#tmp_containers[@]} -gt 0 ]; then
    for name in "${tmp_containers[@]}"; do
      docker rm -vf "${name}" 2>/dev/null
    done
  fi
}

trap dind::cleanup EXIT

function dind::check-image {
  local name="$1"
  if docker inspect --format 'x' "${name}" >&/dev/null; then
    return 0
  else
    return 1
  fi
}

function dind::filter-make-output {
  # these messages make output too long and make Travis CI choke
  egrep -v --line-buffered 'I[0-9][0-9][0-9][0-9] .*(parse|conversion|defaulter|deepcopy)\.go:[0-9]+\]'
}

function dind::run-build-command {
    # this is like build/run.sh, but it doesn't rsync back the binaries,
    # only the generated files.
    local cmd=("$@")
    (
        # The following is taken from build/run.sh and build/common.sh
        # of Kubernetes source tree. It differs in
        # --filter='+ /_output/dockerized/bin/**'
        # being removed from rsync
        . ${build_tools_dir}/common.sh
        kube::build::verify_prereqs
        kube::build::build_image
        kube::build::run_build_command "$@"

        kube::log::status "Syncing out of container"

        kube::build::start_rsyncd_container

        local rsync_extra=""
        if (( ${KUBE_VERBOSE} >= 6 )); then
            rsync_extra="-iv"
        fi

        # The filter syntax for rsync is a little obscure. It filters on files and
        # directories.  If you don't go in to a directory you won't find any files
        # there.  Rules are evaluated in order.  The last two rules are a little
        # magic. '+ */' says to go in to every directory and '- /**' says to ignore
        # any file or directory that isn't already specifically allowed.
        #
        # We are looking to copy out all of the built binaries along with various
        # generated files.
        kube::build::rsync \
            --filter='- /vendor/' \
            --filter='- /_temp/' \
            --filter='+ zz_generated.*' \
            --filter='+ generated.proto' \
            --filter='+ *.pb.go' \
            --filter='+ types.go' \
            --filter='+ */' \
            --filter='- /**' \
            "rsync://k8s@${KUBE_RSYNC_ADDR}/k8s/" "${KUBE_ROOT}"

        kube::build::stop_rsyncd_container
    )
}

function dind::make-for-linux {
  local copy="$1"
  shift
  dind::step "Building binaries:" "$*"
  if [ -n "${KUBEADM_DIND_LOCAL:-}" ]; then
    dind::step "+ make WHAT=\"$*\""
    make WHAT="$*" 2>&1 | dind::filter-make-output
  elif [ "${copy}" = "y" ]; then
    dind::step "+ ${build_tools_dir}/run.sh make WHAT=\"$*\""
    "${build_tools_dir}/run.sh" make WHAT="$*" 2>&1 | dind::filter-make-output
  else
    dind::step "+ [using the build container] make WHAT=\"$*\""
    dind::run-build-command make WHAT="$*" 2>&1 | dind::filter-make-output
  fi
}

function dind::check-binary {
  local filename="$1"
  local dockerized="_output/dockerized/bin/linux/amd64/${filename}"
  local plain="_output/local/bin/linux/amd64/${filename}"
  dind::set-build-volume-args
  # FIXME: don't hardcode amd64 arch
  if [ -n "${KUBEADM_DIND_LOCAL:-${force_local:-}}" ]; then
    if [ -f "${dockerized}" -o -f "${plain}" ]; then
      return 0
    fi
  elif docker run --rm "${build_volume_args[@]}" \
              "${busybox_image}" \
              test -f "/go/src/k8s.io/kubernetes/${dockerized}" >&/dev/null; then
    return 0
  fi
  return 1
}

function dind::ensure-downloaded-kubectl {
  local kubectl_url
  local kubectl_sha1
  local kubectl_sha1_linux
  local kubectl_sha1_darwin
  local kubectl_link
  local kubectl_os

  if [[ ! ${DOWNLOAD_KUBECTL:-} ]]; then
    return 0
  fi

  export PATH="${KUBECTL_DIR}:$PATH"

  eval "$(docker run --entrypoint /bin/bash --rm "${DIND_IMAGE}" -c "cat /dind-env")"

  if [ $(uname) = Darwin ]; then
    kubectl_sha1="${KUBECTL_DARWIN_SHA1}"
    kubectl_url="${KUBECTL_DARWIN_URL}"
  else
    kubectl_sha1="${KUBECTL_LINUX_SHA1}"
    kubectl_url="${KUBECTL_LINUX_URL}"
  fi
  local link_target="kubectl-${KUBECTL_VERSION}"
  local link_name="${KUBECTL_DIR}"/kubectl
  if [[ -h "${link_name}" && "$(readlink "${link_name}")" = "${link_target}" ]]; then
    return 0
  fi

  local path="${KUBECTL_DIR}/${link_target}"
  if [[ ! -f "${path}" ]]; then
    mkdir -p "${KUBECTL_DIR}"
    curl -sSLo "${path}" "${kubectl_url}"
    echo "${kubectl_sha1}  ${path}" | sha1sum -c
    chmod +x "${path}"
  fi

  ln -fs "${link_target}" "${KUBECTL_DIR}/kubectl"
}

function dind::ensure-kubectl {
  if [[ ! ${use_k8s_source} ]]; then
    # already checked on startup
    dind::ensure-downloaded-kubectl
    return 0
  fi
  if [ $(uname) = Darwin ]; then
    dind::step "Building kubectl"
    dind::step "+ make WHAT=cmd/kubectl"
    make WHAT=cmd/kubectl 2>&1 | dind::filter-make-output
  else
    dind::make-for-linux y cmd/kubectl
  fi
}

function dind::ensure-binaries {
  local -a to_build=()
  for name in "$@"; do
    if ! dind::check-binary "$(basename "${name}")"; then
      to_build+=("${name}")
    fi
  done
  if [ "${#to_build[@]}" -gt 0 ]; then
    dind::make-for-linux n "${to_build[@]}"
  fi
  return 0
}

# dind::ensure-network creates the management network for the cluster. For IPv4
# only it will have the management network CIDR. For IPv6 only, it will have
# the IPv6 management network CIDR and the NAT64 V4 mapping network CIDR. For
# dual stack, it will have the IPv4 and IPv6 management CIDRs. Each of the
# management networks (not the NAT64 network) will have a gateway specified.
#
function dind::ensure-network {
  if ! docker network inspect $(dind::net-name) >&/dev/null; then
    local -a args
    for cidr in "${mgmt_net_cidrs[@]}"; do
      if [[ $( dind::family-for ${cidr} ) = "ipv6" ]]; then
                args+=(--ipv6)
      fi
      args+=(--subnet="${cidr}")
      local gw=$( dind::make-ip-from-cidr ${cidr} 1 )
      args+=(--gateway="${gw}")
    done
    if [[ ${IP_MODE} = "ipv6" ]]; then
      # Need second network for NAT64 V4 mapping network
      args+=(--subnet=${NAT64_V4_SUBNET_PREFIX}.0.0/16)
    fi
    docker network create ${args[@]} $(dind::net-name) >/dev/null
  fi
}

function dind::ensure-volume {
  local reuse_volume=
  if [[ $1 = -r ]]; then
    reuse_volume=1
    shift
  fi
  local name="$1"
  if dind::volume-exists "${name}"; then
    if [[ ! ${reuse_volume} ]]; then
      docker volume rm "${name}" >/dev/null
    fi
  fi
  dind::create-volume "${name}"
}

function dind::ensure-dns {
    if [[ ${IP_MODE} = "ipv6" ]]; then
        local dns64_name="bind9$( dind::cluster-suffix )"
        if ! docker inspect ${dns64_name} >&/dev/null; then
            local force_dns64_for=""
            if [[ ! ${DIND_ALLOW_AAAA_USE} ]]; then
                # Normally, if have an AAAA record, it is used. This clause tells
                # bind9 to do ignore AAAA records for the specified networks
                # and/or addresses and lookup A records and synthesize new AAAA
                # records. In this case, we select "any" networks that have AAAA
                # records meaning we ALWAYS use A records and do NAT64.
                force_dns64_for="exclude { any; };"
            fi
            read -r -d '' bind9_conf <<BIND9_EOF
options {
    directory "/var/bind";
    allow-query { any; };
    forwarders {
        ${DNS64_PREFIX}${REMOTE_DNS64_V4SERVER};
    };
    auth-nxdomain no;    # conform to RFC1035
    listen-on-v6 { any; };
    dns64 ${DNS64_PREFIX_CIDR} {
        ${force_dns64_for}
    };
};
BIND9_EOF
            docker run -d --name ${dns64_name} --hostname ${dns64_name} --net "$(dind::net-name)" --label "dind-support$( dind::cluster-suffix )" \
               --sysctl net.ipv6.conf.all.disable_ipv6=0 --sysctl net.ipv6.conf.all.forwarding=1 \
               --privileged=true --ip6 ${dns_server} --dns ${dns_server} \
               -e bind9_conf="${bind9_conf}" \
               diverdane/bind9:latest /bin/sh -c 'echo "${bind9_conf}" >/named.conf && named -c /named.conf -g -u named' >/dev/null
            ipv4_addr="$(docker exec ${dns64_name} ip addr list eth0 | grep "inet" | awk '$1 == "inet" {print $2}')"
            docker exec ${dns64_name} ip addr del ${ipv4_addr} dev eth0
            docker exec ${dns64_name} ip -6 route add ${DNS64_PREFIX_CIDR} via ${LOCAL_NAT64_SERVER}
        fi
    fi
}

function dind::ensure-nat {
    if [[  ${IP_MODE} = "ipv6" ]]; then
        local nat64_name="tayga$( dind::cluster-suffix )"
        if ! docker ps | grep ${nat64_name} >&/dev/null; then
            docker run -d --name ${nat64_name} --hostname ${nat64_name} --net "$(dind::net-name)" --label "dind-support$( dind::cluster-suffix )" \
                   --sysctl net.ipv6.conf.all.disable_ipv6=0 --sysctl net.ipv6.conf.all.forwarding=1 \
                   --privileged=true --ip ${NAT64_V4_SUBNET_PREFIX}.0.200 --ip6 ${LOCAL_NAT64_SERVER} --dns ${REMOTE_DNS64_V4SERVER} --dns ${dns_server} \
                   -e TAYGA_CONF_PREFIX=${DNS64_PREFIX_CIDR} -e TAYGA_CONF_IPV4_ADDR=${NAT64_V4_SUBNET_PREFIX}.0.200 \
                   -e TAYGA_CONF_DYNAMIC_POOL=${NAT64_V4_SUBNET_PREFIX}.0.128/25 danehans/tayga:latest >/dev/null
            # Need to check/create, as "clean" may remove route
            local route="$(ip route | egrep "^${NAT64_V4_SUBNET_PREFIX}.0.128/25")"
            if [[ -z "${route}" ]]; then
                docker run --net=host --rm --privileged ${busybox_image} ip route add ${NAT64_V4_SUBNET_PREFIX}.0.128/25 via ${NAT64_V4_SUBNET_PREFIX}.0.200
            fi
        fi
    fi
}

function dind::run {
  local reuse_volume=
  if [[ $1 = -r ]]; then
    reuse_volume="-r"
    shift
  fi
  local container_name="${1:-}"
  local node_id=${2:-0}
  local portforward="${3:-}"
  if [[ $# -gt 3 ]]; then
    shift 3
  else
    shift $#
  fi

  local -a opts=("$@")
  local ip_mode="--ip"
  for cidr in "${mgmt_net_cidrs[@]}"; do
    if [[ $( dind::family-for ${cidr} ) = "ipv6" ]]; then
      ip_mode="--ip6"
    fi
    opts+=("${ip_mode}" "$( dind::make-ip-from-cidr ${cidr} $((${node_id}+1)) )")
  done
  opts+=("$@")

  local -a args=("systemd.setenv=CNI_PLUGIN=${CNI_PLUGIN}")
  args+=("systemd.setenv=IP_MODE=${IP_MODE}")
  args+=("systemd.setenv=DIND_STORAGE_DRIVER=${DIND_STORAGE_DRIVER}")
  args+=("systemd.setenv=DIND_CRI=${DIND_CRI}")

  if [[ ${IP_MODE} != "ipv4" ]]; then
    opts+=(--sysctl net.ipv6.conf.all.disable_ipv6=0)
    opts+=(--sysctl net.ipv6.conf.all.forwarding=1)
  fi

  if [[ ${IP_MODE} = "ipv6" ]]; then
    opts+=(--dns ${dns_server})
    args+=("systemd.setenv=DNS64_PREFIX_CIDR=${DNS64_PREFIX_CIDR}")
    args+=("systemd.setenv=LOCAL_NAT64_SERVER=${LOCAL_NAT64_SERVER}")
  fi

  declare -a pod_nets
  local i=0
  if [[ ${IP_MODE} = "ipv4" || ${IP_MODE} = "dual-stack" ]]; then
    pod_nets+=("${pod_prefixes[$i]}${node_id}")
    i=$((i+1))
  fi
  if [[ ${IP_MODE} = "ipv6" || ${IP_MODE} = "dual-stack" ]]; then
    # For prefix, if node ID will be in the upper byte, push it over
    if [[ $((${pod_sizes[$i]} % 16)) -ne 0 ]]; then
      n_id=$(printf "%02x00\n" "${node_id}")
    else
      if [[ "${pod_prefixes[$i]: -1}" = ":" ]]; then
        n_id=$(printf "%x\n" "${node_id}")
      else
        n_id=$(printf "%02x\n" "${node_id}")  # In lower byte, so ensure two chars
      fi
    fi
    pod_nets+=("${pod_prefixes[$i]}${n_id}")
  fi

  args+=("systemd.setenv=POD_NET_PREFIX=\"${pod_nets[0]}\"")
  args+=("systemd.setenv=POD_NET_SIZE=\"${pod_sizes[0]}\"")
  args+=("systemd.setenv=POD_NET2_PREFIX=\"${pod_nets[1]:-}\"")
  args+=("systemd.setenv=POD_NET2_SIZE=\"${pod_sizes[1]:-}\"")
  args+=("systemd.setenv=SERVICE_NET_MODE=${SERVICE_NET_MODE}")
  args+=("systemd.setenv=USE_HAIRPIN=${USE_HAIRPIN}")
  args+=("systemd.setenv=DNS_SVC_IP=${DNS_SVC_IP}")
  args+=("systemd.setenv=DNS_SERVICE=${DNS_SERVICE}")
  if [[ ! "${container_name}" ]]; then
    echo >&2 "Must specify container name"
    exit 1
  fi

  # remove any previously created containers with the same name
  docker rm -vf "${container_name}" >&/dev/null || true

  if [[ "${portforward}" ]]; then
    IFS=';' read -ra array <<< "${portforward}"
    for element in "${array[@]}"; do
      opts+=(-p "${element}")
    done
  fi

  opts+=(${sys_volume_args[@]+"${sys_volume_args[@]}"})

  dind::step "Starting DIND container:" "${container_name}"

  if [[ ! -z ${DIND_K8S_BIN_DIR:-} ]]; then
      opts+=(-v ${DIND_K8S_BIN_DIR}:/k8s)
  fi
  if [[ ! ${using_linuxkit} ]]; then
    opts+=(-v /boot:/boot -v /lib/modules:/lib/modules)
  fi

  if [[ ${ENABLE_CEPH} ]]; then
    opts+=(-v /dev:/dev
           -v /sys/bus:/sys/bus
           -v /var/run/docker.sock:/opt/outer-docker.sock)
  fi

  local volume_name="kubeadm-dind-${container_name}"
  dind::ensure-network
  dind::ensure-volume ${reuse_volume} "${volume_name}"
  dind::ensure-nat
  dind::ensure-dns

  # TODO: create named volume for binaries and mount it to /k8s
  # in case of the source build

  # Start the new container.
  docker run \
         -e IP_MODE="${IP_MODE}" \
         -e KUBEADM_SOURCE="${KUBEADM_SOURCE}" \
         -e HYPERKUBE_SOURCE="${HYPERKUBE_SOURCE}" \
         -d --privileged \
         --net "$(dind::net-name)" \
         --name "${container_name}" \
         --hostname "${container_name}" \
         -l "${DIND_LABEL}" \
         -v "${volume_name}:/dind" \
         ${opts[@]+"${opts[@]}"} \
         "${DIND_IMAGE}" \
         ${args[@]+"${args[@]}"}

  if [[ -n ${DIND_CUSTOM_NETWORKS} ]]; then
    local cust_nets
    local IFS=','; read -ra cust_nets <<< "${DIND_CUSTOM_NETWORKS}"
    for cust_net in "${cust_nets[@]}"; do
      docker network connect ${cust_net} ${container_name} >/dev/null
    done
  fi
}

function dind::kubeadm {
  local container_id="$1"
  shift
  dind::step "Running kubeadm:" "$*"
  status=0
  # See image/bare/wrapkubeadm.
  # Capturing output is necessary to grab flags for 'kubeadm join'
  local -a env=(-e KUBELET_FEATURE_GATES="${KUBELET_FEATURE_GATES}"
                -e DIND_CRI="${DIND_CRI}")
  if ! docker exec "${env[@]}" "${container_id}" /usr/local/bin/wrapkubeadm "$@" 2>&1 | tee /dev/fd/2; then
    echo "*** kubeadm failed" >&2
    return 1
  fi
  return ${status}
}

# function dind::bare {
#   local container_name="${1:-}"
#   if [[ ! "${container_name}" ]]; then
#     echo >&2 "Must specify container name"
#     exit 1
#   fi
#   shift
#   run_opts=(${@+"$@"})
#   dind::run "${container_name}"
# }

function dind::configure-kubectl {
  dind::step "Setting cluster config"
  local host="$(dind::localhost)"
  if [[ -z "$using_local_linuxdocker" ]]; then
    host="127.0.0.1"
  fi
  local context_name cluster_name
  context_name="$(dind::context-name)"
  cluster_name="$(dind::context-name)"
  "${kubectl}" config set-cluster "$cluster_name" \
    --server="http://${host}:$(dind::apiserver-port)" \
    --insecure-skip-tls-verify=true
  "${kubectl}" config set-context "$context_name" --cluster="$cluster_name"
  if [[ ${DIND_LABEL} = "${DEFAULT_DIND_LABEL}" ]]; then
      # Single cluster mode
      "${kubectl}" config use-context "$context_name"
  fi
}

force_make_binaries=
function dind::set-master-opts {
  master_opts=()
  if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then
    # share binaries pulled from the build container between nodes
    local dind_k8s_bin_vol_name
    dind_k8s_bin_vol_name="dind-k8s-binaries$(dind::cluster-suffix)"
    dind::ensure-volume -r "${dind_k8s_bin_vol_name}"
    dind::set-build-volume-args
    master_opts+=("${build_volume_args[@]}" -v "${dind_k8s_bin_vol_name}:/k8s")
    local -a bins
    if [[ ${BUILD_KUBEADM} ]]; then
      master_opts+=(-e KUBEADM_SOURCE=build://)
      bins+=(cmd/kubeadm)
    else
      master_opts+=(-e ${KUBEADM_SOURCE})
    fi
    if [[ ${BUILD_HYPERKUBE} ]]; then
      master_opts+=(-e HYPERKUBE_SOURCE=build://)
      bins+=(cmd/hyperkube)
    fi
    if [[ ${force_make_binaries} ]]; then
      dind::make-for-linux n "${bins[@]}"
    else
      dind::ensure-binaries "${bins[@]}"
    fi
  fi
  if [[ ${MASTER_EXTRA_OPTS:-} ]]; then
    master_opts+=( ${MASTER_EXTRA_OPTS} )
  fi
}

function dind::ensure-dashboard-clusterrolebinding {
  local ctx
  ctx="$(dind::context-name)"
  # 'create' may cause etcd timeout, yet create the clusterrolebinding.
  # So use 'apply' to actually create it
  "${kubectl}" --context "$ctx" create clusterrolebinding add-on-cluster-admin \
               --clusterrole=cluster-admin \
               --serviceaccount=kube-system:default \
               -o json --dry-run |
    docker exec -i "$(dind::master-name)" jq '.apiVersion="rbac.authorization.k8s.io/v1beta1"|.kind|="ClusterRoleBinding"' |
    "${kubectl}" --context "$ctx" apply -f -
}

function dind::deploy-dashboard {
  local url="${DASHBOARD_URL:-}"
  if [ ! "$url" ]; then
    local cmp_api_to_1_15=0
    dind::compare-versions 'kubeapi' "$(dind::kubeapi-version)" 1 15 || cmp_api_to_1_15=$?
    if [[ $cmp_api_to_1_15 == 2 ]]; then
      # API version < 1.15
      url='https://rawgit.com/kubernetes/dashboard/bfab10151f012d1acc5dfb1979f3172e2400aa3c/src/deploy/kubernetes-dashboard.yaml'
    else
      # API version >= 1.15
      url='https://rawgit.com/kubernetes/dashboard/v1.10.1/src/deploy/recommended/kubernetes-dashboard.yaml'
    fi
  fi

  dind::step "Deploying k8s dashboard from $url"
  dind::retry "${kubectl}" --context "$(dind::context-name)" apply -f "$url"
  # https://kubernetes-io-vnext-staging.netlify.com/docs/admin/authorization/rbac/#service-account-permissions
  # Thanks @liggitt for the hint
  dind::retry dind::ensure-dashboard-clusterrolebinding
}

function dind::version-from-source {
  (cluster/kubectl.sh version --short 2>/dev/null || true) |
    grep Client |
    sed 's/^.*: v\([0-9.]*\).*/\1/'
}

function dind::kubeapi-version {
  if [[ ${use_k8s_source} ]]; then
    dind::version-from-source
  else
    docker exec "$(dind::master-name)" \
           /bin/bash -c 'kubectl version -o json | jq -r .serverVersion.gitVersion | sed "s/^v\([0-9.]*\).*/\1/"'
  fi
}

function dind::kubeadm-version {
  if [[ ${use_k8s_source} ]]; then
    dind::version-from-source
  else
    docker exec "$(dind::master-name)" \
           /bin/bash -c 'kubeadm version -o json | jq -r .clientVersion.gitVersion' |
      sed 's/^v\([0-9.]*\).*/\1/'
  fi
}

function dind::kubelet-version {
  if [[ ${use_k8s_source} ]]; then
    dind::version-from-source
  else
    docker exec "$(dind::master-name)" \
           /bin/bash -c 'kubelet --version | sed -E "s/^(kubernetes )?v?([0-9]+(\.[0-9]+){1,2})/\2/I"'
  fi
}

# $1 is the name of the software whose version is being compared (eg 'kubeadm')
# $2 is the version as a string (eg '1.14.5')
# $3 is the major version it is being compared to
# $4 is the minor version it is being compared to
# returns 0 if the 2 versions are equal
# returns 1 if the string version is greater
# returns 2 if the other version is greater
# any other return code is an error
function dind::compare-versions {
  local name="$1"
  local version_str="$2"
  local cmp_to_major="$3"
  local cmp_to_minor="$4"

  if [[ ! "$version_str" =~ ^([0-9]+)\.([0-9]+) ]]; then
    echo >&2 "WARNING: can't parse $name version: $version_str"
    return 3
  fi
  local major="${BASH_REMATCH[1]}"
  local minor="${BASH_REMATCH[2]}"
  if [[ $major -gt $cmp_to_major ]]; then
    return 1
  fi
  if [[ $major -lt $cmp_to_major ]]; then
    return 2
  fi
  if [[ $minor -gt $cmp_to_minor ]]; then
    return 1
  fi
  if [[ $minor -lt $cmp_to_minor ]]; then
    return 2
  fi
  return 0
}

function dind::kubeadm-version-at-least {
  local major="${1}"
  local minor="${2}"

  local cmp=0
  dind::compare-versions 'kubeadm' "$(dind::kubeadm-version)" "$major" "$minor" || cmp=$?

  [[ $cmp -lt 2 ]]
}

function dind::verify-image-compatibility {
  # We can't tell in advance, if the image selected supports dual-stack,
  # but will do the next best thing, and check as soon as start up kube-master
  local master_name=$1
  if [[ ${IP_MODE} = "dual-stack" ]]; then
    local dual_stack_support="$(docker exec ${master_name} cat /node-info 2>/dev/null | grep "dual-stack-support" | wc -l)"
    if [[ ${dual_stack_support} -eq 0 ]]; then
      echo "ERROR! DinD image (${DIND_IMAGE}) does not support dual-stack mode - aborting!"
      dind::remove-images "${DIND_LABEL}"
      exit 1
    fi
  fi
}

function dind::check-dns-service-type {
  if [[ ${DNS_SERVICE} = "kube-dns" ]] && dind::kubeadm-version-at-least 1 13; then
    echo >&2 "WARNING: for 1.13+, only coredns can be used as the DNS service"
    DNS_SERVICE="coredns"
  fi
}

function dind::set-version-specific-flags {
  local kubelet_version_specific_flags="$1"
  docker exec "$(dind::master-name)" sed -i "s@KUBELET_VERSION_SPECIFIC_FLAGS=[^\"]*\"@KUBELET_VERSION_SPECIFIC_FLAGS=$kubelet_version_specific_flags\"@" /lib/systemd/system/kubelet.service
}

function dind::init {
  local -a opts
  dind::set-master-opts
  local local_host master_name container_id
  master_name="$(dind::master-name)"
  local_host="$( dind::localhost )"
  container_id=$(dind::run "${master_name}" 1 "${local_host}:$(dind::apiserver-port):${INTERNAL_APISERVER_PORT}" ${master_opts[@]+"${master_opts[@]}"})

  dind::verify-image-compatibility ${master_name}

  # FIXME: I tried using custom tokens with 'kubeadm ex token create' but join failed with:
  # 'failed to parse response as JWS object [square/go-jose: compact JWS format must have three parts]'
  # So we just pick the line from 'kubeadm init' output
  # Using a template file in the image (based on version) to build a kubeadm.conf file and to customize
  # it based on CNI plugin, IP mode, and environment settings. User can add additional
  # customizations to template and then rebuild the image used (build/build-local.sh).
  local pod_subnet_disable="# "
  # TODO: May want to specify each of the plugins that require --pod-network-cidr
  if [[ ${CNI_PLUGIN} != "bridge" && ${CNI_PLUGIN} != "ptp" ]]; then
    pod_subnet_disable=""
  fi
  local bind_address="0.0.0.0"
  if [[ ${SERVICE_NET_MODE} = "ipv6" ]]; then
    bind_address="::"
  fi
  dind::proxy "$master_name"
  dind::custom-docker-opts "$master_name"

  # HACK: Indicating mode, so that wrapkubeadm will not set a cluster CIDR for kube-proxy
  # in IPv6 (only) mode.
  if [[ ${SERVICE_NET_MODE} = "ipv6" ]]; then
    docker exec --privileged -i "$master_name" touch /v6-mode
  fi

  feature_gates="{CoreDNS: false}"
  if [[ ${DNS_SERVICE} == "coredns" ]]; then
    feature_gates="{CoreDNS: true}"
  fi

  kubeadm_version="$(dind::kubeadm-version)"
  case "${kubeadm_version}" in
    1\.12\.*)
      template="1.12"
      ;;
    *)  # Includes 1.13 master branch
      # Will make a separate template if/when it becomes incompatible
      template="1.13"
      # CoreDNS can no longer be switched off
      feature_gates="{}"
      ;;
  esac
  dind::check-dns-service-type

  local kubelet_version_specific_flags=()
  local cmp_kubelet_to_1_15=0
  dind::compare-versions 'kubelet' "$(dind::kubelet-version)" 1 15 || cmp_kubelet_to_1_15=$?
  if [[ "$cmp_kubelet_to_1_15" == 2 ]]; then
    # this option got deprecated in v 1.15
    kubelet_version_specific_flags+=('--allow-privileged=true')
  fi
  # explicit conversion to a string is needed, as calling ${arr[@]} or ${arr[*]}
  # on an empty array will trigger an error on bash < 4.4 (and Travis is 4.3...)
  local kubelet_version_specific_flags_as_str=''
  if [[ ${#kubelet_version_specific_flags[@]} -gt 0 ]]; then
    kubelet_version_specific_flags_as_str="${kubelet_version_specific_flags[*]}"
  fi
  dind::set-version-specific-flags "$kubelet_version_specific_flags_as_str"

  component_feature_gates=""
  if [ "${FEATURE_GATES}" != "none" ]; then
    component_feature_gates="feature-gates: \\\"${FEATURE_GATES}\\\""
  fi

  apiserver_extra_args=""
  for e in $(set -o posix ; set | grep -E "^APISERVER_[a-z_]+=" | cut -d'=' -f 1); do
    opt_name=$(echo ${e#APISERVER_} | sed 's/_/-/g')
    apiserver_extra_args+="    ${opt_name}: \\\"$(eval echo \$$e)\\\"\\n"
  done

  controller_manager_extra_args=""
  for e in $(set -o posix ; set | grep -E "^CONTROLLER_MANAGER_[a-z_]+=" | cut -d'=' -f 1); do
    opt_name=$(echo ${e#CONTROLLER_MANAGER_} | sed 's/_/-/g')
    controller_manager_extra_args+="    ${opt_name}: \\\"$(eval echo \$$e)\\\"\\n"
  done

  scheduler_extra_args=""
  for e in $(set -o posix ; set | grep -E "^SCHEDULER_[a-z_]+=" | cut -d'=' -f 1); do
    opt_name=$(echo ${e#SCHEDULER_} | sed 's/_/-/g')
    scheduler_extra_args+="    ${opt_name}: \\\"$(eval echo \$$e)\\\"\\n"
  done

  local mgmt_cidr=${mgmt_net_cidrs[0]}
  if [[ ${IP_MODE} = "dual-stack" && ${SERVICE_NET_MODE} = "ipv6" ]]; then
      mgmt_cidr=${mgmt_net_cidrs[1]}
  fi
  local master_ip=$( dind::make-ip-from-cidr ${mgmt_cidr} 2 )
  docker exec -i "$master_name" bash <<EOF
sed -e "s|{{ADV_ADDR}}|${master_ip}|" \
    -e "s|{{POD_SUBNET_DISABLE}}|${pod_subnet_disable}|" \
    -e "s|{{POD_NETWORK_CIDR}}|${pod_net_cidrs[0]}|" \
    -e "s|{{SVC_SUBNET}}|${SERVICE_CIDR}|" \
    -e "s|{{BIND_ADDR}}|${bind_address}|" \
    -e "s|{{BIND_PORT}}|${INTERNAL_APISERVER_PORT}|" \
    -e "s|{{FEATURE_GATES}}|${feature_gates}|" \
    -e "s|{{KUBEADM_VERSION}}|${kubeadm_version}|" \
    -e "s|{{COMPONENT_FEATURE_GATES}}|${component_feature_gates}|" \
    -e "s|{{APISERVER_EXTRA_ARGS}}|${apiserver_extra_args}|" \
    -e "s|{{CONTROLLER_MANAGER_EXTRA_ARGS}}|${controller_manager_extra_args}|" \
    -e "s|{{SCHEDULER_EXTRA_ARGS}}|${scheduler_extra_args}|" \
    -e "s|{{KUBE_MASTER_NAME}}|${master_name}|" \
    -e "s|{{DNS_SVC_IP}}|${DNS_SVC_IP}|" \
    -e "s|{{CRI_SOCKET}}|${CRI_SOCKET}|" \
    /etc/kubeadm.conf.${template}.tmpl > /etc/kubeadm.conf
EOF
  init_args=(--config /etc/kubeadm.conf)
  # required when building from source
  if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then
    docker exec "$master_name" mount --make-shared /k8s
  fi
  dind::kubeadm "${container_id}" init "${init_args[@]}" --ignore-preflight-errors=all "$@"
  kubeadm_join_flags="$(docker exec "${container_id}" kubeadm token create --print-join-command | sed 's/^kubeadm join //')"
  dind::configure-kubectl
  dind::start-port-forwarder
}

function dind::create-node-container {
  local reuse_volume next_node_index node_name
  reuse_volume=''
  if [[ ${1:-} = -r ]]; then
    reuse_volume="-r"
    shift
  fi
  # if there's just one node currently, it's master, thus we need to use
  # kube-node-1 hostname, if there are two nodes, we should pick
  # kube-node-2 and so on
  next_node_index=${1:-$(docker ps -q --filter=label="${DIND_LABEL}" | wc -l | sed 's/^ *//g')}
  local -a opts
  if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then
    opts+=(-v "dind-k8s-binaries$(dind::cluster-suffix)":/k8s)
    if [[ ${BUILD_KUBEADM} ]]; then
      opts+=(-e KUBEADM_SOURCE=build://)
    fi
    if [[ ${BUILD_HYPERKUBE} ]]; then
      opts+=(-e HYPERKUBE_SOURCE=build://)
    fi
  fi
  node_name="$(dind::node-name ${next_node_index})"
  dind::run ${reuse_volume} "$node_name" $((next_node_index + 1)) "${EXTRA_PORTS}" ${opts[@]+"${opts[@]}"}
}

function dind::join {
  local container_id="$1"
  shift
  dind::proxy "${container_id}"
  dind::custom-docker-opts "${container_id}"
  local -a join_opts=(--ignore-preflight-errors=all
                      --cri-socket="${CRI_SOCKET}")
  dind::kubeadm "${container_id}" join "${join_opts[@]}" "$@" >/dev/null
}

function dind::escape-e2e-name {
    sed 's/[]\$*.^()[]/\\&/g; s/\s\+/\\s+/g' <<< "$1" | tr -d '\n'
}

function dind::accelerate-kube-dns {
  if [[ ${DNS_SERVICE} == "kube-dns" ]]; then
     dind::step "Patching kube-dns deployment to make it start faster"
     # Could do this on the host, too, but we don't want to require jq here
     # TODO: do this in wrapkubeadm
     docker exec "$(dind::master-name)" /bin/bash -c \
        "kubectl get deployment kube-dns -n kube-system -o json | jq '.spec.template.spec.containers[0].readinessProbe.initialDelaySeconds = 3|.spec.template.spec.containers[0].readinessProbe.periodSeconds = 3' | kubectl apply --force -f -"
 fi
}

function dind::component-ready {
  local label="$1"
  local out
  if ! out="$("${kubectl}" --context "$(dind::context-name)" get pod -l "${label}" -n kube-system \
                           -o jsonpath='{ .items[*].status.conditions[?(@.type == "Ready")].status }' 2>/dev/null)"; then
    return 1
  fi
  if ! grep -v False <<<"${out}" | grep -q True; then
    return 1
  fi
  return 0
}

function dind::kill-failed-pods {
  local pods ctx
  ctx="$(dind::context-name)"
  # workaround for https://github.com/kubernetes/kubernetes/issues/36482
  if ! pods="$(kubectl --context "$ctx" get pod -n kube-system -o jsonpath='{ .items[?(@.status.phase == "Failed")].metadata.name }' 2>/dev/null)"; then
    return
  fi
  for name in ${pods}; do
    kubectl --context "$ctx" delete pod --now -n kube-system "${name}" >&/dev/null || true
  done
}

function dind::create-static-routes {
  echo "Creating static routes for bridge/PTP plugin"
  for ((i=0; i <= NUM_NODES; i++)); do
    if [[ ${i} -eq 0 ]]; then
      node="$(dind::master-name)"
    else
      node="$(dind::node-name $i)"
    fi
    for ((j=0; j <= NUM_NODES; j++)); do
      if [[ ${i} -eq ${j} ]]; then
        continue
      fi
      if [[ ${j} -eq 0 ]]; then
        dest_node="$(dind::master-name)"
      else
        dest_node="$(dind::node-name $j)"
      fi
      id=$((${j}+1))
      if [[ ${IP_MODE} = "ipv4" || ${IP_MODE} = "dual-stack" ]]; then
        # Assuming pod subnets will all be /24
        dest="${pod_prefixes[0]}${id}.0/24"
        gw=`docker exec ${dest_node} ip addr show eth0 | grep -w inet | awk '{ print $2 }' | sed 's,/.*,,'`
        docker exec "${node}" ip route add "${dest}" via "${gw}"
      fi
      if [[ ${IP_MODE} = "ipv6" || ${IP_MODE} = "dual-stack" ]]; then
        local position=0
        if [[ ${IP_MODE} = "dual-stack" ]]; then
            position=1
        fi
        instance=$(printf "%02x" ${id})
        if [[ $((${pod_sizes[$position]} % 16)) -ne 0 ]]; then
          instance+="00" # Move node ID to upper byte
        fi
        dest="${pod_prefixes[$position]}${instance}::/${pod_sizes[$position]}"
        gw=`docker exec ${dest_node} ip addr show eth0 | grep -w inet6 | grep -i global | head -1 | awk '{ print $2 }' | sed 's,/.*,,'`
        docker exec "${node}" ip route add "${dest}" via "${gw}"
      fi
    done
  done
}

# If we are allowing AAAA record use, then provide SNAT for IPv6 packets from
# node containers, and forward packets to bridge used for $(dind::net-name).
# This gives pods access to external IPv6 sites, when using IPv6 addresses.
function dind::setup_external_access_on_host {
  if [[ ! ${DIND_ALLOW_AAAA_USE} ]]; then
    return
  fi
  local main_if=`ip route | grep default | awk '{print $5}'`
  dind::ip6tables-on-hostnet -t nat -A POSTROUTING -o $main_if -j MASQUERADE
  if [[ ${IP_MODE} = "dual-stack" ]]; then
    return
  fi
  local bridge_if=`ip route | grep ${NAT64_V4_SUBNET_PREFIX}.0.0 | awk '{print $3}'`
  if [[ -n "$bridge_if" ]]; then
    dind::ip6tables-on-hostnet -A FORWARD -i $bridge_if -j ACCEPT
  else
    echo "WARNING! No $(dind::net-name) bridge with NAT64 - unable to setup forwarding/SNAT"
  fi
}

# Remove ip6tables rules for SNAT and forwarding, if they exist.
function dind::remove_external_access_on_host {
  if [[ ! ${DIND_ALLOW_AAAA_USE} ]]; then
    return
  fi
  local have_rule
  local main_if="$(ip route | grep default | awk '{print $5}')"
  have_rule="$(dind::ip6tables-on-hostnet -S -t nat | grep "\-o $main_if" || true)"
  if [[ -n "$have_rule" ]]; then
    dind::ip6tables-on-hostnet -t nat -D POSTROUTING -o $main_if -j MASQUERADE
  else
    echo "Skipping delete of ip6tables rule for SNAT, as rule non-existent"
  fi

  if [[ ${IP_MODE} = "dual-stack" ]]; then
    return
  fi
  local bridge_if="$(ip route | grep ${NAT64_V4_SUBNET_PREFIX}.0.0 | awk '{print $3}')"
  if [[ -n "$bridge_if" ]]; then
    have_rule="$(dind::ip6tables-on-hostnet -S | grep "\-i $bridge_if" || true)"
    if [[ -n "$have_rule" ]]; then
      dind::ip6tables-on-hostnet -D FORWARD -i $bridge_if -j ACCEPT
    else
      echo "Skipping delete of ip6tables rule for forwarding, as rule non-existent"
    fi
  else
    echo "Skipping delete of ip6tables rule for forwarding, as no bridge interface using NAT64"
  fi
}

function dind::ip6tables-on-hostnet {
  local mod_path='/lib/modules'
  docker run -v "${mod_path}:${mod_path}" --entrypoint /sbin/ip6tables --net=host --rm --privileged "${DIND_IMAGE}" "$@"
}

function dind::component-ready-by-labels {
  local labels=("$@");
  for label in ${labels[@]}; do
    dind::component-ready "${label}" && return 0
  done
  return 1
}

function dind::wait-for-service-ready {
  local service=$1
  local labels=("${@:2}")
  local ctx="$(dind::context-name)"

  dind::step "Bringing up ${service}"
  # on Travis 'scale' sometimes fails with 'error: Scaling the resource failed with: etcdserver: request timed out; Current resource version 442' here
  dind::retry "${kubectl}" --context "$ctx" scale deployment --replicas=1 -n kube-system ${service}

  local ntries=200
  while ! dind::component-ready-by-labels ${labels[@]}; do
    if ((--ntries == 0)); then
      echo "Error bringing up ${service}" >&2
      exit 1
    fi
    echo -n "." >&2
    dind::kill-failed-pods
    sleep 1
  done
  echo "[done]" >&2
}

function dind::wait-for-ready {
  local app="kube-proxy"
  if [[ ${CNI_PLUGIN} = "kube-router" ]]; then
    app=kube-router
  fi
  dind::step "Waiting for ${app} and the nodes"
  local app_ready
  local nodes_ready
  local n=3
  local ntries=200
  local ctx
  ctx="$(dind::context-name)"
  while true; do
    dind::kill-failed-pods
    if "${kubectl}" --context "$ctx" get nodes 2>/dev/null | grep -q NotReady; then
      nodes_ready=
    else
      nodes_ready=y
    fi
    if dind::component-ready k8s-app=${app}; then
      app_ready=y
    else
      app_ready=
    fi
    if [[ ${nodes_ready} && ${app_ready} ]]; then
      if ((--n == 0)); then
        echo "[done]" >&2
        break
      fi
    else
      n=3
    fi
    if ((--ntries == 0)); then
      echo "Error waiting for ${app} and the nodes" >&2
      exit 1
    fi
    echo -n "." >&2
    sleep 1
  done

  dind::wait-for-service-ready ${DNS_SERVICE} "k8s-app=kube-dns"

  if [[ ! ${SKIP_DASHBOARD} ]]; then
    local service="kubernetes-dashboard"
    dind::wait-for-service-ready ${service} "app=${service}" "k8s-app=${service}"
  fi

  dind::retry "${kubectl}" --context "$ctx" get nodes >&2

  if [[ ! ${SKIP_DASHBOARD} ]]; then
    local local_host
    local_host="$( dind::localhost )"
    local base_url="http://${local_host}:$(dind::apiserver-port)/api/v1/namespaces/kube-system/services"
    dind::step "Access dashboard at:" "${base_url}/kubernetes-dashboard:/proxy"
    dind::step "Access dashboard at:" "${base_url}/https:kubernetes-dashboard:/proxy (if version>1.6 and HTTPS enabled)"
  fi
}

# dind::make-kube-router-yaml creates a temp file with contents of the configuration needed for the kube-router CNI
# plugin at a specific version, instead of using the publically available file, which uses the latest version. This
# allows us to control the version used. If/when updating, be sure to update the KUBE_ROUTER_VERSION env variable
# ensure the YAML contents below, reflect the configuration in:
#
# https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/kubeadm-kuberouter-all-feature.yaml
#
# FUTURE: May be able to remove this, if/when kube-router "latest" is stable, and use the public YAML file instead.
function dind::make-kube-router-yaml {
  tmp_yaml=$(mktemp /tmp/kube-router-yaml.XXXXXX)
  cat >${tmp_yaml} <<KUBE_ROUTER_YAML
apiVersion: v1
kind: ConfigMap
metadata:
  name: kube-router-cfg
  namespace: kube-system
  labels:
    tier: node
    k8s-app: kube-router
data:
  cni-conf.json: |
    {
      "name":"kubernetes",
      "type":"bridge",
      "bridge":"kube-bridge",
      "isDefaultGateway":true,
      "ipam": {
        "type":"host-local"
      }
    }
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  labels:
    k8s-app: kube-router
    tier: node
  name: kube-router
  namespace: kube-system
spec:
  template:
    metadata:
      labels:
        k8s-app: kube-router
        tier: node
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      serviceAccountName: kube-router
      serviceAccount: kube-router
      containers:
      - name: kube-router
        image: cloudnativelabs/kube-router:${KUBE_ROUTER_VERSION}
        imagePullPolicy: Always
        args:
        - --run-router=true
        - --run-firewall=true
        - --run-service-proxy=true
        - --kubeconfig=/var/lib/kube-router/kubeconfig
        env:
        - name: NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        livenessProbe:
          httpGet:
            path: /healthz
            port: 20244
          initialDelaySeconds: 10
          periodSeconds: 3
        resources:
          requests:
            cpu: 250m
            memory: 250Mi
        securityContext:
          privileged: true
        volumeMounts:
        - name: lib-modules
          mountPath: /lib/modules
          readOnly: true
        - name: cni-conf-dir
          mountPath: /etc/cni/net.d
        - name: kubeconfig
          mountPath: /var/lib/kube-router
          readOnly: true
      initContainers:
      - name: install-cni
        image: busybox
        imagePullPolicy: Always
        command:
        - /bin/sh
        - -c
        - set -e -x;
          if [ ! -f /etc/cni/net.d/10-kuberouter.conf ]; then
            TMP=/etc/cni/net.d/.tmp-kuberouter-cfg;
            cp /etc/kube-router/cni-conf.json \${TMP};
            mv \${TMP} /etc/cni/net.d/10-kuberouter.conf;
          fi
        volumeMounts:
        - name: cni-conf-dir
          mountPath: /etc/cni/net.d
        - name: kube-router-cfg
          mountPath: /etc/kube-router
      hostNetwork: true
      tolerations:
      - key: CriticalAddonsOnly
        operator: Exists
      - effect: NoSchedule
        key: node-role.kubernetes.io/master
        operator: Exists
      volumes:
      - name: lib-modules
        hostPath:
          path: /lib/modules
      - name: cni-conf-dir
        hostPath:
          path: /etc/cni/net.d
      - name: kube-router-cfg
        configMap:
          name: kube-router-cfg
      - name: kubeconfig
        configMap:
          name: kube-proxy
          items:
          - key: kubeconfig.conf
            path: kubeconfig
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: kube-router
  namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: kube-router
  namespace: kube-system
rules:
  - apiGroups:
    - ""
    resources:
      - namespaces
      - pods
      - services
      - nodes
      - endpoints
    verbs:
      - list
      - get
      - watch
  - apiGroups:
    - "networking.k8s.io"
    resources:
      - networkpolicies
    verbs:
      - list
      - get
      - watch
  - apiGroups:
    - extensions
    resources:
      - networkpolicies
    verbs:
      - get
      - list
      - watch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: kube-router
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: kube-router
subjects:
- kind: ServiceAccount
  name: kube-router
  namespace: kube-system
KUBE_ROUTER_YAML
  echo $tmp_yaml
}

function dind::up {
  dind::down
  dind::init
  local ctx
  ctx="$(dind::context-name)"
  # pre-create node containers sequentially so they get predictable IPs
  local -a node_containers
  for ((n=1; n <= NUM_NODES; n++)); do
    dind::step "Starting node container:" ${n}
    if ! container_id="$(dind::create-node-container ${n})"; then
      echo >&2 "*** Failed to start node container ${n}"
      exit 1
    else
      node_containers+=(${container_id})
      dind::step "Node container started:" ${n}
    fi
  done
  dind::fix-mounts
  status=0
  local -a pids
  for ((n=1; n <= NUM_NODES; n++)); do
    (
      dind::step "Joining node:" ${n}
      container_id="${node_containers[${n}-1]}"
      if ! dind::join ${container_id} ${kubeadm_join_flags}; then
        echo >&2 "*** Failed to start node container ${n}"
        exit 1
      else
        dind::step "Node joined:" ${n}
      fi
    )&
    pids[${n}]=$!
  done
  if ((NUM_NODES > 0)); then
    for pid in ${pids[*]}; do
      wait ${pid}
    done
  else
    # FIXME: this may fail depending on k8s/kubeadm version
    # FIXME: check for taint & retry if it's there
    "${kubectl}" --context "$ctx" taint nodes $(dind::master-name) node-role.kubernetes.io/master- || true
  fi
  case "${CNI_PLUGIN}" in
    bridge | ptp)
      dind::create-static-routes
      dind::setup_external_access_on_host
      ;;
    flannel)
      # without --validate=false this will fail on older k8s versions
      dind::retry "${kubectl}" --context "$ctx" apply --validate=false -f "https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml?raw=true"
      ;;
    calico)
      manifest_base=https://docs.projectcalico.org/${CALICO_VERSION:-v3.3}/getting-started/kubernetes/installation
      dind::retry "${kubectl}" --context "$ctx" apply -f ${manifest_base}/hosted/etcd.yaml
      if [ "${CALICO_VERSION:-v3.3}" != master ]; then
        dind::retry "${kubectl}" --context "$ctx" apply -f ${manifest_base}/rbac.yaml
      fi
      dind::retry "${kubectl}" --context "$ctx" apply -f ${manifest_base}/hosted/calico.yaml
      dind::retry "${kubectl}" --context "$ctx" apply -f ${manifest_base}/hosted/calicoctl.yaml
      ;;
    calico-kdd)
      manifest_base=https://docs.projectcalico.org/${CALICO_VERSION:-v3.3}/getting-started/kubernetes/installation
      dind::retry "${kubectl}" --context "$ctx" apply -f ${manifest_base}/hosted/rbac-kdd.yaml
      dind::retry "${kubectl}" --context "$ctx" apply -f ${manifest_base}/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml
      ;;
    weave)
      dind::retry "${kubectl}" --context "$ctx" apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(${kubectl} --context "$ctx" version | base64 | tr -d '\n')"
      ;;
    kube-router)
      kube_router_config="$( dind::make-kube-router-yaml )"
      dind::retry "${kubectl}" --context "$ctx" apply -f ${kube_router_config}
      rm "${kube_router_config}"
      dind::retry "${kubectl}" --context "$ctx" -n kube-system delete ds kube-proxy
      docker run --privileged --net=host k8s.gcr.io/kube-proxy-amd64:v1.10.2 kube-proxy --cleanup
      ;;
    *)
      echo "Unsupported CNI plugin '${CNI_PLUGIN}'" >&2
      ;;
  esac

  if [[ ! ${SKIP_DASHBOARD} ]]; then
    dind::deploy-dashboard
  fi

  dind::accelerate-kube-dns
  if [[ (${CNI_PLUGIN} != "bridge" && ${CNI_PLUGIN} != "ptp") || ${SKIP_SNAPSHOT} ]]; then
    # This is especially important in case of Calico -
    # the cluster will not recover after snapshotting
    # (at least not after restarting from the snapshot)
    # if Calico installation is interrupted
    dind::wait-for-ready
  fi
  dind::step "Cluster Info"
  echo "Network Mode: ${IP_MODE}"
  echo "Cluster context: $( dind::context-name )"
  echo "Cluster ID: ${CLUSTER_ID}"
  echo "Management CIDR(s): ${mgmt_net_cidrs[@]}"
  echo "Service CIDR/mode: ${SERVICE_CIDR}/${SERVICE_NET_MODE}"
  echo "Pod CIDR(s): ${pod_net_cidrs[@]}"
}

function dind::fix-mounts {
  local node_name
  for ((n=0; n <= NUM_NODES; n++)); do
    node_name="$(dind::master-name)"
    if ((n > 0)); then
      node_name="$(dind::node-name $n)"
    fi
    docker exec "${node_name}" mount --make-shared /run
    if [[ ! ${using_linuxkit} ]]; then
      docker exec "${node_name}" mount --make-shared /lib/modules/
    fi
    # required when building from source
    if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then
      docker exec "${node_name}" mount --make-shared /k8s
    fi
    # docker exec "${node_name}" mount --make-shared /sys/kernel/debug
  done
}

function dind::snapshot_container {
  local container_name="$1"
  # we must pass DIND_CRI here because in case of containerd
  # a special care must be taken to stop the containers during
  # the snapshot
  docker exec -e DIND_CRI="${DIND_CRI}" -i ${container_name} \
         /usr/local/bin/snapshot prepare
  # remove the hidden *plnk directories
  docker diff ${container_name} | grep -v plnk | docker exec -i ${container_name} /usr/local/bin/snapshot save
}

function dind::snapshot {
  dind::step "Taking snapshot of the cluster"
  dind::snapshot_container "$(dind::master-name)"
  for ((n=1; n <= NUM_NODES; n++)); do
    dind::snapshot_container "$(dind::node-name $n)"
  done
  dind::wait-for-ready
}

restore_cmd=restore
function dind::restore_container {
  local container_id="$1"
  docker exec ${container_id} /usr/local/bin/snapshot "${restore_cmd}"
}

function dind::restore {
  local apiserver_port local_host pid pids
  dind::down
  dind::check-dns-service-type
  dind::step "Restoring containers"
  dind::set-master-opts
  local_host="$( dind::localhost )"
  apiserver_port="$( dind::apiserver-port )"
  for ((n=0; n <= NUM_NODES; n++)); do
    (
      if [[ n -eq 0 ]]; then
        dind::step "Restoring master container"
        dind::restore_container "$(dind::run -r "$(dind::master-name)" 1 "${local_host}:${apiserver_port}:${INTERNAL_APISERVER_PORT}" ${master_opts[@]+"${master_opts[@]}"})"
        dind::verify-image-compatibility "$(dind::master-name)"
        dind::step "Master container restored"
      else
        dind::step "Restoring node container:" ${n}
        if ! container_id="$(dind::create-node-container -r ${n})"; then
          echo >&2 "*** Failed to start node container ${n}"
          exit 1
        else
          dind::restore_container "${container_id}"
          dind::step "Node container restored:" ${n}
        fi
      fi
    )&
    pids[${n}]=$!
  done
  for pid in ${pids[*]}; do
    wait ${pid}
  done
  if [[ ${CNI_PLUGIN} = "bridge" || ${CNI_PLUGIN} = "ptp" ]]; then
    dind::create-static-routes
    dind::setup_external_access_on_host
  fi
  dind::fix-mounts
  # Recheck kubectl config. It's possible that the cluster was started
  # on this docker from different host
  dind::configure-kubectl
  dind::start-port-forwarder
  dind::wait-for-ready
}

function dind::docker-action {
  action=$1
  docker $action "$(dind::master-name)"
  for ((n=1; n <= NUM_NODES; n++)); do
    docker $action "$(dind::node-name $n)"
  done
}
function dind::pause {
  dind::step "Pausing the cluster"
  dind::docker-action pause
}

function dind::unpause {
  dind::step "Unpausing the cluster"
  dind::docker-action unpause
}

function dind::down {
  dind::remove-images "${DIND_LABEL}"
  if [[ ${CNI_PLUGIN} = "bridge" || ${CNI_PLUGIN} = "ptp" ]]; then
    dind::remove_external_access_on_host
  elif [[ "${CNI_PLUGIN}" = "kube-router" ]]; then
    if [[ ${COMMAND} = "down" || ${COMMAND} = "clean" ]]; then
      # FUTURE: Updated pinned version, after verifying operation
      docker run --privileged --net=host cloudnativelabs/kube-router:${KUBE_ROUTER_VERSION} --cleanup-config
    fi
  fi
}

function dind::apiserver-port {
  # APISERVER_PORT is explicitely set
  if [ -n "${APISERVER_PORT:-}" ]
  then
    echo "$APISERVER_PORT"
    return
  fi

  # Get the port from the master
  local master port
  master="$(dind::master-name)"
  # 8080/tcp -> 127.0.0.1:8082  =>  8082
  port="$( docker port "$master" 2>/dev/null | awk -F: "/^${INTERNAL_APISERVER_PORT}/{ print \$NF }" )"
  if [ -n "$port" ]
  then
    APISERVER_PORT="$port"
    echo "$APISERVER_PORT"
    return
  fi

  # get a random free port
  APISERVER_PORT=0
  echo "$APISERVER_PORT"
}

function dind::master-name {
  echo "kube-master$( dind::cluster-suffix )"
}

function dind::node-name {
  local nr="$1"
  echo "kube-node-${nr}$( dind::cluster-suffix )"
}

function dind::context-name {
  echo "dind$( dind::cluster-suffix )"
}

function dind::remove-volumes {
  # docker 1.13+: docker volume ls -q -f label="${DIND_LABEL}"
  local nameRE
  nameRE="^kubeadm-dind-(sys|kube-master|kube-node-[0-9]+)$(dind::cluster-suffix)$"
  docker volume ls -q | (grep -E "$nameRE" || true) | while read -r volume_id; do
    dind::step "Removing volume:" "${volume_id}"
    docker volume rm "${volume_id}"
  done
}

function dind::remove-images {
  local which=$1
  docker ps -a -q --filter=label="${which}" | while read container_id; do
    dind::step "Removing container:" "${container_id}"
    docker rm -fv "${container_id}"
  done
}

function dind::remove-cluster {
  cluster_name="dind$(dind::cluster-suffix)"
  if ${kubectl} config get-clusters | grep -qE "^${cluster_name}$"; then
    dind::step "Removing cluster from config:" "${cluster_name}"
    ${kubectl} config delete-cluster ${cluster_name} 2>/dev/null || true
  fi
}

function dind::remove-context {
  context_name="$(dind::context-name)"
  if ${kubectl} config get-contexts | grep -qE "${context_name}\\s"; then
    dind::step "Removing context from config:" "${context_name}"
    ${kubectl} config delete-context ${context_name} 2>/dev/null || true
  fi
}

function dind::start-port-forwarder {
  local fwdr port
  fwdr="${DIND_PORT_FORWARDER:-}"

  [ -n "$fwdr" ] || return 0

  [ -x "$fwdr" ] || {
    echo "'${fwdr}' is not executable." >&2
    return 1
  }

  port="$( dind::apiserver-port )"
  dind::step "+ Setting up port-forwarding for :${port}"
  "$fwdr" "$port"
}

function dind::check-for-snapshot {
  if ! dind::volume-exists "kubeadm-dind-$(dind::master-name)"; then
    return 1
  fi
  for ((n=1; n <= NUM_NODES; n++)); do
    if ! dind::volume-exists "kubeadm-dind-$(dind::node-name ${n})"; then
      return 1
    fi
  done
}

function dind::do-run-e2e {
  local parallel="${1:-}"
  local focus="${2:-}"
  local skip="${3:-}"
  local host="$(dind::localhost)"
  if [[ -z "$using_local_linuxdocker" ]]; then
    host="127.0.0.1"
  fi
  dind::need-source
  local kubeapi test_args term=
  local -a e2e_volume_opts=()
  kubeapi="http://${host}:$(dind::apiserver-port)"
  test_args="--host=${kubeapi}"
  if [[ ${focus} ]]; then
    test_args="--ginkgo.focus=${focus} ${test_args}"
  fi
  if [[ ${skip} ]]; then
    test_args="--ginkgo.skip=${skip} ${test_args}"
  fi
  if [[ ${E2E_REPORT_DIR} ]]; then
    test_args="--report-dir=/report ${test_args}"
    e2e_volume_opts=(-v "${E2E_REPORT_DIR}:/report")
  fi
  dind::make-for-linux n "cmd/kubectl test/e2e/e2e.test vendor/github.com/onsi/ginkgo/ginkgo"
  dind::step "Running e2e tests with args:" "${test_args}"
  dind::set-build-volume-args
  if [ -t 1 ] ; then
    term="-it"
    test_args="--ginkgo.noColor --num-nodes=2 ${test_args}"
  fi
  docker run \
         --rm ${term} \
         --net=host \
         "${build_volume_args[@]}" \
         -e KUBERNETES_PROVIDER=dind \
         -e KUBE_MASTER_IP="${kubeapi}" \
         -e KUBE_MASTER=local \
         -e KUBERNETES_CONFORMANCE_TEST=y \
         -e GINKGO_PARALLEL=${parallel} \
         ${e2e_volume_opts[@]+"${e2e_volume_opts[@]}"} \
         -w /go/src/k8s.io/kubernetes \
         "${e2e_base_image}" \
         bash -c "cluster/kubectl.sh config set-cluster dind --server='${kubeapi}' --insecure-skip-tls-verify=true &&
         cluster/kubectl.sh config set-context dind --cluster=dind &&
         cluster/kubectl.sh config use-context dind &&
         go run hack/e2e.go -- --v 6 --test --check-version-skew=false --test_args='${test_args}'"
}

function dind::clean {
  dind::ensure-downloaded-kubectl
  dind::down
  dind::remove-images "dind-support$( dind::cluster-suffix )"
  dind::remove-volumes
  local net_name
  net_name="$(dind::net-name)"
  if docker network inspect "$net_name" >&/dev/null; then
    docker network rm "$net_name"
  fi
  dind::remove-cluster
  dind::remove-context
}

function dind::copy-image {
  local image="${2:-}"
  local image_path="/tmp/save_${image//\//_}"
  if [[ -f "${image_path}" ]]; then
    rm -fr "${image_path}"
  fi
  docker save "${image}" -o "${image_path}"
  docker ps -a -q --filter=label="${DIND_LABEL}" | while read container_id; do
    cat "${image_path}" | docker exec -i "${container_id}" docker load
  done
  rm -fr "${image_path}"
}

function dind::run-e2e {
  local focus="${1:-}"
  local skip="${2:-[Serial]}"
  skip="$(dind::escape-e2e-name "${skip}")"
  if [[ "$focus" ]]; then
    focus="$(dind::escape-e2e-name "${focus}")"
  else
    focus="\[Conformance\]"
  fi
  local parallel=y
  if [[ ${DIND_NO_PARALLEL_E2E} ]]; then
    parallel=
  fi
  dind::do-run-e2e "${parallel}" "${focus}" "${skip}"
}

function dind::run-e2e-serial {
  local focus="${1:-}"
  local skip="${2:-}"
  skip="$(dind::escape-e2e-name "${skip}")"
  dind::need-source
  if [[ "$focus" ]]; then
    focus="$(dind::escape-e2e-name "${focus}")"
  else
    focus="\[Serial\].*\[Conformance\]"
  fi
  dind::do-run-e2e n "${focus}" "${skip}"
  # TBD: specify filter
}

function dind::step {
  local OPTS=""
  if [ "$1" = "-n" ]; then
    shift
    OPTS+="-n"
  fi
  GREEN="$1"
  shift
  if [ -t 2 ] ; then
    echo -e ${OPTS} "\x1B[97m* \x1B[92m${GREEN}\x1B[39m $*" 1>&2
  else
    echo ${OPTS} "* ${GREEN} $*" 1>&2
  fi
}

function dind::dump {
  set +e
  echo "*** Dumping cluster state ***"
  for node in $(docker ps --format '{{.Names}}' --filter label="${DIND_LABEL}"); do
    for service in kubelet.service dindnet.service criproxy.service dockershim.service; do
      if docker exec "${node}" systemctl is-enabled "${service}" >&/dev/null; then
        echo "@@@ service-${node}-${service}.log @@@"
        docker exec "${node}" systemctl status "${service}"
        docker exec "${node}" journalctl -xe -n all -u "${service}"
      fi
    done
    echo "@@@ psaux-${node}.txt @@@"
    docker exec "${node}" ps auxww
    echo "@@@ dockerps-a-${node}.txt @@@"
    docker exec "${node}" docker ps -a
    echo "@@@ ip-a-${node}.txt @@@"
    docker exec "${node}" ip a
    echo "@@@ ip-r-${node}.txt @@@"
    docker exec "${node}" ip r
  done
  local ctx master_name
  master_name="$(dind::master-name)"
  ctx="$(dind::context-name)"
  docker exec "$master_name" kubectl get pods --all-namespaces \
          -o go-template='{{range $x := .items}}{{range $x.spec.containers}}{{$x.spec.nodeName}}{{" "}}{{$x.metadata.namespace}}{{" "}}{{$x.metadata.name}}{{" "}}{{.name}}{{"\n"}}{{end}}{{end}}' |
    while read node ns pod container; do
      echo "@@@ pod-${node}-${ns}-${pod}--${container}.log @@@"
      docker exec "$master_name" kubectl logs -n "${ns}" -c "${container}" "${pod}"
    done
  echo "@@@ kubectl-all.txt @@@"
  docker exec "$master_name" kubectl get all --all-namespaces -o wide
  echo "@@@ describe-all.txt @@@"
  docker exec "$master_name" kubectl describe all --all-namespaces
  echo "@@@ nodes.txt @@@"
  docker exec "$master_name" kubectl get nodes -o wide
}

function dind::dump64 {
  echo "%%% start-base64 %%%"
  dind::dump | docker exec -i "$(dind::master-name)" /bin/sh -c "lzma | base64 -w 100"
  echo "%%% end-base64 %%%"
}

function dind::split-dump {
  mkdir -p cluster-dump
  cd cluster-dump
  awk '!/^@@@ .* @@@$/{print >out}; /^@@@ .* @@@$/{out=$2}' out=/dev/null
  ls -l
}

function dind::split-dump64 {
  decode_opt=-d
  if base64 --help | grep -q '^ *-D'; then
    # Mac OS X
    decode_opt=-D
  fi
  sed -n '/^%%% start-base64 %%%$/,/^%%% end-base64 %%%$/p' |
    sed '1d;$d' |
    base64 "${decode_opt}" |
    lzma -dc |
    dind::split-dump
}

function dind::proxy {
  local container_id="$1"
  if [[ ${DIND_CA_CERT_URL} ]] ; then
    dind::step "+ Adding certificate on ${container_id}"
    docker exec ${container_id} /bin/sh -c "cd /usr/local/share/ca-certificates; curl -sSO ${DIND_CA_CERT_URL}"
    docker exec ${container_id} update-ca-certificates
  fi
  if [[ "${DIND_PROPAGATE_HTTP_PROXY}" || "${DIND_HTTP_PROXY}" || "${DIND_HTTPS_PROXY}" || "${DIND_NO_PROXY}" ]]; then
    dind::step "+ Setting *_PROXY for docker service on ${container_id}"
    local proxy_env="[Service]"$'\n'"Environment="
    if [[ "${DIND_PROPAGATE_HTTP_PROXY}" ]]; then
      # take *_PROXY values from container environment
      proxy_env+=$(docker exec ${container_id} env | grep -i _proxy | awk '{ print "\""$0"\""}' | xargs -d'\n')
    else
      if [[ "${DIND_HTTP_PROXY}" ]] ;  then proxy_env+="\"HTTP_PROXY=${DIND_HTTP_PROXY}\" "; fi
      if [[ "${DIND_HTTPS_PROXY}" ]] ; then proxy_env+="\"HTTPS_PROXY=${DIND_HTTPS_PROXY}\" "; fi
      if [[ "${DIND_NO_PROXY}" ]] ;    then proxy_env+="\"NO_PROXY=${DIND_NO_PROXY}\" "; fi
    fi
    docker exec -i ${container_id} /bin/sh -c "cat > /etc/systemd/system/docker.service.d/30-proxy.conf" <<< "${proxy_env}"
    docker exec ${container_id} systemctl daemon-reload
    docker exec ${container_id} systemctl restart docker
  fi
}

function dind::custom-docker-opts {
  local container_id="$1"
  local -a jq=()
  local got_changes=""
  if [[ ! -f ${DIND_DAEMON_JSON_FILE} ]] ; then
    jq[0]="{}"
  else
    jq+=("$(cat ${DIND_DAEMON_JSON_FILE})")
    if [[ ${DIND_DAEMON_JSON_FILE} != "/etc/docker/daemon.json" ]]; then
      got_changes=1
    fi
  fi
  if [[ ${DIND_REGISTRY_MIRROR} ]] ; then
    dind::step "+ Setting up registry mirror on ${container_id}"
    jq+=("{\"registry-mirrors\": [\"${DIND_REGISTRY_MIRROR}\"]}")
    got_changes=1
  fi
  if [[ ${DIND_INSECURE_REGISTRIES} ]] ; then
    dind::step "+ Setting up insecure-registries on ${container_id}"
    jq+=("{\"insecure-registries\": ${DIND_INSECURE_REGISTRIES}}")
    got_changes=1
  fi
  if [[ ${got_changes} ]] ; then
    local json=$(IFS="+"; echo "${jq[*]}")
    docker exec -i ${container_id} /bin/sh -c "mkdir -p /etc/docker && jq -n '${json}' > /etc/docker/daemon.json"
    docker exec ${container_id} systemctl daemon-reload
    docker exec ${container_id} systemctl restart docker
  fi
}

COMMAND="${1:-}"

case ${COMMAND} in
  up)
    if [[ ! ( ${DIND_IMAGE} =~ local ) && ! ${DIND_SKIP_PULL:-} ]]; then
      dind::step "Making sure DIND image is up to date"
      docker pull "${DIND_IMAGE}" >&2
    fi

    dind::prepare-sys-mounts
    dind::ensure-kubectl
    if [[ ${SKIP_SNAPSHOT} ]]; then
      force_make_binaries=y dind::up
    elif ! dind::check-for-snapshot; then
      force_make_binaries=y dind::up
      dind::snapshot
    else
      dind::restore
    fi
    ;;
  reup)
    dind::prepare-sys-mounts
    dind::ensure-kubectl
    if [[ ${SKIP_SNAPSHOT} ]]; then
      force_make_binaries=y dind::up
    elif ! dind::check-for-snapshot; then
      force_make_binaries=y dind::up
      dind::snapshot
    else
      force_make_binaries=y
      restore_cmd=update_and_restore
      dind::restore
    fi
    ;;
  down)
    dind::down
    ;;
  init)
    shift
    dind::prepare-sys-mounts
    dind::ensure-kubectl
    dind::init "$@"
    ;;
  join)
    shift
    dind::prepare-sys-mounts
    dind::ensure-kubectl
    dind::join "$(dind::create-node-container)" "$@"
    ;;
  pause)
    shift
    dind::pause
    ;;
  unpause)
    shift
    dind::unpause
    ;;
  snapshot)
    shift
    dind::snapshot
    ;;
  restore)
    shift
    dind::restore
    ;;
  clean)
    dind::clean
    ;;
  copy-image)
    dind::copy-image "$@"
    ;;
  e2e)
    shift
    dind::run-e2e "$@"
    ;;
  e2e-serial)
    shift
    dind::run-e2e-serial "$@"
    ;;
  dump)
    dind::dump
    ;;
  dump64)
    dind::dump64
    ;;
  split-dump)
    dind::split-dump
    ;;
  split-dump64)
    dind::split-dump64
    ;;
  apiserver-port)
    dind::apiserver-port
    ;;
  *)
    echo "usage:" >&2
    echo "  $0 up" >&2
    echo "  $0 reup" >&2
    echo "  $0 down" >&2
    echo "  $0 init kubeadm-args..." >&2
    echo "  $0 join kubeadm-args..." >&2
    echo "  $0 clean"
    echo "  $0 pause"
    echo "  $0 unpause"
    echo "  $0 snapshot"
    echo "  $0 restore"
    echo "  $0 copy-image [image_name]" >&2
    echo "  $0 e2e [test-name-substring]" >&2
    echo "  $0 e2e-serial [test-name-substring]" >&2
    echo "  $0 dump" >&2
    echo "  $0 dump64" >&2
    echo "  $0 split-dump" >&2
    echo "  $0 split-dump64" >&2
    exit 1
    ;;
esac


By Using BeyondBasicBlogging.com, you are consenting to our terms and conditions as outlined below.
When you purchase Beyond Basic Blogging or any of our other products, you understand you are purchasing a license to use the site for your own use. Sharing access to Beyond Basic Blogging is NOT allowed and is a breach of our terms and conditions. Your access may not be shared or transferred. If any employee of B3 finds that students are sharing access, purchasing the course for multiple users, or has made any offers or agreements to do such, their access to B3 will be immediately revoked, you will forfeit your payment, and you will be liable for damages to Beyond Basic Blogging. Please respect the proprietary nature of this information.
To access or use the Site, you must be 18 years of age or older and have the requisite power and authority to enter into these Terms and Conditions. Children under the age of 18 are prohibited from using the Site. Information provided on the Site and in the Service related to membership sites and other information are subject to change. Beyond Basic Blogging ("B3") makes no representation or warranty that the information provided, regardless of its source (the "Content"), is accurate, complete, reliable, current, or error-free.
Any claim relating to Company's web site shall be governed by the laws of the State of Washington without regard to its conflict of law provisions.
In order to use the Service, you will be required to provide information about yourself including your name, email address, username and password, and other personal information. You agree that any registration information you give to B3 will always be accurate, correct, and up to date. You must not impersonate someone else or provide account information or an email address other than your own. Your account must not be used for any illegal or unauthorized purpose. You must not, in the use of the Service, violate any laws in your jurisdiction.
Once confirmed, we will provide you access to the purchased Service. You agree and understand that access to the Service may at times be influenced and affected by third parties that we use to provide the Service (web hosting, membership site plugins, etc.). You agree and understand that no breach of contract action may be initiated against B3 when there are reasonable delays in the access of the Service.
B3 reserves the right to terminate the Service, and or access to certain features of the Service, with or without prior notice to you. We will make reasonable efforts to provide notice but is not required to do so under the terms of this agreement.
Lifetime Access is for the lifetime of the Service. If for any reason, B3 should dissolve or cease to exist, then your access to the Service terminates.
We offer a 14 day money back guarantee should you be unhappy with the Service in your first month of membership. To request a refund please contact support at hello@beyondbasicblogging.com.
We do not offer any refunds on downloadable products, including but not limited to: downloadable worksheets, workbooks, checklists, or challenges.
YOU AGREE THAT UNDER NO CIRCUMSTANCES SHALL WE BE LIABLE FOR DIRECT, INDIRECT, INCIDENTAL, CONSEQUENTIAL, SPECIAL, PUNITIVE, EXEMPLARY, OR ANY OTHER DAMAGES ARISING OUT OF YOUR USE OF THE SITE OR SERVICE. ADDITIONALLY, B3 IS NOT LIABLE FOR DAMAGES IN CONNECTION WITH (I) ANY FAILURE OF PERFORMANCE, ERROR, OMISSION, DENIAL OF SERVICE, ATTACK, INTERRUPTION, DELETION, DEFECT, DELAY IN OPERATION OR TRANSMISSION, COMPUTER VIRUS, OR LINE OR SYSTEM FAILURE; (II) LOSS OF REVENUE, ANTICIPATED PROFITS, BUSINESS, SAVINGS, GOODWILL OR DATA; AND (III) THIRD PARTY THEFT OF, DESTRUCTION OF, UNAUTHORIZED ACCESS TO, ALTERATION OF, OR USE OF YOUR INFORMATION OR PROPERTY, REGARDLESS OF OUR NEGLIGENCE, GROSS NEGLIGENCE, FAILURE OF AN ESSENTIAL PURPOSE AND WHETHER SUCH LIABILITY ARISES IN NEGLIGENCE, CONTRACT, TORT, OR ANY OTHER THEORY OF LEGAL LIABILITY. THE FOREGOING APPLIES EVEN IF B3 HAS BEEN ADVISED OF THE POSSIBILITY OF OR COULD HAVE FORESEEN THE DAMAGES. IN THOSE STATES THAT DO NOT ALLOW THE EXCLUSION OR LIMITATION OF LIABILITY FOR THE DAMAGES, OUR LIABILITY IS LIMITED TO THE FULLEST POSSIBLE EXTENT PERMITTED BY LAW. IN NO EVENT SHALL B3 CUMULATIVE LIABILITY TO YOU EXCEED THE TOTAL PURCHASE PRICE OF THE SERVICE YOU HAVE PURCHASED FROM B3, AND IF NO PURCHASE HAS BEEN MADE BY YOU B3 CUMULATIVE LIABILITY TO YOU SHALL NOT EXCEED $100.
No waiver of any of the provisions of this Agreement by B3 shall be deemed, or shall constitute, a waiver of any other provision, whether or not similar, nor shall any waiver constitute a continuing waiver. No waiver shall be binding unless executed in writing by B3.
Copyright 2019 by Beyond Basic Blogging.


Organic Sweet Wheatgrass seeds perfected for indoor growing. Grows four flats of Sweet Wheatgrass.
Wheatgrass helps your body build red blood cells, which carry oxygen to every cell. By increasing the oxygenation in your body, you can help offset carbon monoxide and increase your endurance during physical exercise. It cleanses the body by activating white blood cells, boost the body's immune system, and contains more vitamin C than oranges, and twice as much vitamin A as carrots. Pound for pound, wheatgrass is more than 20x denser in nutrients than other choice vegetables. Since it is considered a vegetable in the grass stage, wheatgrass is safe for people with wheat allergies.


Winter, such as it was, is pretty much over. Over the next few weeks, the daily temperatures will continue to climb as we get closer to the summer season. You're going to be relying on your air conditioner extensively to keep cool during daylight hours for a while, which means you need to make sure that you have an air conditioning system you can count on installed in your home. Have a look at some of the signs that it's time for you to install a new air conditioning system.
Do I Need to Replace My Air Conditioner This Season?
We've been dealing with hot and humid weather for the last few weeks, long before the rest of the country has started to thaw out. It is only going to get hotter in the coming months, though, which means that if you don't have a reliable air conditioner installed already, you're in trouble. If you have an air conditioner, but you're not sure whether or not it's going to last another season, it's a good idea to replace it now. Otherwise, you run the risk of having it die on you in the middle of the hottest months of the year. Make sure you're familiar with the warning signs that your system is on its last legs, so that you won't be caught off-guard.
Posted in Air Conditioning | Comments Off on Do I Need to Replace My Air Conditioner This Season?
Commercial air conditioners are among the more sturdy HVAC systems on the market. They have to be, considering how much strain they're being put under every time they're in use. However, though they may be able to last a long time, there will come a point where they need to be replaced. The sooner you can recognize when your commercial air conditioner needs replacing, the more money you'll save that would otherwise be wasted on an inefficient system. With that in mind, check out some of the signs that your commercial air conditioner is nearing the end of its life.


Robert O'Brien of O'Brien Guitars demonstrates how to fix a buzzing nut slot with Hot Stuff Original CA glue and baking soda. Baking soda is a tool that has been used in conjunction with CA glue for decades in many different fields. When you fill a gap with baking soda and drip Hot Stuff over it, the glue will react with the baking soda and cure very quickly. You then end up with a very strong and hard filling which can be sanded as needed.


text:
Back-to-back wins and second place in the AJ Bell NBL delight Suffolk Saxons coach Anthony Clarke
Edmund Crosthwaite
Two-time Olympian and Suffolk Saxons head coach Anthony Clarke is delighted with his team's start to the season. - Credit: Su Anderson
Anthony Clarke has said the start to his team's debut season in the National Badminton League has been "crazy" – but is delighted with their performances.
Sean Vendy and Fee Teng Liew in the mixed doubles for Suffolk Saxons against Loughborough Lightning - Credit: Archant
The Suffolk Saxons were only formed earlier this year as a professional outfit and stepped into the highly competitive AJ Bell NBL at the start of October.
Their first match ended in a 4-1 defeat – though Clarke has highlighted some extenuating circumstances – but since then have refused to lose.
A 4-1 win against Surrey Smashers in the Saxons' first home match at Ipswich's Corn Exchange on November 7 was followed by a thrilling battle at Loughborough Lightning on Monday.
The 3-2 win for the visitors was a showcase for the league's competitiveness (twice Saxons players battled back from 2-0 down to win events) and will have extended highlights screened on BT Sport tonight.
"It's been a crazy start to be honest," Clarke said. "I'm over the moon with the way it's gone.
"We were very disappointed in the very first match that four of our players couldn't play because of the Danish Open."
But he added: "Having taken out two of the best teams in the league is amazing. Going away to Loughborough I was expecting a really tough match."
The Saxons laid on free coach travel for fans following the team to Loughborough and Clarke praised their support, saying he expected the home crowd to pull their players over the line when actually it was the travelling fans who made the most noise.
The Suffolk side maintained second place in the league having played half of their six fixtures for the season – though some teams below them have a game in hand – but Clarke said things could have been different if they had been able to field a full strength team against now-league leaders Birmingham.
"Birmingham Lions had their full team out when we played them. We took a 4-1 hit in that match and we're only one point behind them.
"It could have been very different, we could have been top. We're just very proud of where we are right now."
Clarke also had praise for new signing, 17-year-old Fee Teng Liew, who played her first match for the Saxons in the mixed doubles with Sean Vendy – one of the events where the team won three straight matches from 2-0 down for a victory.
"She took the final match of the night," Clarke said. "She looked nervous to start with but that's understandable. There is pressure with the NBL because it is a short game."
Along with Vendy and Liew's 3-2 win in the mixed doubles on Monday there were triumphs from Julie Finne-Ipsen in the women's singles, and Sarah Walker and Finne-Ipsen in the women's doubles.
Next up for the Saxons is a home tie against bottom-of-the-table Bristol Jets, the league's other new team this season.
Clarke said: "We are looking to try and win again, of course. Our home crowd were wonderful on the first night and we're expecting more people to come this time."

redpajama_set_name: RedPajamaCommonCrawl


text:
Nfl Week 7 Schedule, Betting Odds, Point Spreads For Every Game
Tennessee Titans At Indianapolis Colts Sunday, 1 P M Et
Nfl Previews Week 16
Sunday Football
The Seahawks gained 173 rushing yards in the win over Green Bay. Minnesota (6-5-1) is coming off a loss to New England last week and has been playing pretty good defense as of late. The problem for the Vikings is that the running game continues to sputter. Quarterback Kirk Cousins makes up for much of the Vikings' inability to run the football.
The Patriots opened with the second-largest spread this week, starting as 9.5-point favorites against the Texans.
"They treated this game as if they are one of those teams that belongs."
A win over Washington on Monday night would surely help ease the pain of what has been a long year for head coach Ron Rivera.
The ticket count for this matchup is pretty even, with the Chiefs holding a 52-48% advantage at the time of writing.
The Giants average 77.8 rushing yards per game, 30thin the NFL. Without the ability to run the ball, teams can rush QB Eli Manning and the results produce just 16 points per game for the Giants. With their best offensive player now out, things might even get worse. If Beckham's injury wasn't enough, receivers Brandon Marshall and Sterling Shepard both left last Sunday's game with ankle injuries.
Coach Kevin Stefanski says try what he says Mayfield is still on track to start this contest. Given the third-year signal caller's caliber of play Sunday, that may not exactly be good news for Cleveland's fortunes. Every season, a few big favorites will lose and cause some havoc. Since the departure of stud wideout Julio Jones, Atlanta can't seem to make up for its defensive ineptitude with offensive production. The Falcons only beat Zach Wilson and the Jets by seven last week, despite New York putting up just 230 yards. Don't count on Tua and the Dolphins being that bad this week, or for that matter, Miami allowing Atlanta to amass 450 total yards of offense like the Jets did.
Pittsburgh's running game, which features RB Le'Veon Bell, has not performed well thus far this season. With a big game informative post last week, WR Antonio Brown now leads the NFL in receiving yards with 354. Brown, Bell, and quarterback Ben Roethlisberger must have a big day against the Baltimore defense to pull off a win in Baltimore. It's no secret that the Steelers will have to shut down the Baltimore running game. The Ravens are currently fourth in the NFL in rushing averaging 142.3 yards per game.
Week 7 Nfl Odds, Spreads, Lines
It was also a rare cover in a favorite role for the Chiefs, who improved to 2-4 ATS with that victory. Kansas City is 4-2 to the Over to this point and this is a total that has been on the rise. The Washington Football Team has covered just once in six games so far this season. The 1-5 ATS record is pretty indicative of how poorly the Football Team has played, especially on the defensive side of the ball. Washington is 4-2 to the Over while going just 2-4 SU on the season.
Finally, the last two stand-alone games of the week on "Sunday Night Football" and "Monday Night Football" saw both the underdogs cover again. Quite a few divisional games in the mix and Jon and his team have the inside scoop on all of this week's NFL action. The AFC East kicks things off Thursday night with Rex Ryan and the Bills visiting his old team, the NY Jets.
Robby Anderson has 40 catches for 566 yards (2nd-most in the NFL) in a breakout debut season with his new team. Christian McCaffrey likely won't be able to return for this game, but Mike Davis has been very strong aside from a poor performance this past week against a tough Bears defense. The Saints are coming off a bye week and are averaging 30.6 points per game, the fifth-most in the NFL. All-Pro wide receiver Michael Thomas is expected to return this week and will be a much-needed addition to an offense that has remained effective in his absence.