<file_sep>#!/bin/bash

##
## Move EC2 RAID array from a given instance to a given instance
##
## <NAME> <<EMAIL>>
##
##

##
## The MIT License (MIT)
## Copyright (c) 2012-2013 <NAME>
##
## Permission is hereby granted, free of charge, to any person obtaining a
## copy of this software and associated documentation files (the "Software"),
## to deal in the Software without restriction, including without limitation
## the rights to use, copy, modify, merge, publish, distribute, sublicense,
## and/or sell copies of the Software, and to permit persons to whom the
## Software is furnished to do so, subject to the following conditions:
##  
## The above copyright notice and this permission notice shall be included
## in all copies or substantial portions of the Software.
##
## THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
## IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
## FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
## AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
## LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
## FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
## DEALINGS IN THE SOFTWARE.
#

# Confirmation Prompt
#
confirm () {

    read -r -p "${1:-Continue? [y/N]} " response
    case $response in
        [yY][eE][sS]|[yY]) true
        ;;
        *) false
        ;;
    esac
}

# Print Usage
#
usage() {

    echo -e "\n*** This script will migrate a RAID array from one instance to another instance in the same availability zone. A safer way to do this is to build a new array on the destination instance and rsync everything over.\n"
    echo -e "\nUsage: $0 -f <from instance> -t <to instance> -d <drive letter> (-h for help)"

}

# Process Command Line Args
while getopts ":f:t:d:h" optname
do
    case ${optname} in
        h|H) usage
             echo -e "\n\tRequired Arguments:\n"
             echo -e "\t-f <from instance> - the instance id the array is currently attached to"
             echo -e "\t-t <to instance> - the instance id to copy the array to"
             echo -e "\t-d <drive> - the drive identifier (last letter), ie 'h' for /dev/xvdh*"
             echo -e "\n"
             exit 0
             ;;
        f|F) FROMINSTANCE=${OPTARG}
             ;;
        t|T) TOINSTANCE=${OPTARG}
             ;;
        d|D) DRIVEID=${OPTARG}
             ;;
          *) echo "No such option ${optname}."
             usage
             exit 1
             ;;
    esac 
done

# Do we have values for required arguments?
[[ $FROMINSTANCE && ${FROMINSTANCE-x} ]] || { echo "No From instance given"; usage; exit 1; }
[[ $TOINSTANCE && ${TOINSTANCE-x} ]] || { echo "No To instance given"; usage; exit 1; }
[[ $DRIVEID && ${DRIVEID-x} ]] || { echo "No Drive Letter given"; usage; exit 1; }

# Do we have a valid DRIVEID
driveidletters=`echo -n $DRIVEID | wc -c`
[[ $driveidletters -gt 1 ]] && { echo -e "Only specify one drive letter d-z."; exit 1; }
driveidcheck=`echo $DRIVEID | grep [d-z] | wc -c`
[[ $driveidcheck -gt 0 ]] || { echo -e "Drive Letter ${DRIVEID} is invalid. Please specify d-z."; exit 1; }


# Check to make sure instances are in the same availability zone
echo -n "Checking Availability Zone..."
FROMAZ=`ec2-describe-instances -v $FROMINSTANCE | grep "<availabilityZone>" | awk -F'<availabilityZone>' '{print$2}' | awk -F'<' '{print$1}'`
echo -n "$FROMINSTANCE is in $FROMAZ..."
TOAZ=`ec2-describe-instances -v $TOINSTANCE | grep "<availabilityZone>" | awk -F'<availabilityZone>' '{print$2}' | awk -F'<' '{print$1}'`
echo "$TOINSTANCE is in $TOAZ."
[[ $FROMAZ != $TOAZ ]] && { echo "Instances $FROMINSTANCE and $TOINSTANCE are not in the same availability zone."; exit 1; }

# Get Array Info
echo -n "Getting array info..."
SAVEIFS=$IFS
IFS=$'\n'
DRIVEARRAY=(`ec2-describe-volumes | grep $FROMINSTANCE | grep "/dev/sd${DRIVEID}"`)
IFS=$SAVEIFS
ARRAYSIZE=${#DRIVEARRAY[@]}  
# Check to make sure given drive letter exists on given from instance
[[ $ARRAYSIZE -gt 2 ]] || { echo -e "No array found as ${DRIVEID} on ${FROMINSTANCE}"; exit 1; }

echo -e "Moving an array of $ARRAYSIZE disks from $FROMINSTANCE to $TOINSTANCE in $FROMAZ.\n"

echo "Found $ARRAYSIZE volumes:"
for volume in "${DRIVEARRAY[@]}"
do
    VOL=`echo $volume | awk '{print$2}'`
    DEV=`echo $volume | awk '{print$4}'`
    echo "$VOL as $DEV"
    echo "$VOL:$DEV" >> /tmp/ec2raid-safetyfile.dat
done

echo ""

confirm && {

    echo "Moving EBS volumes...";
    for volume in "${DRIVEARRAY[@]}";
    do    
        VOL=`echo $volume | awk '{print$2}'`;
        DEV=`echo $volume | awk '{print$4}'`;
        echo -en "\nDetaching $VOL from ${FROMINSTANCE}...";
        ec2-detach-volume ${VOL};
        sleep 8;
        echo -en "\nAttaching ${VOL} to ${TOINSTANCE} as ${DEV}...";
        ec2-attach-volume ${VOL} -i ${TOINSTANCE} -d ${DEV};
    done;

    echo "Done. There is a backup file of the mapping in /tmp/ec2raid-safetyfile.dat.";

}
<file_sep>#!/bin/bash

########
##
##  Build RAID 10 array in Amazon EC2
## -----------------------------------
##
##  <NAME> <<EMAIL>>
##
##  Creates EBS volumes in Amazon EC2 and associates them with an EC2 instance for use as a RAID array
#

##  Requirements
## --------------
##
##  ec2-api-tools - http://aws.amazon.com/developertools/351
##
##  !!!!!!!!        !!!!!!!!
##  !!!!!!!! NOTICE !!!!!!!!  BEFORE CALLING THIS SCRIPT:
##  !!!!!!!!        !!!!!!!!
##
##  ec2-api-tools must be working and your environment variables set up:
##
##    AWS_ACCESS_KEY
##    AWS_SECRET_KEY
#

##  Usage
## -------
##
##  buildec2raid.sh -s <size> -z <zone> -i <instance> [-d drive letter] [-n number of disks] [-o iops] [-v]
##
##       size - the usable size of the raid array (in GB)
##       zone - the availability zone
##       instance - the ec2 instance id to attach to
##       drive letter (optional) - the drive letter to use in association with the instance (defaults to h)
##       number of disks (optional) - the number of disks to create for the array (defaults to 8, minimum 4)
##       iops (optional) - the requested number of I/O operations per second that the volume can support
##       -v to specify HVM instance (uses a different drive assignment scheme)
##

##  Examples
## ----------
##
##  ./buildec2raid.sh -s 1024 -z us-east-1a -i i-9i8u7y7y
##
##      - this would create a 1TB array in us-east-1a attached to i-918u7y7y
##
##  ./buildec2raid.sh -s 128 -z us-east-1d -i i-715e8e8v -o 100
##
##      - this would create a 128GB array in us-east-1d attached to i-715e8e8v with 100 IOPS per second provisioned for each volume in the array.
#

##
## The MIT License (MIT)
## Copyright (c) 2012-2014 <NAME>
##
## Permission is hereby granted, free of charge, to any person obtaining a
## copy of this software and associated documentation files (the "Software"),
## to deal in the Software without restriction, including without limitation
## the rights to use, copy, modify, merge, publish, distribute, sublicense,
## and/or sell copies of the Software, and to permit persons to whom the
## Software is furnished to do so, subject to the following conditions:
##
## The above copyright notice and this permission notice shall be included
## in all copies or substantial portions of the Software.
##
## THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
## IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
## FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
## AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
## LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
## FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
## DEALINGS IN THE SOFTWARE.
#

##
## VARIABLES
#

# Default Number of Disks for the array
DISKS=4

# Default Drive ID
DRIVEID="h"

# Is HVM
HVM=1

# IOPS
PROVIOPS=0

##
## END VARIABILE DEFINITIONS
#

##
## FUNCTIONS
#

# Confirmation Prompt
#
confirm () {

    read -r -p "${1:-Continue? [y/N]} " response
    case $response in
        [yY][eE][sS]|[yY]) true
        ;;
        *) false
        ;;
    esac
}

# Print Usage
#
usage() {

    echo -e "\nUsage: $0 -s <size> -z <availability zone> -i <instance> [-d <drive letter>] [-n <disks>] [-o <iops>] (-h for help)"

}

# Check for AWS Environment Vars as we can't do much without them
[[ $AWS_ACCESS_KEY && ${AWS_ACCESS_KEY-x} ]] || { echo  "AWS_ACCESS_KEY not defined. Please set up your AWS credentials. See http://docs.amazonwebservices.com/AWSEC2/latest/UserGuide/index.html?SettingUp_CommandLine.html for more information."; exit 1; }
[[ $AWS_SECRET_KEY && ${AWS_SECRET_KEY-x} ]] || { echo  "AWS_SECRET_KEY not defined. Please set up your AWS credentials. See http://docs.amazonwebservices.com/AWSEC2/latest/UserGuide/index.html?SettingUp_CommandLine.html for more information."; exit 1; }

# Process Command Line Args
while getopts ":d:s:z:i:n:o:h:v" optname
do

    case ${optname} in
         h|H) usage
              echo -e "\n\tRequired Arguments:\n"
              echo -e "\t-s <size> - the usable size of the raid array (in GB)"
              echo -e "\t-z <region> - the AWS region"
              echo -e "\t-i <instance> - the EC2 instance id to attach to"
              echo -e "\n\tOptional Arguments:\n"
              echo -e "\t-d <drive> - the drive identifier to use (defaults to h)"
              echo -e "\t-n <number of disks> - the number of disks to create in the array (defaults to 4)"
              echo -e "\t-o <iops> - the requested number of I/O operations per second that the volume can support"
              echo -e "\t-v specify PV instance (Defaults to HVM without this flag)"
              echo -e "\n"
              exit 0
              ;;
         s|S) TOTALSIZE=${OPTARG}
              ;;
         z|Z) AWSZONE=${OPTARG}
              ;;
         i|I) EC2INSTANCE=${OPTARG}
              ;;
         n|N) DISKS=${OPTARG}
              ;;
         o|O) IOPS=${OPTARG}
              PROVIOPS=1
              ;;
         d|D) DRIVEID=${OPTARG}
              ;;
         v|V) HVM=0
              ;;
         * )  echo "No such option ${optname}."
              usage
              exit 1
              ;;
     esac
done

# Do we have values for required arguments?
[[ $TOTALSIZE && ${TOTALSIZE-x} ]] || { echo "No size given"; usage; exit 1; }
[[ $AWSZONE && ${AWSZONE-x} ]] || { echo "No AWS Zone given"; usage; exit 1; }
[[ $EC2INSTANCE && ${EC2INSTANCE-x} ]] || { echo "No EC2 Instance given"; usage; exit 1; }

# Get list of AWS regions
AWSZONES=$(ec2-describe-availability-zones | awk '{printf("%s ", $2)}')
# Check given AWS region for validity
[[ "${AWSZONES}" =~ "${AWSZONE}" ]] || { echo -e "$AWSZONE is not a valid AWS zone.\n\nValid Zones: $AWSZONES\n"; exit 1; }

# Do we have a valid DRIVEID
driveidletters=$(echo -n $DRIVEID | wc -c)
[[ $driveidletters -gt 1 ]] && { echo -e "Only specify one drive letter d-z."; exit 1; }
driveidcheck=$(echo $DRIVEID | grep [d-z] | wc -c)
[[ $driveidcheck -gt 0 ]] || { echo -e "Drive Letter ${DRIVEID} is invalid. Please specify d-z."; exit 1; }

# Do we have the minimum number of disks for RAID 10
[[ $DISKS -gt 3 ]] || { echo -e "You need at least 4 disks for RAID10."; exit 1; }

# Make sure the instance is in the same zone
echo -n "Checking for instance $EC2INSTANCE in region ${AWSZONE}..."
# Get AWS endpoint for region
AWSURL=$(ec2-describe-regions | grep us-west-2 | awk '{print$3}')
AWSINSTANCECHECK=$(ec2-describe-instances | grep INSTANCE | grep $EC2INSTANCE | wc -c)
[[ $AWSINSTANCECHECK -gt 3 ]] || { echo -e "Instance ID: $EC2INSTANCE not found in ${AWSZONE}. Check your credentials, the instance id, and availability zone for the instance.\n\n"; exit 1; }

echo "found."
echo -n "Creating a $TOTALSIZE GB array in $AWSZONE for instance $EC2INSTANCE"

[[ $PROVIOPS -eq 1 ]] && { echo -n " with $IOPS I/O operations per second"; IOPSARGS="--type io1 --iops ${IOPS}"; }

echo "."

# Do the Math: 2X total capacity, each disk 2*(capacity)/(number of disks)
CAPACITY=$(expr $TOTALSIZE \* 2)
EACHDISK=$(expr $CAPACITY / $DISKS)

echo -e "This means a total of $CAPACITY GB in $DISKS EBS volumes of $EACHDISK GB each will be added to the instance.\n"

# Error check: IOPS volumes must be at least 10GB in size
[[ $PROVIOPS -eq 1 ]] && [[ $EACHDISK -lt 10 ]] && { echo -e "** EBS volumes with IOPS must be at least 10GB in size. Increase the array size or reduce the number of disks (-n <disks>)\n\n"; exit 1; }

confirm && {

   HVMDISKARRAY=$DRIVEID
   PVDISKARRAY="1"
   echo "Creating EBS Volumes...";

   for (( disk=1; disk<=$DISKS; disk++));
   do

     echo -en "\tCreating volume $disk of $DISKS...";

     # Create Volumes
     createvolume=$(ec2-create-volume --size ${EACHDISK} --availability-zone ${AWSZONE} ${IOPSARGS})
     # exit if it didn't work
     [[ $createvolume && ${createvolume-x} ]] || { echo "Volume Creation Unsuccessful. Exiting." exit 1; }

     # pause to allow amazon's api to catch up
     sleep 4
 
      # Associate with Instance, exit if unsuccessful
     echo -en "Associating with instance...\n\t";
     volume=$(echo $createvolume | awk '{print$2}');
     if [[ $HVM -eq 0 ]]
     then
        ec2-attach-volume $volume -i ${EC2INSTANCE} -d /dev/sd${DRIVEID}${disk} || { echo "Association of volume $volume to instance ${EC2INSTANCE} as /dev/sd${DRIVEID}${disk} failed! Exiting..."; exit 1; };
     else
        ec2-attach-volume $volume -i ${EC2INSTANCE} -d /dev/sd${DRIVEID} || { echo "Association of volume $volume to instance ${EC2INSTANCE} as /dev/sd/${DRIVEID} failed! Exiting..."; exit 1; };
        LASTDRIVEID=$DRIVEID
        newdriveletter=$(echo $DRIVEID | perl -nle 'print ++$_')
        DRIVEID=${newdriveletter}
     fi

   done;

   echo -en "EC2 volumes creation is complete. You can now log into the instance and create the raid array:\n\tmdadm --create -l10 -n$DISKS /dev/md0 ";
   if [[ $HVM -eq 1 ]]
   then
      echo "/dev/xvd[$HVMDISKARRAY-$LASTDRIVEID]"
   else
      echo "/dev/sd$DRIVEID[$PVDISKARRAY-$DISKS]"
   fi
}
<file_sep># Build RAID 10 array in Amazon EC2

Creates EBS volumes in Amazon EC2 and associates them with an EC2 instance for use as a RAID array

##  Requirements

### ec2-api-tools - http://aws.amazon.com/developertools/351

ec2-api-tools must be working and your environment variables set up:

* AWS_ACCESS_KEY
* AWS_SECRET_KEY

##  Usage

     $ buildec2raid.sh -s <size> -z <zone> -i <instance> [-d drive letter] [-n number of disks] [-o iops] -v 

* size - the usable size of the raid array (in GB)
* zone - the availability zone
* instance - the ec2 instance id to attach to
* drive letter (optional) - the drive letter to use in association with the instance (defaults to h)
* number of disks (optional) - the number of disks to create for the array (defaults to 4)
* iops (optional) - the requested number of I/O operations per second that the volume can support
* -v to specify PV instances (HVM instances use a different drive assignment scheme than PV)

##  Example

     $ ./buildec2raid.sh -s 1024 -z us-east-1a -i i-9i8u7y7y

This would create a 1TB array in us-east-1a attached to i-918u7y7y

      $ ./buildec2raid.sh -s 1024 -z us-east-1a -i i-9i8u7y7y -n 6 -o 100 -d j

This would create a 1TB array in us-east-1a attached to i-9i8u7y7y using 6 drives provisioned with 100 IOPS per second and using j as the drive letter
(/dev/sdj and /dev/xvdj)

## More information

After completing the creation of the EBS volumes using this script, you can log into the instance and initialize the raid array. For example:

     $ mdadm --create -l10 -n8 /dev/md0 /dev/xvdh*

This creates a RAID 10 volume from 8 EBS disks. For more information on software raid, see https://raid.wiki.kernel.org/index.php/Linux_Raid

## License

This script is distributed under the MIT License (see LICENSE)

## Migrating a RAID array to another instance

Typically if you want to migrate data between hosts within the same availability zone, you would create a new array on the destination instance and copy or rsync the data over. 

However the <strong>moveec2raid.sh</strong> script in this repository will migrate a RAID array to another instance. 

The syntax is:

     $  ./moveec2raid.sh -f <from instance> -t <to instance> -d <drive letter>

All three fields are required. The drive letter is the drive identifier of the origin instance. ie. /dev/xvdh*.

<em>The script assumes that all volumes mounted with that drive letter are part of your RAID array and they will be migrated in place.</em> If you used the <strong>buildec2raid.sh</strong> script, that is how the array is set up. 

Make note of your <em>/etc/mdadm.conf</em> file as it contains the identification information for your array. 


[![Analytics](https://ga-beacon.appspot.com/UA-24846718-5/buildec2raid/README.md)](https://github.com/igrigorik/ga-beacon)
