figo12396/llama · Hugging Face

#!/bin/bash

read -p "Enter the URL from email: " https://download.llamameta.net/*?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoieD8%7ENVhcdTAwMDNqXC8iLCJSZXNvdXJjZSI6Imh0dHBzOlwvXC9kb3dubG9hZC5sbGFtYW1ldGEubmV0XC8qIiwiQ29uZGl0aW9uIjp7IkRhdGVMZXNzVGhhbiI6eyJBV1M6RXBvY2hUaW1lIjoxNjkyMTk2NTA5fX19XX0_&Signature=UH2Au8t6FRQ6V3GNf0YAJVNk4Mr5z7FjLq5T3MXyYgiWb5RhIq4PPz3-qrLZ%7ETmyb%7E7GMfmXLb7UCpPv2zBeW3lP-OBsbFGusVgoOi07h8c1X3E0NARtl1rjbT2kuhbK183WC4WIaXPmOvh6WAP3s3Hc6H8OOWUNjrSm7aySSeCkZ%7EFgw2Y4m2SMgrC-1IBQTeU2XJ1AVrHJUcF0%7Ed5DBgwij3MrAlMjkOQ7EcQr8mo9ypC84%7EBIYUnGXyZeccHu6mrMrRy%7EMWWNkOok%7EnV33uqwgTHNqx6MIpVDWrMISy2%7EB8GnDBVG2iLEjr%7Ew5lY493TEEfA9Pcv2ZcN3gbUYyw__&Key-Pair-Id=K15QRJLYKIFSLZ echo "" read -p "Enter the list of models to download without spaces (7B,13B,70B,7B-chat,13B-chat,70B-chat), or press Enter for all: " MODEL_SIZE TARGET_FOLDER="." # where all files should end up mkdir -p ${TARGET_FOLDER}

if [[ $MODEL_SIZE == "" ]]; then MODEL_SIZE="7B,13B,70B,7B-chat,13B-chat,70B-chat" fi

echo "Downloading LICENSE and Acceptable Usage Policy" wget ${PRESIGNED_URL/''/"LICENSE"} -O ${TARGET_FOLDER}"/LICENSE" wget ${PRESIGNED_URL/''/"USE_POLICY.md"} -O ${TARGET_FOLDER}"/USE_POLICY.md"

echo "Downloading tokenizer" wget ${PRESIGNED_URL/''/"tokenizer.model"} -O ${TARGET_FOLDER}"/tokenizer.model" wget ${PRESIGNED_URL/''/"tokenizer_checklist.chk"} -O ${TARGET_FOLDER}"/tokenizer_checklist.chk" (cd ${TARGET_FOLDER} && md5sum -c tokenizer_checklist.chk)

for m in ${MODEL_SIZE//,/ } do if [[ $m == "7B" ]]; then SHARD=0 MODEL_PATH="llama-2-7b" elif [[ $m == "7B-chat" ]]; then SHARD=0 MODEL_PATH="llama-2-7b-chat" elif [[ $m == "13B" ]]; then SHARD=1 MODEL_PATH="llama-2-13b" elif [[ $m == "13B-chat" ]]; then SHARD=1 MODEL_PATH="llama-2-13b-chat" elif [[ $m == "70B" ]]; then SHARD=7 MODEL_PATH="llama-2-70b" elif [[ $m == "70B-chat" ]]; then SHARD=7 MODEL_PATH="llama-2-70b-chat" fi

echo "Downloading ${MODEL_PATH}"
mkdir -p ${TARGET_FOLDER}"/${MODEL_PATH}"

for s in $(seq -f "0%g" 0 ${SHARD})
do
    wget ${PRESIGNED_URL/'*'/"${MODEL_PATH}/consolidated.${s}.pth"} -O ${TARGET_FOLDER}"/${MODEL_PATH}/consolidated.${s}.pth"
done

wget ${PRESIGNED_URL/'*'/"${MODEL_PATH}/params.json"} -O ${TARGET_FOLDER}"/${MODEL_PATH}/params.json"
wget ${PRESIGNED_URL/'*'/"${MODEL_PATH}/checklist.chk"} -O ${TARGET_FOLDER}"/${MODEL_PATH}/checklist.chk"
echo "Checking checksums"
(cd ${TARGET_FOLDER}"/${MODEL_PATH}" && md5sum -c checklist.chk)

done