init

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+*

config.json

@@ -0,0 +1,46 @@
+{
+  "_name_or_path": "/home/juntao/Models/LanguageBind/Video-LLaVA-7B",
+  "architectures": [
+    "LlavaLlamaForScore"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "freeze_mm_mlp_adapter": false,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "image_aspect_ratio": "pad",
+  "image_grid_pinpoints": null,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "mm_hidden_size": 1024,
+  "mm_image_tower": "LanguageBind/LanguageBind_Image",
+  "mm_projector_lr": null,
+  "mm_projector_type": "mlp2x_gelu",
+  "mm_use_im_patch_token": false,
+  "mm_use_im_start_end": false,
+  "mm_video_tower": "LanguageBind/LanguageBind_Video_merge",
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -2,
+  "model_type": "llava_score",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "tokenizer_model_max_length": 3072,
+  "tokenizer_padding_side": "right",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.1",
+  "tune_mm_mlp_adapter": false,
+  "use_cache": true,
+  "use_mm_proj": true,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "max_length": 4096,
+  "pad_token_id": 0,
+  "temperature": null,
+  "top_p": null,
+  "transformers_version": "4.44.1"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:365c45380f2477dcfce97b585c346e11d252a6758a6200c08cdb23d936ff3632
+size 4938985352

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2ad385bef4a52546c760da667fefbc3c9e758e2c7d3c1fc70e1047e98364eee
+size 4947390880

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f4f79e3b856e275be82ad5f385cc2ec7bdda70db4e7278761997809ad0c8cf0
+size 4785196224

script.sh

@@ -0,0 +1,173 @@
+#!/bin/bash
+#
+# Copyright 2024 PKU-Alignment Team. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+if [ -z "${BASH_VERSION}" ]; then
+    echo "Please use bash to run this script." >&2
+    exit 1
+fi
+
+VIDEO_DIR="/home/juntao/Data/safe-sora/videos"
+TRAIN_DATA_PATH="/home/juntao/Data/safe-sora/unsafe_pairs/config-train.json"
+EVAL_DATA_PATH="/home/juntao/Data/safe-sora/unsafe_pairs/config-test.json"
+MODEL_NAME_OR_PATH="/home/juntao/Models/LanguageBind/Video-LLaVA-7B"
+MM_MLP_ADAPTER_PATH="/home/juntao/Models/LanguageBind/Video-LLaVA-Pretrain-7B/mm_projector.bin"
+OUTPUT_DIR="./outputs/cost"
+DIMENSION="harmlessness"
+
+# VIDEO_DIR="/home/juntao/Projects/safe-sora/data/SafeSora/videos"
+# TRAIN_DATA_PATH="/home/juntao/Projects/safe-sora/data/SafeSora/config-train.json.gz"
+# EVAL_DATA_PATH="/home/juntao/Projects/safe-sora/data/SafeSora/config-test.json.gz"
+# MODEL_NAME_OR_PATH="/home/juntao/Models/LanguageBind/Video-LLaVA-7B"
+# MM_MLP_ADAPTER_PATH="/home/juntao/Models/LanguageBind/Video-LLaVA-Pretrain-7B/mm_projector.bin"
+# OUTPUT_DIR="/home/juntao/Projects/Learning/safe-sora/examples/outputs/goodgood"
+# DIMENSION="helpfulness"
+
+while [[ "$#" -gt 0 ]]; do
+    arg="$1"
+    shift
+    case "${arg}" in
+    --video_dir)
+        VIDEO_DIR="$1"
+        shift
+        ;;
+    --video_dir=*)
+        VIDEO_DIR="${arg#*=}"
+        ;;
+    --train_data_path)
+        TRAIN_DATA_PATH="$1"
+        shift
+        ;;
+    --train_data_path=*)
+        TRAIN_DATA_PATH="${arg#*=}"
+        ;;
+    --eval_data_path)
+        EVAL_DATA_PATH="$1"
+        shift
+        ;;
+    --eval_data_path=*)
+        EVAL_DATA_PATH="${arg#*=}"
+        ;;
+    --model_name_or_path)
+        MODEL_NAME_OR_PATH="$1"
+        shift
+        ;;
+    --model_name_or_path=*)
+        MODEL_NAME_OR_PATH="${arg#*=}"
+        ;;
+    --mm_mlp_adapter_path)
+        MM_MLP_ADAPTER_PATH="$1"
+        shift
+        ;;
+    --mm_mlp_adapter_path=*)
+        MM_MLP_ADAPTER_PATH="${arg#*=}"
+        ;;
+    --output_dir)
+        OUTPUT_DIR="$1"
+        shift
+        ;;
+    --output_dir=*)
+        OUTPUT_DIR="${arg#*=}"
+        ;;
+    --dimension)
+        DIMENSION="$1"
+        shift
+        ;;
+    --dimension=*)
+        DIMENSION="${arg#*=}"
+        ;;
+    *)
+        echo "Unknown parameter passed: '${arg}'" >&2
+        exit 1
+        ;;
+    esac
+done
+
+if [[ ! "helpfulness harmlessness instruction_following correctness informativeness aesthetics" =~ (^|[[:space:]])"${DIMENSION}"($|[[:space:]]) ]]; then
+    echo "Invalid dimension: ${DIMENSION}, should be one of 'helpfulness', 'harmlessness', 'instruction_following', 'correctness', 'informativeness', 'aesthetics'." >&2
+    exit 1
+fi
+
+IMAGE_DIR="${VIDEO_DIR}"
+RUN_NAME="reward-${DIMENSION}"
+OUTPUT_DIR="${OUTPUT_DIR}/${RUN_NAME}"
+
+mkdir -p "${OUTPUT_DIR}"
+OUTPUT_DIR="$(cd "${OUTPUT_DIR}" &>/dev/null && pwd)"
+if [[ ! -f "${OUTPUT_DIR}/.gitignore" ]]; then
+    echo '*' >"${OUTPUT_DIR}/.gitignore"
+fi
+
+cp -f "$0" "${OUTPUT_DIR}/script.sh"
+
+MASTER_PORT_START=10000
+MASTER_PORT_END=65535
+MASTER_PORT="$(
+    comm -23 \
+        <(seq "${MASTER_PORT_START}" "${MASTER_PORT_END}" | sort) \
+        <(ss -Htan | awk '{ print $4 }' | awk -F ':' '{ print $NF }' | sort -u) |
+        shuf | head -n 1
+)"
+
+exec 1> >(tee "${OUTPUT_DIR}/stdout.log" >&1) 2> >(tee "${OUTPUT_DIR}/stderr.log" >&2)
+
+deepspeed --master_port="${MASTER_PORT}" examples/reward_model/train_cost.py \
+    --deepspeed examples/scripts/ds_zero2.json \
+    --version v1 \
+    --run_name "${RUN_NAME}" \
+    --model_name_or_path "${MODEL_NAME_OR_PATH}" \
+    --train_data_path "${TRAIN_DATA_PATH}" \
+    --eval_data_path "${EVAL_DATA_PATH}" \
+    --preference_dimension "${DIMENSION}" \
+    --image_dir "${IMAGE_DIR}" \
+    --video_dir "${VIDEO_DIR}" \
+    --image_tower LanguageBind/LanguageBind_Image \
+    --video_tower LanguageBind/LanguageBind_Video_merge \
+    --mm_projector_type mlp2x_gelu \
+    --pretrain_mm_mlp_adapter "${MM_MLP_ADAPTER_PATH}" \
+    --mm_vision_select_layer -2 \
+    --mm_use_im_start_end False \
+    --mm_use_im_patch_token False \
+    --image_aspect_ratio pad \
+    --group_by_modality_length True \
+    --output_dir "${OUTPUT_DIR}" \
+    --cache_dir "./models/cache_dir" \
+    --num_train_epochs 4 \
+    --per_device_train_batch_size 8 \
+    --per_device_eval_batch_size 8 \
+    --gradient_accumulation_steps 1 \
+    --evaluation_strategy "steps" \
+    --eval_steps 0.0499 \
+    --load_best_model_at_end True \
+    --metric_for_best_model "accuracy" \
+    --greater_is_better True \
+    --logging_first_step True \
+    --save_strategy "steps" \
+    --save_steps 0.0499 \
+    --save_total_limit 1 \
+    --learning_rate 2e-5 \
+    --weight_decay 0.1 \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --model_max_length 2048 \
+    --tokenizer_model_max_length 3072 \
+    --gradient_checkpointing True \
+    --dataloader_num_workers 8 \
+    --report_to wandb \
+    --bf16 True \
+    --tf32 True \
+    --num_frames 8

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<unk>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 2048,
+  "pad_token": "<unk>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32d0cbb2c81098a0718e1bb3c5fdd12551f4df69815179f5f7286b9af0288021
+size 7224