llava_baseline.py

import argparse
import os
import random

import cv2
import numpy as np
import torch
import torch.backends.cudnn as cudnn
from PIL import Image
import json
import pandas as pd

def rtp_read(text_file):
    dataset = []
    lines = open(text_file).read().split("\n")
    for li in lines:
        obj = json.loads(li)
        if obj['challenging']:
            dataset.append(obj['prompt']['text'])
    return dataset

def parse_args():

    parser = argparse.ArgumentParser(description="Demo")
    parser.add_argument("--model-path", type=str, default="ckpts/llava_llama_2_13b_chat_freeze")
    parser.add_argument("--model-base", type=str, default=None)
    parser.add_argument("--mode", type=int, default=1, help="testing against benign prompts (mode=0) and adversarial prompts (mode=1).")
    parser.add_argument("--image_path", type=str, default='./image.bmp',
                        help="Image file")
    parser.add_argument("--output_file", type=str, default='./result.jsonl',
                        help="Output file.")


    

    args = parser.parse_args()
    return args


def load_image(image_path):
    image = Image.open(image_path).convert('RGB')
    return image


# ========================================
#             Model Initialization
# ========================================

print('>>> Initializing Models')

from llava.utils import get_model
args = parse_args()

print('model = ', args.model_path)

tokenizer, model, image_processor, model_name = get_model(args)
model = model.cuda()
print(model)
print(model_name)
model.eval()

print('[Initialization Finished]\n')


from llava_utils import prompt_wrapper, generator


my_generator = generator.Generator(model=model, tokenizer=tokenizer)


# ========================================
#             Inference
# ========================================

##  TODO: expose interface.
if args.mode==1:
    prompts = rtp_read('harmful_corpus/rtp_prompts.jsonl')
else:
    lines = open('prompts/benign').read().split("\n")
    prompts = [li for li in lines]

#text_prompt = 'Please extend the following sentence: %s'
text_prompt = '%s'

out = []
with torch.no_grad():

    for i, user_message in enumerate(prompts):

        print(f" ----- {i} ----")
        print(" -- prompt: ---")

        print(text_prompt % user_message)

        # blur kernel baseline
        #image = cv2.imread(args.image_path+str(np.random.randint(25))+'.bmp')
        #image = cv2.blur(image,(3,3))

        # compression decompression baseline
        image = load_image(args.image_path+str(np.random.randint(25))+'.bmp')
        image = image.save('baseline/compressed_q10.jpg', quality=10)
        image = load_image('baseline/compressed_q10.jpg')

        safe_image = image_processor.preprocess(image, return_tensors='pt')['pixel_values'].cuda()
        
        text_prompt_template = prompt_wrapper.prepare_text_prompt(text_prompt % user_message)
        print(text_prompt_template)
        prompt = prompt_wrapper.Prompt(model, tokenizer, text_prompts=text_prompt_template, device=model.device)

        response = my_generator.generate(prompt, safe_image)

        print(" -- continuation: ---")
        print(response)
        out.append({'prompt': user_message, 'continuation': response})
        print()

with open(args.output_file, 'w') as f:
    f.write(json.dumps({
        "args": vars(args),
        "prompt": text_prompt
    }))
    f.write("\n")

    for li in out:
        f.write(json.dumps(li))
        f.write("\n")