llava_attack.py

import argparse
import torch
import os
from torchvision.utils import save_image
import numpy as np
from PIL import Image
import cv2
import torchvision

def parse_args():

    parser = argparse.ArgumentParser(description="Demo")
    parser.add_argument("--model-path", type=str, default="ckpts/llava_llama_2_13b_chat_freeze")
    parser.add_argument("--model-base", type=str, default=None)
    parser.add_argument("--gpu_id", type=int, default=0, help="specify the gpu to load the model.")
    parser.add_argument("--n_iters", type=int, default=5000, help="specify the number of iterations for attack.")
    parser.add_argument('--eps', type=int, default=64, help="epsilon of the attack budget")
    parser.add_argument('--alpha', type=int, default=1, help="step_size of the attack")
    parser.add_argument("--constrained", default=False, action='store_true')

    parser.add_argument("--save_dir", type=str, default='output',
                        help="save directory")

    args = parser.parse_args()
    return args

def load_image(image_path):
    image = Image.open(image_path).convert('RGB')
    return image

# ========================================
#             Model Initialization
# ========================================

print('>>> Initializing Models')

from llava.utils import get_model
args = parse_args()

print('model = ', args.model_path)

tokenizer, model, image_processor, model_name = get_model(args)
model.eval()
print('[Initialization Finished]\n')


if not os.path.exists(args.save_dir):
    os.mkdir(args.save_dir)

lines = open('harmful_corpus/harmful_strings.csv').read().split("\n")
targets = [li for li in lines if len(li)>0]
print(targets)

files = []
for file_name in os.listdir('datasets/coco/val2017/'):
    file_path = os.path.join('datasets/coco/val2017/', file_name)
    if os.path.isfile(file_path):
        files.append(file_path)
print(files[:2])

for i in range(25):
    image_path = files[np.random.randint(len(files))]
    original_image = load_image(image_path)
    original_image.save(args.save_dir+'original_'+str(i)+'.bmp')
    image = image_processor.preprocess(original_image, return_tensors='pt')['pixel_values'].cuda()
    print(image.shape)

    from llava_utils import visual_attacker

    print('device = ', model.device)
    my_attacker = visual_attacker.Attacker(args, model, tokenizer, targets, device=model.device, image_processor=image_processor)

    from llava_utils import prompt_wrapper
    text_prompt_template = prompt_wrapper.prepare_text_prompt('')
    print(text_prompt_template)

    if not args.constrained:
        print('[unconstrained]')
        adv_img_prompt = my_attacker.attack_unconstrained(text_prompt_template,
                                                                img=image, batch_size = 1,
                                                                num_iter=args.n_iters, alpha=args.alpha/255)

    else:
        adv_img_prompt = my_attacker.attack_constrained(text_prompt_template,
                                                                img=image, batch_size= 1,
                                                                num_iter=args.n_iters, alpha=args.alpha / 255,
                                                                epsilon=args.eps / 255)

    save_image(adv_img_prompt, args.save_dir+'adversarial_'+str(i)+'.bmp')
print('[Done]')