Some Question About MMbench Prompt

[lhqxcc]

Thank you for your outstanding work. During the reproduction process, I used the Pink_Object weights you provided to evaluate on MMBench, but the results showed a significant deviation from those in the paper, as shown in the following image:

I believe this might be because I didn't use the corresponding prompts. Could you please provide the prompts you used for evaluating MMBench? Thank you very much.

[SY-Xuan]

Thanks for your interest in our work. You can use the prompt like:

select_question = random.choice(ChoiceQuestionAnswer)
select_question = select_question.replace(" <image>", "")
select_question = select_question.replace("<question>", "{}{}{}".format(BEGIN_QUESTION, question.replace("Question: ", ""), END_QUESTION))
select_question = select_question.replace("<option>", "{}{}{}".format(BEGIN_OPTIONS, options_prompt, END_OPTIONS))

conv.append_message(conv.roles[0], select_question)
conv.append_message(conv.roles[1], None)
conv.set_system(PREFIX_IMAGE + image_token_len * DEFAULT_IMAGE_PATCH_TOKEN)

However, I need to point out that the version of MMBench has been changed. Moreover, to improve the performance on MMBench, we use option shuffle tricks, which means that for each question, we shuffle the options many times and use the final prediction results.

[lhqxcc]

Thank you very much for your response. I referred to your reply and the LLaVA evaluation MMBench code to write the following evaluation code for the PINK model:
`
def eval_model(args):

model_name = os.path.expanduser(args.model_name)
tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False, padding_side='left')
patch_config(model_name)
config = AutoConfig.from_pretrained(model_name, use_cache=True)
if config.llama_path != model_name:
    # need to merge parameters
    llama_path = config.llama_path

    weight_map_index = json.load(open(os.path.join(llama_path, "pytorch_model.bin.index.json"), "r"))
    shard_files = list(set(weight_map_index["weight_map"].values()))
    loaded_keys = weight_map_index["weight_map"].keys()
    state_dict = {}
    for index, shard_file in enumerate(shard_files):
        state_dict.update(torch.load(os.path.join(llama_path, shard_file), map_location="cpu"))
    peft_parameters = torch.load(os.path.join(model_name, "saved_parameters.pth"), map_location="cpu")
    for k, v in peft_parameters.items():
        state_dict[k] = v
else:
    state_dict = None

model = AutoModelForCausalLM.from_pretrained(None, config=config, state_dict=state_dict)
for name, param in model.model.named_parameters():
    if not ("adapter_" in name or "lora_" in name):
        param.data = param.data.half()
model.lm_head.to(torch.float16)
model = model.cuda()
crop_size = model.config.crop_size
norm_mean = (0.48145466, 0.4578275, 0.40821073)
norm_std = (0.26862954, 0.26130258, 0.27577711)
# NOTE: fix the data augmentation parameters ugly!!!
image_processor = transforms.Compose(
[
    transforms.Resize((crop_size, crop_size), interpolation=InterpolationMode.BICUBIC),
    transforms.ToTensor(),
    transforms.Normalize(norm_mean, norm_std),
])
image_token_len = model.config.num_patches
model.eval()
questions = pd.read_table(os.path.expanduser(args.question_file))
questions = get_chunk(questions, args.num_chunks, args.chunk_idx)
answers_file = os.path.expanduser(args.answers_file)
os.makedirs(os.path.dirname(answers_file), exist_ok=True)
ans_file = open(answers_file, "w")
conv = conv_templates[args.conv_mode].copy()
cnt = -1
for index, row in tqdm(questions.iterrows(), total=len(questions)):
    options = get_options(row, all_options)
    cur_option_char = all_options[:len(options)]
    if args.all_rounds:
        num_rounds = len(options)
    else:
        num_rounds = 1

    for round_idx in range(num_rounds):
        cnt += 1
        idx = row['index']
        question = row['question']
        hint = row['hint']
        image = load_image_from_base64(row['image'])

        # 设计相应的prompt
        select_question = random.choice(ChoiceQuestionAnswerNew)
        select_question = select_question.replace("<image>", "")

        if not is_none(hint):
            select_question = select_question.replace("<question>", "{}{}{}".format(BEGIN_QUESTION, question, END_QUESTION))
            select_question = select_question.replace("<hint>", "{}".format(hint))
        else:
            select_question = select_question.replace("<question>", "{}{}{}".format(BEGIN_QUESTION, question, END_QUESTION))
            select_question = select_question.replace("<hint>", "")     

        prompt_option = ""
        for option_char, option in zip(all_options[:len(options)], options):
            prompt_option += '\n' + option_char + '. ' + option

        select_question = select_question.replace("<option>", "{}{}{}".format(BEGIN_OPTIONS, prompt_option, END_OPTIONS))

        if args.single_pred_prompt:
            if args.lang == 'cn':
                select_question = select_question + '\n' + "请直接回答选项字母。"
            else:
                select_question = select_question + '\n' + "Answer with the option's letter from the given choices directly."

        cur_prompt = select_question
        conv.messages = []
        conv.set_system(PREFIX_IMAGE + image_token_len * DEFAULT_IMAGE_PATCH_TOKEN)
        conv.append_message(conv.roles[0], select_question)
        conv.append_message(conv.roles[1], None)
        prompt = conv.get_prompt()

        image_tensor = image_processor(image)
        image_tensor = image_tensor.unsqueeze(0).cuda()

        tokenized_output = tokenizer(
            prompt,
            return_tensors="pt",
            padding="longest",
            max_length=tokenizer.model_max_length,
            truncation=True,
        )

        input_ids = torch.as_tensor(tokenized_output.input_ids).cuda()
        attention_mask = torch.as_tensor(tokenized_output.attention_mask).cuda()


        with torch.inference_mode():
            output_ids = model.generate(
                input_ids,
                images=image_tensor,
                has_images=[True],
                attention_mask=attention_mask,
                do_sample=False,
                num_beams=1,
                temperature=0.7,
                max_new_tokens=1024,
                )

        for input_id, output_id in zip(input_ids, output_ids):
            input_token_len = input_id.shape[0]
            n_diff_input_output = (input_id != output_id[:input_token_len]).sum().item()
            if n_diff_input_output > 0:
                print('[Warning] Sample output_ids are not the same as the input_ids')
            output = tokenizer.batch_decode(output_id[input_token_len:].unsqueeze(0), skip_special_tokens=True)[0]
            output = output.strip()

            if args.verbose:
                print(f"Output: {output}")
                print(f"Current answer: {current_answer}, Original answer: {original_answer}")

        ans_id = shortuuid.uuid()
        ans_file.write(json.dumps({
            "question_id": idx,
            "round_id": round_idx,
            "prompt": cur_prompt,
            "text": output,
            "options": options,
            "option_char": cur_option_char,
            "answer_id": ans_id,
            "model_id": model_name,
            "metadata": {}
        }) + "\n")
        ans_file.flush()

        options = options[1:] + options[:1]
        cur_option_char = cur_option_char[1:] + cur_option_char[:1]`

Unfortunately, the results still differ from what was reported in the paper. Is this due to an issue with my envaluation code or with the MMBench version? By the way， I use the weights of pink_base you provided.

A_Overall (dev) | np.float64(0.5893470790378007)
B_AR (dev) | np.float64(0.6432160804020101)
B_CP (dev) | np.float64(0.7297297297297297)
B_FP-C (dev) | np.float64(0.5314685314685315)
B_FP-S (dev) | np.float64(0.5426621160409556)
B_LR (dev) | np.float64(0.3305084745762712)
B_RR (dev) | np.float64(0.591304347826087)

I look forward to your reply.

[SY-Xuan]

You can try option shuffle tricks. For example, with the order of options (A. 1 B. 2 C. 3, A.2 B. 3 C. 1, A. 3, B. 1, C.2 ), you can make three predictions and select the most selected option.

[lhqxcc]

Thank you very much for your answer. May your academic endeavors flourish.