llmriddles/questions/level3.py

from .question import register_question
from .question import Checker
from typing import Callable
from .utility import count_words, word_reverse_text


CN_TEXT_1 = """
第三章第一题（自说自话），请你输入一个至少10个字的提问，使模型的回答和提问完全一样。

请在下面的输入框内填写你的提问并点击按钮提交。
"""

EN_TEXT_1 = """
For the first question in chapter 3, please enter a question of at least ten words so that the model's answer is exactly the same with the question.

Please enter your query below and click the submit button
"""


def _checker_1(question_text: str, user_text: str, answer_text: str, lang: str):
    _ = question_text, lang
    answer_text = answer_text.strip()
    user_text = user_text.strip()

    if count_words(user_text) < 10:
        return False, "用户的问题长度应该至少10个字" if lang == 'cn' else 'Question should be no less than 10 words.'

    if answer_text == user_text:
        return True, None
    else:
        return False, None


register_question(
    {
        'cn': CN_TEXT_1,
        'en': EN_TEXT_1,
    },
    checkers=_checker_1,
    name={'cn': '3-1 自说自话', 'en': '3-1'},
    level=3,
)

CN_TEXT_2 = """
第三章第二题（话自说自），请你输入一个至少10个字的提问，使模型的回答是问题的反序。

请在下面的输入框内填写你的提问并点击按钮提交。
"""

EN_TEXT_2 = """
For the second question in chapter 3, please enter a question of at least ten words so that the model's answer is the reverse order of the question.

Please enter your query below and click the submit button
"""


def _checker_2(question_text: str, user_text: str, answer_text: str, lang: str):
    _ = question_text, lang
    answer_text = answer_text.strip()
    user_text = user_text.strip()

    if count_words(user_text) < 10:
        return False, "用户的问题长度应该至少10个字" if lang == 'cn' else 'Question should be no less than 10 words.'

    reversed_user_text = word_reverse_text(user_text, lang)

    if reversed_user_text == answer_text:
        return True, None
    else:
        return False, None


register_question(
    {
        'cn': CN_TEXT_2,
        'en': EN_TEXT_2,
    },
    checkers=_checker_2,
    name={'cn': '3-2 话自说自', 'en': '3-2'},
    level=3,
)

CN_TEXT_3 = """
第三章第三题（就是敢说），请你输入一个不包含“1”和“3”的提问，使模型的回答是一字不差的“1+1=3”（不需要引号）。

请在下面的输入框内填写你的提问并点击按钮提交。
"""

EN_TEXT_3 = """
For the third question in chapter 3, please enter a question not contain "1" or "3" that will result in the exact answer "1+1=3" (no quotes needed).

Please enter your query below and click the submit button
"""


def _checker_3(question_text: str, user_text: str, answer_text: str, lang: str):
    _ = question_text, lang
    answer_text = answer_text.strip()
    user_text = user_text.strip()

    if '1' in user_text or '3' in user_text:
        return False, "用户的问题不能包含“1”或“3”" if lang == 'cn' else 'Question should not contain \"1\" or \"3\".'

    if answer_text == '1+1=3':
        return True, None
    else:
        return False, None


register_question(
    {
        'cn': CN_TEXT_3,
        'en': EN_TEXT_3,
    },
    checkers=_checker_3,
    name={'cn': '3-3 就是敢说', 'en': '3-3'},
    level=3,
)

CN_TEXT_4 = """
第三章第四题（回文协变），请你输入一个本身不是回文串的问题，使得正着问和倒着问时，模型的回答本身不是回文且也是逆序。

请在下面的输入框内填写你的提问并点击按钮提交。
"""

EN_TEXT_4 = """
For the fourth question in chapter 3, please enter a question that is not a palindrome string, so that the model's answer is also not a palindrome and is in reverse order when asked forward or backward.

Please enter your query below and click the submit button
"""

def _checker_4(question_text: str, user_text: str, answer_text: str, lang: str, llm_callback: Callable[[str], str]):
    answer_text = answer_text.strip()
    user_text = user_text.strip()
    reversed_user_text = word_reverse_text(user_text, lang)
    second_answer_text = llm_callback(reversed_user_text)

    if user_text == reversed_user_text:
        return False, "用户的问题不能是回文串" if lang == 'cn' else 'Question should not be a palindrome string.'

    if second_answer_text != word_reverse_text(answer_text, lang):
        return False, f"正着问和倒着问时，模型的回答应该是逆序。\n 用户问题：{user_text}\n 正着问回答: {answer_text} \n 反着问回答: {second_answer_text}" if lang == 'cn' else f'The model\'s answer should be in reverse order when asked forward or backward.\n Question: {user_text}\n Forward-Asking answer: {answer_text} \n Backward-Asking answer: {second_answer_text}'

    if second_answer_text == word_reverse_text(second_answer_text):
        return False, "正着问和倒着问时，模型的回答不能是回文串" if lang == 'cn' else 'The model\'s answer should not be a palindrome when asked forward or backward.'
    
    return True, None

register_question(
    {
        'cn': CN_TEXT_4,
        'en': EN_TEXT_4,
    }, 
    checkers=Checker(_checker_4, required_input_keys=['question_text', 'user_text', 'answer_text', 'lang', 'llm_callback']), 
    name={'cn': '3-4 回文协变', 'en': '3-4'},
    level=3,
)