Cannot use apply_chat_template() because tokenizer.chat_template is not set and no template argument was passed!

#3
by CCChen - opened

pipeline = transformers.pipeline(
"text-generation",
model=text_model,
tokenizer = tokenizer,
model_kwargs={"torch_dtype": torch.bfloat16},
device_map="auto",
)

    messages = [
        {"role": "system", "content": "You are a pirate chatbot who always responds in pirate speak!"},
        {"role": "user", "content": "Who are you?"},
    ]

    prompt = pipeline.tokenizer.apply_chat_template(
        messages, 
        tokenize=False, 
        add_generation_prompt=True
)
    
    terminators = [
        pipeline.tokenizer.eos_token_id,
        pipeline.tokenizer.convert_tokens_to_ids("<|eot_id|>")
    ]

    outputs = pipeline(
        prompt,
        max_new_tokens=256,
        eos_token_id=terminators,
        do_sample=True,
        temperature=0.6,
        top_p=0.9,
    )
    print(outputs[0]["generated_text"][len(prompt):])

Sign up or log in to comment