Update model to NTQAI/Nxcode-CQ-7B-orpo

app.py

@@ -2,10 +2,8 @@ import gradio as gr
 from huggingface_hub import login
 from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
 from threading import Thread
-import torch
 
-MODEL = "m-a-p/OpenCodeInterpreter-DS-33B"
-CHAT_TEMPLATE = "{%- set found_item = false -%}\n{%- for message in messages -%}\n    {%- if message['role'] == 'system' -%}\n        {%- set found_item = true -%}\n    {%- endif -%}\n{%- endfor -%}\n{%- if not found_item -%}\n{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer.\\n'}}\n{%- endif %}\n{%- for message in messages %}\n    {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n    {%- else %}\n        {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n        {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|EOT|>\\n'}}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{{'### Response:\\n'}}\n"
+MODEL = "NTQAI/Nxcode-CQ-7B-orpo"
 
 system_message = "You are a computer programmer that can translate python code to C++ in order to improve performance"
 
@@ -22,31 +20,26 @@ def messages_for(python):
     ]
 
 tokenizer = AutoTokenizer.from_pretrained(MODEL)
-tokenizer.chat_template = CHAT_TEMPLATE
-model = AutoModelForCausalLM.from_pretrained(MODEL, torch_dtype=torch.bfloat16, device_map="auto")
-model.eval()
+model = AutoModelForCausalLM.from_pretrained(MODEL, torch_dtype="auto", device_map="auto")
 
 decode_kwargs = dict(skip_special_tokens=True)
 streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, decode_kwargs=decode_kwargs)
 
 cplusplus = None
 def translate(python):
-    formatted_prompt = tokenizer.apply_chat_template(
+    inputs = tokenizer.apply_chat_template(
                         messages_for(python),
-                        tokenize=False,
                         add_generation_prompt=True,
-                        return_tensors="pt")
-    inputs = tokenizer(formatted_prompt, return_tensors="pt", padding=True).to(model.device)
-    attention_mask = inputs.attention_mask
-    input_ids = inputs.input_ids
+                        return_tensors="pt").to(model.device)
 
     generation_kwargs = dict(
-        input_ids=input_ids,
+        inputs,
         streamer=streamer,
-        attention_mask=attention_mask,
-        max_new_tokens=1024,
+        max_new_tokens=512,
         do_sample=False,
-        pad_token_id=tokenizer.eos_token_id,
+        top_k=50,
+        top_p=0.95,
+        num_sequences=1,
         eos_token_id=tokenizer.eos_token_id,
     )