fix

app.py

@@ -32,14 +32,12 @@ bnb_config = BitsAndBytesConfig(
 )
 
 tokenizer = AutoTokenizer.from_pretrained("bigscience/bloomz-7b1")
-model = AutoModelForCausalLM.from_pretrained("bigscience/bloomz-7b1", device_map="auto")
-if torch.__version__ >= "2":
-    model = torch.compile(model)
-print(f"Successfully loaded the model {model_name} into memory")
+model = AutoModelForCausalLM.from_pretrained("bigscience/bloomz-7b1", quantization_config=bnb_config, trust_remote_code=True, device_map="auto")
+print(f"Successfully loaded the model into memory")
 
 
-# Define stopping criteria
-stop_tokens = ["###", "Human", "\n###"]
+# Define stopping criteria. We do not use it for bloom model family but it can be used for llama model family
+stop_tokens = ["\n###"]
 stop_token_ids = tokenizer.convert_tokens_to_ids(stop_tokens)
 
 class StopOnTokens(StoppingCriteria):
@@ -50,43 +48,42 @@ class StopOnTokens(StoppingCriteria):
         return False
 
 
-
-#Prompts 
+# Prompts 
 instruction_with_q = """
-A chat between a curious human and an artificial intelligence assistant.
-The assistant's job is to answer the given question using only the information provided in the RDF triplet format. The assistant's answer should be in a human-readable format, with proper sentences and grammar and should be concise and short.
+A chat between a curious USER and an artificial intelligence assistant.
+The assistant's job is to answer the given question using only the information provided in the RDF triplet format. The assistant's answer should be in a USER-readable format, with proper sentences and grammar and should be concise and short.
 The RDF triplets will be provided in triplets, where triplets are always in the (subject, relation, object) format and are separated by a semicolon. The assistant should understand that if multiple triplets are provided, the answer to the question should use all of the information from triplets and make aggregation. The assistant MUST NOT add any additional information, beside form the one proveded in the triplets.
 The assistant should try to reply as short as possible, and perform counting or aggregation operations over triplets by himself when necessary.
 """
 
 instruction_wo_q = """
-A chat between a curious human and an artificial intelligence assistant.
-The assistant's job is convert the provided input in RDF triplet format into human-readable text format, with proper sentences and grammar. The triplets are always in the (subject, relation, object) format, where each triplet is separated by a semicolon. The assistant should understand that if multiple triplets are provided, the generated human-readable text should use all of the information from input. The assistant MUST NOT add any additional information, beside form the one proveded in the input.
+A chat between a curious USER and an artificial intelligence assistant.
+The assistant's job is convert the provided input in RDF triplet format into USER-readable text format, with proper sentences and grammar. The triplets are always in the (subject, relation, object) format, where each triplet is separated by a semicolon. The assistant should understand that if multiple triplets are provided, the generated USER-readable text should use all of the information from input. The assistant MUST NOT add any additional information, beside form the one proveded in the input.
 """
 
 
 history_with_q = [
-        ("Human", "Question: Is Essex the Ceremonial County of West Tilbury? Triplets: ('West Tilbury', 'Ceremonial County', 'Essex');"),
-        ("Assistant", "Essex is the Ceremonial County of West Tributary"),
-        ("Human", "Question: What nation is Hornito located in, where Jamie Bateman Cayn died too? Triplets: ('Jaime Bateman Cayón', 'death place', 'Panama'); ('Hornito, Chiriquí', 'country', 'Panama');"),
-        ("Assistant", "Hornito, Chiriquí is located in Panama, where Jaime Bateman Cayón died."),
-        ("Human", "Question: Who are the shareholder of the soccer club for whom Steve Holland plays? Triplets: ('Steve Holland', 'current club', 'Chelsea F.C.'); ('Chelsea F.C.', 'owner', 'Roman Abramovich');"),
-        ("Assistant", "Roman Abramovich owns Chelsea F.C., where Steve Holland plays."),
-        ("Human", "Question: Who is the chancellor of Falmouth University? Triplets: ('Falmouth University', 'chancellor', 'Dawn French');"),
-        ("Assistant", "The chancellor of the Falmouth University is Dawn French.")
+        ("USER", "Question: Is Essex the Ceremonial County of West Tilbury? Triplets: ('West Tilbury', 'Ceremonial County', 'Essex');"),
+        ("ASSISTANT", "Essex is the Ceremonial County of West Tributary"),
+        ("USER", "Question: What nation is Hornito located in, where Jamie Bateman Cayn died too? Triplets: ('Jaime Bateman Cayón', 'death place', 'Panama'); ('Hornito, Chiriquí', 'country', 'Panama');"),
+        ("ASSISTANT", "Hornito, Chiriquí is located in Panama, where Jaime Bateman Cayón died."),
+        ("USER", "Question: Who are the shareholder of the soccer club for whom Steve Holland plays? Triplets: ('Steve Holland', 'current club', 'Chelsea F.C.'); ('Chelsea F.C.', 'owner', 'Roman Abramovich');"),
+        ("ASSISTANT", "Roman Abramovich owns Chelsea F.C., where Steve Holland plays."),
+        ("USER", "Question: Who is the chancellor of Falmouth University? Triplets: ('Falmouth University', 'chancellor', 'Dawn French');"),
+        ("ASSISTANT", "The chancellor of the Falmouth University is Dawn French.")
 
     ]
 
 
 history_wo_q = [
-        ("Human", "('West Tilbury', 'Ceremonial County', 'Essex');"),
-        ("Assistant", "Essex is the Ceremonial County of West Tributary"),
-        ("Human", "('Jaime Bateman Cayón', 'death place', 'Panama'); ('Hornito, Chiriquí', 'country', 'Panama');"),
-        ("Assistant", "Hornito, Chiriquí is located in Panama, where Jaime Bateman Cayón died."),
-        ("Human", "('Steve Holland', 'current club', 'Chelsea F.C.'); ('Chelsea F.C.', 'owner', 'Roman Abramovich');"),
-        ("Assistant", "Roman Abramovich owns Chelsea F.C., where Steve Holland plays."),
-        ("Human", "('Falmouth University', 'chancellor', 'Dawn French');"),
-        ("Assistant", "The chancellor of the Falmouth University is Dawn French.")
+        ("USER", "('West Tilbury', 'Ceremonial County', 'Essex');"),
+        ("ASSISTANT", "Essex is the Ceremonial County of West Tributary"),
+        ("USER", "('Jaime Bateman Cayón', 'death place', 'Panama'); ('Hornito, Chiriquí', 'country', 'Panama');"),
+        ("ASSISTANT", "Hornito, Chiriquí is located in Panama, where Jaime Bateman Cayón died."),
+        ("USER", "('Steve Holland', 'current club', 'Chelsea F.C.'); ('Chelsea F.C.', 'owner', 'Roman Abramovich');"),
+        ("ASSISTANT", "Roman Abramovich owns Chelsea F.C., where Steve Holland plays."),
+        ("USER", "('Falmouth University', 'chancellor', 'Dawn French');"),
+        ("ASSISTANT", "The chancellor of the Falmouth University is Dawn French.")
 
     ]
 
@@ -95,7 +92,7 @@ history_wo_q = [
 def prepare_input(linearized_triplets, question=None) ->  str:
     if question and "List all" in question:
         question = question.replace("List all ", "Which are ")
-    if "question" in style:
+    if question:
         input_text = f"Question: {question.strip()} Triplets: {linearized_triplets}"
     else:
         input_text = linearized_triplets
@@ -107,10 +104,10 @@ def make_prompt(
     instruction: str,
     history: List[Tuple[str, str]]=None,
 ) -> str:
-    ret = f"{instruction}\n###"
+    ret = f"{instruction}\n"
     for i, (role, message) in enumerate(history):
-        ret += f"{role}: {message}\n###"
-    ret += f"Human: {curr_input}\n###Assistant: \n"
+        ret += f"{role}: {message}\n"
+    ret += f"USER: {curr_input}\nASSISTANT: "
     return ret
 
 
@@ -128,6 +125,7 @@ def generate_output(
     else:
         instruction = make_prompt(curr_input, instruction_wo_q, history_wo_q)
 
+    stop = StopOnTokens()
     input_ids = tokenizer(instruction, return_tensors="pt").input_ids
     input_ids = input_ids.to(model.device)
 
@@ -139,11 +137,10 @@ def generate_output(
         top_p=top_p,
         top_k=top_k,
         repetition_penalty=repetition_penalty,
-        stopping_criteria=StoppingCriteriaList([stop]),
     )
 
     with torch.no_grad():
-        outputs = model.generate(generate_kwargs)
+        outputs = model.generate(**generate_kwargs, return_dict_in_generate=True, output_scores=True)
 
     response = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)
     for tok in tokenizer.additional_special_tokens+[tokenizer.eos_token]:
@@ -186,7 +183,7 @@ with gr.Blocks(theme='gradio/soft') as demo:
     examples = gr.Examples([
         ['("Google Videos", "developer", "Google"), ("Google Web Toolkit", "author", "Google")', ""],
         ['("Katyayana", "religion", "Buddhism")', "What is the relegious affiliations of Katyayana?"],
-    ], inputs=[triplets, question, temperature, top_p, top_k, repetition_penalty], fn=generate, cache_examples=False if platform.system() == "Windows" or platform.system() == "Darwin" else True, outputs=output_box)
+    ], inputs=[triplets, question, temperature, top_p, top_k, repetition_penalty], fn=generate_output, cache_examples=False if platform.system() == "Windows" or platform.system() == "Darwin" else True, outputs=output_box)
 
 
     #readme_content = requests.get(f"https://huggingface.co/HF_MODEL_PATH/raw/main/README.md").text
@@ -197,7 +194,7 @@ with gr.Blocks(theme='gradio/soft') as demo:
     #        readme_content,
     #    )
     
-    run_button.click(fn=generate, inputs=[triplets, question, temperature, top_p, top_k, repetition_penalty], outputs=output_box, api_name="rdf2text")
+    run_button.click(fn=generate_output, inputs=[triplets, question, temperature, top_p, top_k, repetition_penalty], outputs=output_box, api_name="rdf2text")
     clear_button.add([triplets, question, output_box])
 
 demo.queue(concurrency_count=1, max_size=10).launch(debug=True)