Frist Commit

app.py

@@ -0,0 +1,29 @@
+import json
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+
+def formatting_func(document):
+    instruction = "You are a model designed to rephrase medical summaries for a general audience. Please summarize the following article in such a way a normal person could understand it, while also ensuring the same factual accuracy. Replace any technical terms with their equivalents in ordinary language, and be concise (< 100 words) and approachable.\n---------\n" 
+    text = f"### {instruction} \n### Conclusion: {document} \n### Summary: "
+    return text
+
+def genenrate(text):
+    ft_model = AutoModelForCausalLM.from_pretrained("BiswajitPadhi99/mistral-7b-finetuned-medical-summarizer", 
+                                                device_map="cuda", load_in_4bit=True)
+    eval_tokenizer = AutoTokenizer.from_pretrained("BiswajitPadhi99/mistral-7b-finetuned-medical-summarizer", add_bos_token=True,
+                                               device_map="cuda")
+    ft_model.eval()
+    with torch.no_grad():
+        eval_prompt = formatting_func(text)
+        model_input = eval_tokenizer(eval_prompt, return_tensors="pt").to("cuda")
+        response = eval_tokenizer.decode(ft_model.generate(**model_input, max_new_tokens=200)[0], skip_special_tokens=True)
+        if(eval_prompt in response):
+            response = response.replace(eval_prompt, '')
+        return response
+
+input = st.text_input(label= "Input Text")
+if input is not None:
+    col =st.columns(1)
+    output = generate(input)
+    col.header("Summary")
+    col.write(output)

requirements.txt

@@ -0,0 +1,2 @@
+transformers
+torch