Update app.py

app.py

@@ -11,18 +11,44 @@ from huggingface_hub import hf_hub_download
 
 # モデルのダウンロード
 hf_hub_download(
-   repo_id="bartowski/gemma-2-9b-it-GGUF",
-   filename="gemma-2-9b-it-Q5_K_M.gguf",
-   local_dir="./models"
+    repo_id="Aratako/Oumuamua-7b-RP-GGUF",
+    filename="Oumuamua-7b-RP_Q4_K_M.gguf",
+    local_dir="./models"
 )
 
 hf_hub_download(
-   repo_id="bartowski/Gemma-2-9B-It-SPPO-Iter3-GGUF",
-   filename="Gemma-2-9B-It-SPPO-Iter3-Q5_K_M.gguf",
-   local_dir="./models"
+    repo_id="bartowski/Oumuamua-7b-instruct-v2-GGUF",
+    filename="Oumuamua-7b-instruct-v2-Q4_K_M.gguf",
+    local_dir="./models"
 )
 
-# 推論関数
+hf_hub_download(
+    repo_id="mmnga/umiyuki-Umievo-itr012-Gleipnir-7B-gguf",
+    filename="umiyuki-Umievo-itr012-Gleipnir-7B-Q4_K_M.gguf",
+    local_dir="./models"
+)
+
+hf_hub_download(
+    repo_id="Local-Novel-LLM-project/Ninja-V3-GGUF",
+    filename="Ninja-V3-Q4_K_M.gguf",
+    local_dir="./models"
+)
+
+hf_hub_download(
+    repo_id="Local-Novel-LLM-project/Kagemusya-7B-v1-GGUF",
+    filename="kagemusya-7b-v1Q8_0.gguf",
+    local_dir="./models"
+)
+
+hf_hub_download(
+    repo_id="elyza/Llama-3-ELYZA-JP-8B-GGUF",
+    filename="Llama-3-ELYZA-JP-8B-q4_k_m.gguf",
+    local_dir="./models"
+)
+
+llm = None
+llm_model = None
+
 @spaces.GPU(duration=120)
 def respond(
     message,
@@ -34,25 +60,32 @@ def respond(
     top_p,
     top_k,
     repeat_penalty,
-    ):
-    chat_template = MessagesFormatterType.GEMMA_2
+    template,
+):
+    chat_template = MessagesFormatterType[template]
+
+    global llm
+    global llm_model
     
-    llm = Llama(
-       model_path=f"models/{model}",
-       flash_attn=True,
-       n_gpu_layers=81,
-       n_batch=1024,
-       n_ctx=8192,
-    )
+    if llm is None or llm_model != model:
+        llm = Llama(
+            model_path=f"models/{model}",
+            flash_attn=True,
+            n_gpu_layers=81,
+            n_batch=1024,
+            n_ctx=8192,
+        )
+        llm_model = model
+
     provider = LlamaCppPythonProvider(llm)
-    
+
     agent = LlamaCppAgent(
-       provider,
-       system_prompt=f"{system_message}",
-       predefined_messages_formatter_type=chat_template,
-       debug_output=True
+        provider,
+        system_prompt=f"{system_message}",
+        predefined_messages_formatter_type=chat_template,
+        debug_output=True
     )
-
+    
     settings = provider.get_provider_default_settings()
     settings.temperature = temperature
     settings.top_k = top_k
@@ -60,112 +93,107 @@ def respond(
     settings.max_tokens = max_tokens
     settings.repeat_penalty = repeat_penalty
     settings.stream = True
-    
+
     messages = BasicChatHistory()
-    
-    one_shot_prompt = """
-    あなたは優れた翻訳者です。以下の文章を日本語から英語に翻訳してください。翻訳は文法的に正しく、自然な表現を使用し、文脈に適した内容にしてください。また、専門用語や文化的なニュアンスを正確に伝えるよう心がけてください。
-    
-    期待する品質基準:
-    1. 文法的に正確であること。
-    2. 自然な英語表現を使用すること。
-    3. 文脈に適した訳を提供すること。
-    4. 専門用語や文化的なニュアンスを正確に伝えること。
-    
-    以下に翻訳の例を示します。
-    
-    例:
-    日本語: 優れた翻訳は、原文の意味を正確に伝えるだけでなく、読み手にとって自然な表現である必要があります。文法的な正確さはもちろんのこと、文化的なニュアンスも重要です。専門用語の正確な訳も求められます。さらに、文脈に適した訳を提供することが、翻訳の品質を高めます。最終的には、読み手にとってわかりやすい訳を目指してください。
-    英語: A good translation should not only convey the meaning of the original text accurately but also be expressed in a natural way for the reader. In addition to grammatical accuracy, cultural nuances are important. Accurate translation of technical terms is also required. Furthermore, providing a translation that fits the context enhances the quality of the translation. Ultimately, aim for a translation that is easy for the reader to understand.
-    
-    以下の日本語の文章を英語に翻訳してください:
-    """
-    system_message += one_shot_prompt
-    
+
     for msn in history:
-       user = {
-           'role': Roles.user,
-           'content': msn[0]
-       }
-       assistant = {
-           'role': Roles.assistant,
-           'content': msn[1]
-       }
-       messages.add_message(user)
-       messages.add_message(assistant)
+        user = {
+            'role': Roles.user,
+            'content': msn[0]
+        }
+        assistant = {
+            'role': Roles.assistant,
+            'content': msn[1]
+        }
+        messages.add_message(user)
+        messages.add_message(assistant)
     
     stream = agent.get_chat_response(
-       message,
-       llm_sampling_settings=settings,
-       chat_history=messages,
-       returns_streaming_generator=True,
-       print_output=False
+        message,
+        llm_sampling_settings=settings,
+        chat_history=messages,
+        returns_streaming_generator=True,
+        print_output=False
     )
     
     outputs = ""
     for output in stream:
-       outputs += output
-       yield outputs
-
-# Gradioのインターフェースを作成
-def create_interface(model_name, description):
-   return gr.ChatInterface(
-       respond,
-       additional_inputs=[
-           gr.Textbox(value=model_name, label="Model", interactive=False),
-           gr.Textbox(value="You are a helpful assistant.", label="System message"),
-           gr.Slider(minimum=1, maximum=4096, value=2048, step=1, label="Max tokens"),
-           gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-           gr.Slider(
-               minimum=0.1,
-               maximum=1.0,
-               value=0.95,
-               step=0.05,
-               label="Top-p",
-           ),
-           gr.Slider(
-               minimum=0,
-               maximum=100,
-               value=40,
-               step=1,
-               label="Top-k",
-           ),
-           gr.Slider(
-               minimum=0.0,
-               maximum=2.0,
-               value=1.1,
-               step=0.1,
-               label="Repetition penalty",
-           ),
-       ],
-       retry_btn="Retry",
-       undo_btn="Undo",
-       clear_btn="Clear",
-       submit_btn="Send",
-       title=f"Chat with Gemma 2 using llama.cpp - {model_name}",
-       description=description,
-       chatbot=gr.Chatbot(
-           scale=1,
-           likeable=False,
-           show_copy_button=True
-       )
-   )
-
-# 各モデルのインターフェース
-description_9b = """<p align="center">Gemma-2 9B it Model</p>"""
-description_27b = """<p align="center">Gemma-2 9B SPPO it Model</p>"""
-
-interface_9b = create_interface('gemma-2-9b-it-Q5_K_M.gguf', description_9b)
-interface_27b = create_interface('Gemma-2-9B-It-SPPO-Iter3-Q5_K_M.gguf', description_27b)
-
-# Gradio Blocksで2つのインターフェースを並べて表示
-with gr.Blocks() as demo:
-   #gr.Markdown("# Compare Gemma-2 9B and 27B Models")
-   with gr.Row():
-       with gr.Column():
-           interface_9b.render()
-       with gr.Column():
-           interface_27b.render()
+        outputs += output
+        yield outputs
+
+description = """<p align="center">Defaults to Oumuamua-7b-RP (you can switch to other models from additional inputs)</p>
+<p><center>
+<a href="https://huggingface.co/Aratako/Oumuamua-7b-RP-GGUF" target="_blank">[Oumuamua-7b-RP Model]</a>
+<a href="https://huggingface.co/bartowski/Oumuamua-7b-instruct-v2-GGUF" target="_blank">[Oumuamua-7b-instruct-v2 Model]</a>
+<a href="https://huggingface.co/mmnga/umiyuki-Umievo-itr012-Gleipnir-7B-gguf" target="_blank">[Umievo-itr012-Gleipnir-7B Model]</a>
+<a href="https://huggingface.co/Local-Novel-LLM-project/Ninja-V3-GGUF" target="_blank">[Ninja-V3 Model]</a>
+<a href="https://huggingface.co/Local-Novel-LLM-project/Kagemusya-7B-v1-GGUF" target="_blank">[Kagemusya-7B-v1 Model]</a>
+<a href="https://huggingface.co/elyza/Llama-3-ELYZA-JP-8B-GGUF" target="_blank">[Llama-3-ELYZA-JP-8B Model]</a>
+</center></p>
+"""
+
+templates = [
+    "MISTRAL", "CHATML", "VICUNA", "LLAMA_2", "SYNTHIA",
+    "NEURAL_CHAT", "SOLAR", "OPEN_CHAT", "ALPACA", "CODE_DS",
+    "B22", "LLAMA_3", "PHI_3"
+]
+
+demo = gr.ChatInterface(
+    respond,
+    additional_inputs=[
+        gr.Dropdown([
+                'Oumuamua-7b-RP_Q4_K_M.gguf',
+                'Oumuamua-7b-instruct-v2-Q4_K_M.gguf',
+                'umiyuki-Umievo-itr012-Gleipnir-7B-Q4_K_M.gguf',
+                'Ninja-V3-Q4_K_M.gguf',
+                'kagemusya-7b-v1Q8_0.gguf',
+                'Llama-3-ELYZA-JP-8B-q4_k_m.gguf'
+            ],
+            value="Oumuamua-7b-RP_Q4_K_M.gguf",
+            label="Model"
+        ),
+        gr.Textbox(value="You are a helpful assistant.", label="System message"),
+        gr.Slider(minimum=1, maximum=4096, value=2048, step=1, label="Max tokens"),
+        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
+        gr.Slider(
+            minimum=0.1,
+            maximum=1.0,
+            value=0.95,
+            step=0.05,
+            label="Top-p",
+        ),
+        gr.Slider(
+            minimum=0,
+            maximum=100,
+            value=40,
+            step=1,
+            label="Top-k",
+        ),
+        gr.Slider(
+            minimum=0.0,
+            maximum=2.0,
+            value=1.1,
+            step=0.1,
+            label="Repetition penalty",
+        ),
+        gr.Dropdown(
+            choices=templates,
+            value="LLAMA_3",
+            label="Template"
+        ),
+    ],
+    retry_btn="Retry",
+    undo_btn="Undo",
+    clear_btn="Clear",
+    submit_btn="Send",
+    title="Chat with various models using llama.cpp", 
+    description=description,
+    chatbot=gr.Chatbot(
+        scale=1, 
+        likeable=False,
+        show_copy_button=True
+    )
+)
 
 if __name__ == "__main__":
-   demo.launch()
+    demo.launch()