Spaces:
Running
Running
| # Llamafile_tab.py | |
| # Description: Gradio interface for configuring and launching Llamafile with Local LLMs | |
| # Imports | |
| import os | |
| import logging | |
| from typing import Tuple, Optional | |
| import gradio as gr | |
| from App_Function_Libraries.Local_LLM.Local_LLM_Inference_Engine_Lib import ( | |
| download_llm_model, | |
| llm_models, | |
| start_llamafile, | |
| get_gguf_llamafile_files | |
| ) | |
| # | |
| ####################################################################################################################### | |
| # | |
| # Functions: | |
| BASE_DIR = os.path.dirname(os.path.abspath(__file__)) | |
| MODELS_DIR = os.path.join(BASE_DIR, "Models") | |
| def create_chat_with_llamafile_tab(): | |
| # Function to update model path based on selection | |
| def on_local_model_change(selected_model: str, search_directory: str) -> str: | |
| if selected_model and isinstance(search_directory, str): | |
| model_path = os.path.abspath(os.path.join(search_directory, selected_model)) | |
| logging.debug(f"Selected model path: {model_path}") # Debug print for selected model path | |
| return model_path | |
| return "Invalid selection or directory." | |
| # Function to update the dropdown with available models | |
| def update_dropdowns(search_directory: str) -> Tuple[dict, str]: | |
| logging.debug(f"User-entered directory: {search_directory}") # Debug print for directory | |
| if not os.path.isdir(search_directory): | |
| logging.debug(f"Directory does not exist: {search_directory}") # Debug print for non-existing directory | |
| return gr.update(choices=[], value=None), "Directory does not exist." | |
| try: | |
| logging.debug(f"Directory exists: {search_directory}, scanning for files...") # Confirm directory exists | |
| model_files = get_gguf_llamafile_files(search_directory) | |
| logging.debug("Completed scanning for model files.") | |
| except Exception as e: | |
| logging.error(f"Error scanning directory: {e}") | |
| return gr.update(choices=[], value=None), f"Error scanning directory: {e}" | |
| if not model_files: | |
| logging.debug(f"No model files found in {search_directory}") # Debug print for no files found | |
| return gr.update(choices=[], value=None), "No model files found in the specified directory." | |
| # Update the dropdown choices with the model files found | |
| logging.debug(f"Models loaded from {search_directory}: {model_files}") # Debug: Print model files loaded | |
| return gr.update(choices=model_files, value=None), f"Models loaded from {search_directory}." | |
| def download_preset_model(selected_model: str) -> Tuple[str, str]: | |
| """ | |
| Downloads the selected preset model. | |
| Args: | |
| selected_model (str): The key of the selected preset model. | |
| Returns: | |
| Tuple[str, str]: Status message and the path to the downloaded model. | |
| """ | |
| model_info = llm_models.get(selected_model) | |
| if not model_info: | |
| return "Invalid model selection.", "" | |
| try: | |
| model_path = download_llm_model( | |
| model_name=model_info["name"], | |
| model_url=model_info["url"], | |
| model_filename=model_info["filename"], | |
| model_hash=model_info["hash"] | |
| ) | |
| return f"Model '{model_info['name']}' downloaded successfully.", model_path | |
| except Exception as e: | |
| logging.error(f"Error downloading model: {e}") | |
| return f"Failed to download model: {e}", "" | |
| with gr.TabItem("Local LLM with Llamafile", visible=True): | |
| gr.Markdown("# Settings for Llamafile") | |
| with gr.Row(): | |
| with gr.Column(): | |
| am_noob = gr.Checkbox(label="Enable Sane Defaults", value=False, visible=True) | |
| advanced_mode_toggle = gr.Checkbox(label="Advanced Mode - Show All Settings", value=False) | |
| # Advanced Inputs | |
| verbose_checked = gr.Checkbox(label="Enable Verbose Output", value=False, visible=False) | |
| threads_checked = gr.Checkbox(label="Set CPU Threads", value=False, visible=False) | |
| threads_value = gr.Number(label="Number of CPU Threads", value=None, precision=0, visible=False) | |
| threads_batched_checked = gr.Checkbox(label="Enable Batched Inference", value=False, visible=False) | |
| threads_batched_value = gr.Number(label="Batch Size for Inference", value=None, precision=0, visible=False) | |
| model_alias_checked = gr.Checkbox(label="Set Model Alias", value=False, visible=False) | |
| model_alias_value = gr.Textbox(label="Model Alias", value="", visible=False) | |
| ctx_size_checked = gr.Checkbox(label="Set Prompt Context Size", value=False, visible=False) | |
| ctx_size_value = gr.Number(label="Prompt Context Size", value=8124, precision=0, visible=False) | |
| ngl_checked = gr.Checkbox(label="Enable GPU Layers", value=False, visible=True) | |
| ngl_value = gr.Number(label="Number of GPU Layers", value=None, precision=0, visible=True) | |
| batch_size_checked = gr.Checkbox(label="Set Batch Size", value=False, visible=False) | |
| batch_size_value = gr.Number(label="Batch Size", value=512, visible=False) | |
| memory_f32_checked = gr.Checkbox(label="Use 32-bit Floating Point", value=False, visible=False) | |
| numa_checked = gr.Checkbox(label="Enable NUMA", value=False, visible=False) | |
| server_timeout_value = gr.Number(label="Server Timeout", value=600, precision=0, visible=False) | |
| host_checked = gr.Checkbox(label="Set IP to Listen On", value=False, visible=False) | |
| host_value = gr.Textbox(label="Host IP Address", value="", visible=False) | |
| port_checked = gr.Checkbox(label="Set Server Port", value=False, visible=False) | |
| port_value = gr.Number(label="Port Number", value=8080, precision=0, visible=False) | |
| api_key_checked = gr.Checkbox(label="Set API Key", value=False, visible=False) | |
| api_key_value = gr.Textbox(label="API Key", value="", visible=False) | |
| http_threads_checked = gr.Checkbox(label="Set HTTP Server Threads", value=False, visible=False) | |
| http_threads_value = gr.Number(label="Number of HTTP Server Threads", value=None, precision=0, visible=False) | |
| hf_repo_checked = gr.Checkbox(label="Use Huggingface Repo Model", value=False, visible=False) | |
| hf_repo_value = gr.Textbox(label="Huggingface Repo Name", value="", visible=False) | |
| hf_file_checked = gr.Checkbox(label="Set Huggingface Model File", value=False, visible=False) | |
| hf_file_value = gr.Textbox(label="Huggingface Model File", value="", visible=False) | |
| with gr.Column(): | |
| # Model Selection Section | |
| gr.Markdown("## Model Selection") | |
| # Option 1: Select from Local Filesystem | |
| with gr.Row(): | |
| search_directory = gr.Textbox( | |
| label="Model Directory", | |
| placeholder="Enter directory path (currently './Models')", | |
| value=MODELS_DIR, | |
| interactive=True | |
| ) | |
| # Initial population of local models | |
| initial_dropdown_update, _ = update_dropdowns(MODELS_DIR) | |
| logging.debug(f"Scanning directory: {MODELS_DIR}") | |
| refresh_button = gr.Button("Refresh Models") | |
| local_model_dropdown = gr.Dropdown( | |
| label="Select Model from Directory", | |
| choices=initial_dropdown_update["choices"], | |
| value=None | |
| ) | |
| # Display selected model path | |
| model_value = gr.Textbox(label="Selected Model File Path", value="", interactive=False) | |
| # Option 2: Download Preset Models | |
| gr.Markdown("## Download Preset Models") | |
| preset_model_dropdown = gr.Dropdown( | |
| label="Select a Preset Model", | |
| choices=list(llm_models.keys()), | |
| value=None, | |
| interactive=True, | |
| info="Choose a preset model to download." | |
| ) | |
| download_preset_button = gr.Button("Download Selected Preset") | |
| with gr.Row(): | |
| with gr.Column(): | |
| start_button = gr.Button("Start Llamafile") | |
| stop_button = gr.Button("Stop Llamafile (doesn't work)") | |
| output_display = gr.Markdown() | |
| # Show/hide advanced inputs based on toggle | |
| def update_visibility(show_advanced: bool): | |
| components = [ | |
| verbose_checked, threads_checked, threads_value, | |
| http_threads_checked, http_threads_value, | |
| hf_repo_checked, hf_repo_value, | |
| hf_file_checked, hf_file_value, | |
| ctx_size_checked, ctx_size_value, | |
| ngl_checked, ngl_value, | |
| host_checked, host_value, | |
| port_checked, port_value | |
| ] | |
| return [gr.update(visible=show_advanced) for _ in components] | |
| def on_start_button_click( | |
| am_noob: bool, | |
| verbose_checked: bool, | |
| threads_checked: bool, | |
| threads_value: Optional[int], | |
| threads_batched_checked: bool, | |
| threads_batched_value: Optional[int], | |
| model_alias_checked: bool, | |
| model_alias_value: str, | |
| http_threads_checked: bool, | |
| http_threads_value: Optional[int], | |
| model_value: str, | |
| hf_repo_checked: bool, | |
| hf_repo_value: str, | |
| hf_file_checked: bool, | |
| hf_file_value: str, | |
| ctx_size_checked: bool, | |
| ctx_size_value: Optional[int], | |
| ngl_checked: bool, | |
| ngl_value: Optional[int], | |
| batch_size_checked: bool, | |
| batch_size_value: Optional[int], | |
| memory_f32_checked: bool, | |
| numa_checked: bool, | |
| server_timeout_value: Optional[int], | |
| host_checked: bool, | |
| host_value: str, | |
| port_checked: bool, | |
| port_value: Optional[int], | |
| api_key_checked: bool, | |
| api_key_value: str | |
| ) -> str: | |
| """ | |
| Event handler for the Start Llamafile button. | |
| """ | |
| try: | |
| result = start_llamafile( | |
| am_noob, | |
| verbose_checked, | |
| threads_checked, | |
| threads_value, | |
| threads_batched_checked, | |
| threads_batched_value, | |
| model_alias_checked, | |
| model_alias_value, | |
| http_threads_checked, | |
| http_threads_value, | |
| model_value, | |
| hf_repo_checked, | |
| hf_repo_value, | |
| hf_file_checked, | |
| hf_file_value, | |
| ctx_size_checked, | |
| ctx_size_value, | |
| ngl_checked, | |
| ngl_value, | |
| batch_size_checked, | |
| batch_size_value, | |
| memory_f32_checked, | |
| numa_checked, | |
| server_timeout_value, | |
| host_checked, | |
| host_value, | |
| port_checked, | |
| port_value, | |
| api_key_checked, | |
| api_key_value | |
| ) | |
| return result | |
| except Exception as e: | |
| logging.error(f"Error starting Llamafile: {e}") | |
| return f"Failed to start Llamafile: {e}" | |
| advanced_mode_toggle.change( | |
| fn=update_visibility, | |
| inputs=[advanced_mode_toggle], | |
| outputs=[ | |
| verbose_checked, threads_checked, threads_value, | |
| http_threads_checked, http_threads_value, | |
| hf_repo_checked, hf_repo_value, | |
| hf_file_checked, hf_file_value, | |
| ctx_size_checked, ctx_size_value, | |
| ngl_checked, ngl_value, | |
| host_checked, host_value, | |
| port_checked, port_value | |
| ] | |
| ) | |
| start_button.click( | |
| fn=on_start_button_click, | |
| inputs=[ | |
| am_noob, | |
| verbose_checked, | |
| threads_checked, | |
| threads_value, | |
| threads_batched_checked, | |
| threads_batched_value, | |
| model_alias_checked, | |
| model_alias_value, | |
| http_threads_checked, | |
| http_threads_value, | |
| model_value, | |
| hf_repo_checked, | |
| hf_repo_value, | |
| hf_file_checked, | |
| hf_file_value, | |
| ctx_size_checked, | |
| ctx_size_value, | |
| ngl_checked, | |
| ngl_value, | |
| batch_size_checked, | |
| batch_size_value, | |
| memory_f32_checked, | |
| numa_checked, | |
| server_timeout_value, | |
| host_checked, | |
| host_value, | |
| port_checked, | |
| port_value, | |
| api_key_checked, | |
| api_key_value | |
| ], | |
| outputs=output_display | |
| ) | |
| download_preset_button.click( | |
| fn=download_preset_model, | |
| inputs=[preset_model_dropdown], | |
| outputs=[output_display, model_value] | |
| ) | |
| # Click event for refreshing models | |
| refresh_button.click( | |
| fn=update_dropdowns, | |
| inputs=[search_directory], # Ensure that the directory path (string) is passed | |
| outputs=[local_model_dropdown, output_display] # Update dropdown and status | |
| ) | |
| # Event to update model_value when a model is selected from the dropdown | |
| local_model_dropdown.change( | |
| fn=on_local_model_change, # Function that calculates the model path | |
| inputs=[local_model_dropdown, search_directory], # Inputs: selected model and directory | |
| outputs=[model_value] # Output: Update the model_value textbox with the selected model path | |
| ) | |
| # | |
| # | |
| ####################################################################################################################### |