import PIL.Image import gradio as gr import base64 import time import os import google.generativeai as genai import pathlib txt_model = genai.GenerativeModel('gemini-pro') vis_model = genai.GenerativeModel('gemini-pro-vision') txt_prompt_1 = """The image contains the contents of a letter. I'd like to follow the request mentioned in the letter. Please provide 3 actionable items to assist me. When responding, use the following format: # Sender and Subject # 1- Action 1 (no more than 20 words) 2- Action 2 (no more than 20 words) 3- Action 3 (no more than 20 words) For example: # From Richard regarding 'Shipping to Customer ABC' # 1- Pack Product A 2- Ship before 3:00 PM today 3- Notify Richard after shipment """ txt_display_1 = 'content of email' import os GOOGLE_API_KEY=os.getenv('GOOGLE_API_KEY') genai.configure(api_key=GOOGLE_API_KEY) # Image to Base 64 Converter def image_to_base64(image_path): with open(image_path, 'rb') as img: encoded_string = base64.b64encode(img.read()) return encoded_string.decode('utf-8') # Function that takes User Inputs and displays it on ChatUI def app2_query(history,txt,img): if not img: history += [(txt,None)] return history base64 = image_to_base64(img) data_url = f"data:image/jpeg;base64,{base64}" history += [(f"{txt} ![]({data_url})", None)] return history # Function that takes User Inputs, generates Response and displays on Chat UI def app2_response(history,text,img): if not img: response = txt_model.generate_content(text) history += [(None,response.text)] return history else: img = PIL.Image.open(img) response = vis_model.generate_content([text,img]) history += [(None,response.text)] return history # Function that takes User Inputs and displays it on ChatUI def app1_query(img): if not img: return txt_prompt_1 base64 = image_to_base64(img) data_url = f"data:image/jpeg;base64,{base64}" outputText = [(f"{txt_display_1} ![]({data_url})", None)] return outputText # Function that takes User Inputs, generates Response and displays on Chat UI def app1_response(img): if not img: response = txt_model.generate_content(txt_prompt_1) return response else: img = PIL.Image.open(img) response = vis_model.generate_content([txt_prompt_1,img]) return response.txt # Interface Code- Selector method def sentence_builder(animal, place): return f"""how many {animal}s from the {place} are shown in the picture?""" # gradio block with gr.Blocks(theme='snehilsanyal/scikit-learn') as app1: with gr.Column(): outputbox = gr.Textbox(label="here are the plans...") image_box = gr.Image(type="filepath") btn = gr.Button("Make a Plan") clicked = btn.click(app1_query, [image_box], outputbox ).then(app1_response, [image_box], outputbox ) gr.Markdown(""" # Make a Plan # - screen capture (Win + shift + S) - click **Make a Plan** to upload - await LLM Bot (Gemini, in this case) response - receive THREE actionable items [demo](https://youtu.be/lJ4jIAEVRNY) """) with gr.Blocks(theme='snehilsanyal/scikit-learn') as app2: gr.Markdown("check the image...") with gr.Row(): image_box = gr.Image(type="filepath") chatbot = gr.Chatbot( scale = 2, height=750 ) text_box = gr.Dropdown( ["what is in the image", "provide alternative title for the image", "how many parts can be seen in the picture?", "check ID and expiration date"], label="Select--", info="ask Bot" ) btn = gr.Button("Submit") clicked = btn.click(app2_query, [chatbot,text_box,image_box], chatbot ).then(app2_response, [chatbot,text_box], chatbot ) with gr.Blocks(theme='snehilsanyal/scikit-learn') as demo: gr.Markdown("## Workflow Bot ##") gr.TabbedInterface([app1, app2], ["Make a Plan!", "Check This!"]) demo.queue() demo.launch()