# refer to repo https://github.com/gradio-app/gradio/blob/main/demo/chatbot_multimodal/run.ipynb for enhancement import PIL.Image import gradio as gr import base64 import time import os import google.generativeai as genai import requests import pathlib txt_model = genai.GenerativeModel('gemini-pro') vis_model = genai.GenerativeModel('gemini-pro-vision') txt_prompt_1 = """The image contains the contents of a letter. I'd like to follow the request mentioned in the letter. Please provide 3 actionable items to assist me. When responding, use the following format: # Sender and Subject # 1- Action 1 (no more than 20 words) 2- Action 2 (no more than 20 words) 3- Action 3 (no more than 20 words) For example: # From Richard regarding 'Shipping to Customer ABC' # 1- Pack Product A 2- Ship before 3:00 PM today 3- Notify Richard after shipment """ txt_display_1 = 'content of the letter: ' txt_prompt_2 = """First, to determine if the image is related to inventory and contains necessary information, consider the base reference of "a valid inventory table should be a table format with columns for Item ID, Quantity, Minimum, and Contact Phone." If the image does not align with this base reference, please respond accordingly with "not applicable, as the image appears not related or lacks necessary information." If the image aligns with this base reference, proceed with the following steps to generate a response: Check the Quantity against the respective Minimum for each row. If the Quantity is less than the Minimum, then the respective Item ID is in a shortage state, and the shortage is defined as the negative value of Quantity - Minimum. Provide a list of all items in shortage in the following format: ** SOS shortage (MM/DD/YYYY) ** Item ID, Shortage (Quantity - Minimum), Contact Phone For example, considering the below Table, | Item ID | Quantity | Minimum | Contact Phone | | ----------- | ----------- | | #01 | 30 | 50 | 1234 | #02 | 40 | 20 | 3456 | #03 | 5 | 10 | 4567 the respective response should be as follow: ** SOS shortage (02/19/2024) ** #01, -20 (30 - 50), 1234 #03, -5 (5 - 10), 4567 When rendering the response, please ensure ALL items in shortage state are on the list. Also, be aware that there may also be NO item in shortage state. """ txt_display_2 = '--- ' import os GOOGLE_API_KEY=os.getenv('GOOGLE_API_KEY') genai.configure(api_key=GOOGLE_API_KEY) SMS_URL =os.getenv('SMS_URL') SMS_TOK =os.getenv('SMS_TOK') sms_text ="..." # Image to Base 64 Converter def image_to_base64(image_path): with open(image_path, 'rb') as img: encoded_string = base64.b64encode(img.read()) return encoded_string.decode('utf-8') # Function that takes User Inputs, generates Response and displays on Chat UI def app1_response(img): if not img: response = txt_model.generate_content(txt_prompt_1) return response else: img = PIL.Image.open(img) response = vis_model.generate_content([txt_prompt_1,img]) return response.text def app2_response(img): if not img: response = txt_model.generate_content(txt_prompt_2) return response else: img = PIL.Image.open(img) response = vis_model.generate_content([txt_prompt_2,img]) return response.text # SMS service ends in March 2024, to restore service @Sinch Simple Text def send_SMS(resp_text): url = SMS_URL headers = { "Authorization": SMS_TOK, "Content-Type": "application/json" } data = { "from": "12085686834", "to": ["18587331029"], "body": resp_text } # response = requests.post(url, json=data, headers=headers) # return response.text return "....." # gradio block with gr.Blocks() as app1: with gr.Column(): gr.Markdown("## πŸ₯· to Samuraize ##") gr.Markdown("```for email βœ‰ and/or assigment descriptions ⌦, paste screenshot here...```") image_box = gr.Image(label="βœ‚ email screen", type="filepath") btn1 = gr.Button("Generate To-Dos β˜‘") out1 = gr.Textbox(label="here are the actionables...") btn2 = gr.Button("send to Mobile ↗️") out2 = gr.Textbox(label="response from SMS gateway...") btn1.click(fn=app1_response, inputs=[image_box], outputs=out1) btn2.click(fn=send_SMS, inputs=out1, outputs=out2) gr.Markdown(""" # πŸ₯· Summerize eMail & Make a Plan # - screen capture (Win + shift + S) - click β˜‘ to upload - await LLM Bot (Gemini, in this case) response - receive THREE actionable items [demo video](https://youtu.be/lJ4jIAEVRNY) """) gr.Markdown(""" # try with the mockup email below# ![Alt text](missionemail.png) """) with gr.Blocks() as app2: with gr.Column(): gr.Markdown("## 🦫 Stock-Out Squirrel ##") gr.Markdown("```Win+Screenshot, paste ERP Inv ⌨ screenshot here...```") image_box = gr.Image(label="βœ‚ ERP screen",type="filepath") btn1 = gr.Button("Check ROQ β˜‘") out1 = gr.Textbox(label="here is the watch list πŸ“„...") btn2 = gr.Button("send out reminders ↗️") out2 = gr.Textbox(label="response or feed back?") btn1.click(fn=app2_response, inputs=[image_box], outputs=out1) btn2.click(fn=send_SMS, inputs=out1, outputs=out2) gr.Markdown(""" # 🦫 Check Inventory ROQ # (contains bugs, more works needed) - screen capture (Win + shift + S) - click β˜‘ to upload - await LLM Bot (Gemini, in this case) response - send ↗️ to related parties """) gr.Markdown(""" - [x] Sample Inventory Table - [X] Screen Capture and Paste """) gr.Markdown(""" | Item ID | Quantity | Minimum|Contact Phone| |--|--|--|--| | #01 | 50 | 20 | 1234| | #02 | 40 | 50 | 3456| | #03 | 80 | 20 | 1234| """) with gr.Blocks() as app3: with gr.Column(): gr.Markdown("## πŸ‡ Route Planning Rabbit ##") gr.Markdown("```Win+Screenshot, paste ERP Inv ⌨ screenshot here...```") image_box = gr.Image(label="βœ‚ Pick list",type="filepath") btn1 = gr.Button("Generate a Route Plan β˜‘") out1 = gr.Textbox(label="here is the watch list πŸ“„...") btn2 = gr.Button("send to Mobile (disabled) ↗️") out2 = gr.Textbox(label="no message been sent, this is just a mockup confirmation...") btn1.click(fn=app2_response, inputs=[image_box], outputs=out1) btn2.click(fn=send_SMS, inputs=out1, outputs=out2) gr.Markdown(""" # πŸ‡ Route Planning # (concept prototype) - screen capture (Win + shift + S) - click β˜‘ to upload - await LLM Bot (Gemini, in this case) response - send ↗️ to related parties """) with gr.Blocks() as demo: gr.Markdown("## To-Do Samuraizer πŸ₯· + Stock-Out Squirrel🦫 + Routing πŸ‡ ##") gr.TabbedInterface([app1, app2, app3], ["βž€ To-Do", "➁ SOS", "βž‚ Picking Routes"]) demo.queue() demo.launch()