app.py

# refer to repo https://github.com/gradio-app/gradio/blob/main/demo/chatbot_multimodal/run.ipynb for enhancement

import PIL.Image
import gradio as gr
import base64
import time
import os
import google.generativeai as genai
import requests

import pathlib

txt_model = genai.GenerativeModel('gemini-pro')
vis_model = genai.GenerativeModel('gemini-pro-vision')

txt_prompt_1 = """The image contains the contents of a letter. I'd like to follow the request mentioned in the letter. Please provide 3 actionable items to assist me. When responding, use the following format:

# Sender and Subject #
1- Action 1 (no more than 20 words)
2- Action 2 (no more than 20 words)
3- Action 3 (no more than 20 words)

For example:
# From Richard regarding 'Shipping to Customer ABC' #
1- Pack Product A
2- Ship before 3:00 PM today
3- Notify Richard after shipment
"""

txt_display_1 = 'content of the letter: '


txt_prompt_2 = """First, to determine if the image is related to inventory and contains necessary information, consider the base reference of "a valid inventory table should be a table format with columns for Item ID, Quantity, Minimum, and Contact Phone."

If the image does not align with this base reference, please respond accordingly with "not applicable, as the image appears not related or lacks necessary information."

If the image aligns with this base reference, proceed with the following steps to generate a response:

Check the Quantity against the respective Minimum for each row. If the Quantity is less than the Minimum, then the respective Item ID is in a shortage state, and the shortage is defined as the negative value of Quantity - Minimum. Provide a list of all items in shortage in the following format:

** SOS shortage (MM/DD/YYYY) **
Item ID, Shortage (Quantity - Minimum), Contact Phone

For example, considering the below Table,
| Item ID | Quantity | Minimum | Contact Phone |
| ----------- | ----------- |
| #01 | 30 | 50 | 1234
| #02 | 40 | 20 | 3456
| #03 | 5 | 10 | 4567

the respective response should be as follow:

** SOS shortage (02/19/2024) **
#01, -20 (30 - 50), 1234
#03, -5 (5 - 10), 4567

When rendering the response, please ensure ALL items in shortage state are on the list. Also, be aware that there may also be NO item in shortage state.

"""
txt_display_2 = '--- '

import os
GOOGLE_API_KEY=os.getenv('GOOGLE_API_KEY')
genai.configure(api_key=GOOGLE_API_KEY)
SMS_URL =os.getenv('SMS_URL')
SMS_TOK =os.getenv('SMS_TOK')

sms_text ="..."
# Image to Base 64 Converter
def image_to_base64(image_path):
    with open(image_path, 'rb') as img:
        encoded_string = base64.b64encode(img.read())
    return encoded_string.decode('utf-8')

# Function that takes User Inputs, generates Response and displays on Chat UI
def app1_response(img):
    if not img:
        response = txt_model.generate_content(txt_prompt_1)
        return response

    else:
        img = PIL.Image.open(img)
        response = vis_model.generate_content([txt_prompt_1,img])
        return response.text

def app2_response(img):
    if not img:
        response = txt_model.generate_content(txt_prompt_2)
        return response

    else:
        img = PIL.Image.open(img)
        response = vis_model.generate_content([txt_prompt_2,img])
        return response.text


# SMS service ends in March 2024, to restore service @Sinch Simple Text
def send_SMS(resp_text):
    url = SMS_URL
    headers = {
        "Authorization": SMS_TOK,
        "Content-Type": "application/json"
    }

    data = {
        "from": "12085686834",
        "to": ["18587331029"],
        "body": resp_text
    }

    # response = requests.post(url, json=data, headers=headers)
    # return response.text
    return "....."
    
# gradio block
with gr.Blocks() as app1:
    with gr.Column():
        gr.Markdown("## 🥷 to Samuraize ##")
        gr.Markdown("```for email ✉ and/or assigment descriptions ⌦, paste screenshot here...```")
        image_box = gr.Image(label="✂ email screen", type="filepath")
        btn1 = gr.Button("Generate To-Dos ☑")
        out1 = gr.Textbox(label="here are the actionables...")
        btn2 = gr.Button("send to Mobile ↗️")
        out2 = gr.Textbox(label="response from SMS gateway...")
    
        btn1.click(fn=app1_response, inputs=[image_box], outputs=out1)
        btn2.click(fn=send_SMS, inputs=out1, outputs=out2)

    gr.Markdown("""
    # 🥷 Summerize eMail & Make a Plan #
    
    	- screen capture (Win + shift + S)
        - click ☑ to upload
        - await LLM Bot (Gemini, in this case) response
        - receive THREE actionable items
    
    [demo video](https://youtu.be/lJ4jIAEVRNY)

    """)

    gr.Markdown("""
    # try with the mockup email below#
    ![email](./missionemail.png)
    
    ![email](https://huggingface.co/spaces/hsienchen/todo/blob/main/missionemail.png)
    """)
    
with gr.Blocks() as app2:
    with gr.Column():
        gr.Markdown("## 🦫 Stock-Out Squirrel ##")
        gr.Markdown("```Win+Screenshot, paste ERP Inv ⌨ screenshot here...```")
        image_box = gr.Image(label="✂ ERP screen",type="filepath")
        btn1 = gr.Button("Check ROQ ☑")
        out1 = gr.Textbox(label="here is the watch list 📄...")
        btn2 = gr.Button("send out reminders ↗️")
        out2 = gr.Textbox(label="response or feed back?")
    
        btn1.click(fn=app2_response, inputs=[image_box], outputs=out1)
        btn2.click(fn=send_SMS, inputs=out1, outputs=out2)

    gr.Markdown("""
    # 🦫 Check Inventory ROQ #
    (contains bugs, more works needed)
    	- screen capture (Win + shift + S)
        - click ☑ to upload
        - await LLM Bot (Gemini, in this case) response
        - send ↗️ to related parties

    """)

    gr.Markdown("""
- [x] Sample Inventory Table
- [X] Screen Capture and Paste
    """)
    gr.Markdown("""
| Item ID  | Quantity  | Minimum|Contact Phone|
|--|--|--|--|
| #01 | 50 | 20 | 1234|
| #02 | 40 | 50 | 3456|
| #03 | 80 | 20 | 1234|

    """)
with gr.Blocks() as app3:
    with gr.Column():
        gr.Markdown("## 🐇 Route Planning Rabbit ##")
        gr.Markdown("```Win+Screenshot, paste ERP Inv ⌨ screenshot here...```")
        image_box = gr.Image(label="✂ Pick list",type="filepath")
        btn1 = gr.Button("Generate a Route Plan ☑")
        out1 = gr.Textbox(label="here is the watch list 📄...")
        btn2 = gr.Button("send to Mobile (disabled) ↗️")
        out2 = gr.Textbox(label="no message been sent, this is just a mockup confirmation...")
    
        btn1.click(fn=app2_response, inputs=[image_box], outputs=out1)
        btn2.click(fn=send_SMS, inputs=out1, outputs=out2)

    gr.Markdown("""
    # 🐇 Route Planning #
    (concept prototype)
    	- screen capture (Win + shift + S)
        - click ☑ to upload
        - await LLM Bot (Gemini, in this case) response
        - send ↗️ to related parties
        

    """)

with gr.Blocks() as demo:
    gr.Markdown("## To-Do Samuraizer 🥷 + Stock-Out Squirrel🦫 + Routing 🐇 ##")
    gr.TabbedInterface([app1, app2, app3], ["➀ To-Do", "➁ SOS", "➂ Picking Routes"])

demo.queue()
demo.launch()