app.py

!pip install accelerate>=0.12.0 transformers[torch]==4.25.1
!pip install bitsandbytes
from transformers import AutoModelForCausalLM, AutoTokenizer
from instruct_pipeline import InstructionTextGenerationPipeline
tokenizer = AutoTokenizer.from_pretrained("databricks/dolly-v2-7b", padding_side="left")
model = AutoModelForCausalLM.from_pretrained("databricks/dolly-v2-7b", device_map="auto", offload_folder="/content/temp",
                                             load_in_8bit=True)
                                             
generate_text = InstructionTextGenerationPipeline(model=model, tokenizer=tokenizer)

import gradio as gr

def greet(name):
    return generate_text(name)
    

iface = gr.Interface(fn=greet, inputs="text", outputs="text")
iface.launch(share=True)