bilgeyucel commited on
Commit
b8462d5
1 Parent(s): 9b95df0
Files changed (1) hide show
  1. app.py +22 -16
app.py CHANGED
@@ -1,9 +1,11 @@
1
  import os
2
  import gradio as gr
3
 
4
- from haystack.nodes import TransformersImageToText
5
- from haystack.nodes import PromptNode, PromptTemplate
6
  from haystack import Pipeline
 
 
7
 
8
  description = """
9
  # Captionate 📸
@@ -15,31 +17,35 @@ description = """
15
 
16
  `mistralai/Mistral-7B-Instruct-v0.2` performs best, but try different models to see how they react to the same prompt.
17
 
18
- Built by [Bilge Yucel](https://twitter.com/bilgeycl) using [Haystack](https://github.com/deepset-ai/haystack) 💙
19
  """
20
 
21
- image_to_text = TransformersImageToText(
22
- model_name_or_path="nlpconnect/vit-gpt2-image-captioning",
23
- progress_bar=True
24
- )
25
-
26
- prompt_template = PromptTemplate(prompt="""
27
  You will receive a descriptive text of a photo.
28
  Try to generate a nice Instagram caption with a phrase rhyming with the text. Include emojis in the caption.
29
 
30
- Descriptive text: {documents};
31
  Instagram Caption:
32
- """)
33
 
34
  hf_api_key = os.environ["HF_API_KEY"]
35
 
36
  def generate_caption(image_file_paths, model_name):
 
 
 
 
 
37
  captioning_pipeline = Pipeline()
38
- prompt_node = PromptNode(model_name_or_path=model_name, api_key=hf_api_key, default_prompt_template=prompt_template, model_kwargs={"trust_remote_code":True})
39
- captioning_pipeline.add_node(component=image_to_text, name="image_to_text", inputs=["File"])
40
- captioning_pipeline.add_node(component=prompt_node, name="prompt_node", inputs=["image_to_text"])
41
- caption = captioning_pipeline.run(file_paths=[image_file_paths])
42
- return caption["results"][0]
 
 
 
 
43
 
44
  with gr.Blocks(theme="soft") as demo:
45
  gr.Markdown(value=description)
 
1
  import os
2
  import gradio as gr
3
 
4
+ from haystack.components.generators import HuggingFaceTGIGenerator
5
+ from haystack.components.builders.prompt_builder import PromptBuilder
6
  from haystack import Pipeline
7
+ from haystack.utils import Secret
8
+ from image_captioner import ImageCaptioner
9
 
10
  description = """
11
  # Captionate 📸
 
17
 
18
  `mistralai/Mistral-7B-Instruct-v0.2` performs best, but try different models to see how they react to the same prompt.
19
 
20
+ Built by [Bilge Yucel](https://twitter.com/bilgeycl) using [Haystack 2.0](https://github.com/deepset-ai/haystack) 💙
21
  """
22
 
23
+ prompt_template = """
 
 
 
 
 
24
  You will receive a descriptive text of a photo.
25
  Try to generate a nice Instagram caption with a phrase rhyming with the text. Include emojis in the caption.
26
 
27
+ Descriptive text: {{captions[0]}};
28
  Instagram Caption:
29
+ """
30
 
31
  hf_api_key = os.environ["HF_API_KEY"]
32
 
33
  def generate_caption(image_file_paths, model_name):
34
+ image_to_text = ImageCaptioner(
35
+ model_name="nlpconnect/vit-gpt2-image-captioning",
36
+ )
37
+ prompt_builder = PromptBuilder(template=prompt_template)
38
+ generator = HuggingFaceTGIGenerator(model=model_name, token=Secret.from_token(hf_api_key))
39
  captioning_pipeline = Pipeline()
40
+ captioning_pipeline.add_component("image_to_text", image_to_text)
41
+ captioning_pipeline.add_component("prompt_builder", prompt_builder)
42
+ captioning_pipeline.add_component("generator", generator)
43
+
44
+ captioning_pipeline.connect("image_to_text.captions", "prompt_builder.captions")
45
+ captioning_pipeline.connect("prompt_builder", "generator")
46
+
47
+ result = captioning_pipeline.run({"image_to_text":{"image_file_paths":image_file_paths}})
48
+ return result["generator"][0]
49
 
50
  with gr.Blocks(theme="soft") as demo:
51
  gr.Markdown(value=description)