jbilcke-hf HF staff commited on
Commit
7448744
1 Parent(s): c141ccf

update to use the new improved API

Browse files
.env CHANGED
@@ -1,7 +1,5 @@
1
-
2
-
3
-
4
- AUTH_REPLICATE_API_TOKEN="<YOUR SECRET>"
5
 
6
  # TODO: those could be
7
  # VIDEO_HOTSHOT_XL_API_OFFICIAL
@@ -16,12 +14,12 @@ VIDEO_HOTSHOT_XL_API_OFFICIAL=""
16
 
17
  # the Node API developed by @jbilcke-hf (me) to allow using any LoRA
18
  # note: doesn't work yet
19
- VIDEO_HOTSHOT_XL_API_NODE="https://jbilcke-hf-hotshot-xl-api.hf.space/"
20
 
21
  # the Gradio Space (with the auto API) developed by @fffiloni
22
  # forked to support multiple LoRAs too
23
  # note: work in progress
24
- VIDEO_HOTSHOT_XL_API_GRADIO="https://jbilcke-hf-hotshot-xl-server.hf.space/"
25
 
26
  # If you decided to use Replicate for the RENDERING engine
27
  VIDEO_HOTSHOT_XL_API_REPLICATE_MODEL="cloneofsimo/hotshot-xl-lora-controlnet"
 
1
+ AUTH_REPLICATE_API_TOKEN="<USE YOUR OWN>"
2
+ AUTH_HOTSHOT_XL_API_GRADIO_ACCESS_TOKEN="<USE YOUR OWN>"
 
 
3
 
4
  # TODO: those could be
5
  # VIDEO_HOTSHOT_XL_API_OFFICIAL
 
14
 
15
  # the Node API developed by @jbilcke-hf (me) to allow using any LoRA
16
  # note: doesn't work yet
17
+ VIDEO_HOTSHOT_XL_API_NODE="https://jbilcke-hf-hotshot-xl-api.hf.space"
18
 
19
  # the Gradio Space (with the auto API) developed by @fffiloni
20
  # forked to support multiple LoRAs too
21
  # note: work in progress
22
+ VIDEO_HOTSHOT_XL_API_GRADIO="https://jbilcke-hf-hotshot-xl-server-1.hf.space"
23
 
24
  # If you decided to use Replicate for the RENDERING engine
25
  VIDEO_HOTSHOT_XL_API_REPLICATE_MODEL="cloneofsimo/hotshot-xl-lora-controlnet"
src/app/interface/generate/index.tsx CHANGED
@@ -46,8 +46,31 @@ export function Generate() {
46
  console.log("starting transition, calling generateAnimation")
47
  const newAssetUrl = await generateAnimation({
48
  prompt: promptDraft,
49
- lora: "https://huggingface.co/ostris/crayon_style_lora_sdxl/resolve/main/crayons_v1_sdxl.safetensors",
50
- size: "512x512" // "320x768"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
51
  })
52
  setAssetUrl(newAssetUrl)
53
  } catch (err) {
@@ -120,7 +143,8 @@ export function Generate() {
120
  ? `bg-sky-100 text-sky-500 border-transparent`
121
  : `bg-sky-200 text-sky-600 selection:bg-sky-200`,
122
  `text-left`,
123
- `text-xl leading-10 px-6 h-16 pt-1`
 
124
  )}
125
  value={promptDraft}
126
  onChange={e => setPromptDraft(e.target.value)}
 
46
  console.log("starting transition, calling generateAnimation")
47
  const newAssetUrl = await generateAnimation({
48
  prompt: promptDraft,
49
+ huggingFaceLora: "KappaNeuro/studio-ghibli-style",
50
+ // huggingFaceLora: "veryVANYA/ps1-graphics-sdxl-v2", //
51
+ // huggingFaceLora: "ostris/crayon_style_lora_sdxl", // "https://huggingface.co/ostris/crayon_style_lora_sdxl/resolve/main/crayons_v1_sdxl.safetensors",
52
+ // replicateLora: "https://replicate.com/jbilcke/sdxl-panorama",
53
+
54
+ // ---- replicate models -----
55
+ // use: "in the style of TOK" in the prompt!
56
+ // or this? "in the style of <s0><s1>"
57
+ // I don't see a lot of diff
58
+ //
59
+ // Zelda BOTW
60
+ // replicateLora: "https://pbxt.replicate.delivery/8UkalcGbGnrNHxGeqeCrhKcPbrRDlx4vLToRRlUWqzpnfieFB/trained_model.tar",
61
+
62
+ // Zelda64
63
+ // replicateLora: "https://pbxt.replicate.delivery/HPZlvCwDWtb5KpefUUcofwvZwTbrZAH9oLvzrn24hqUcQBfFB/trained_model.tar",
64
+
65
+ // panorama lora
66
+ // replicateLora: "https://pbxt.replicate.delivery/nuXez5QNfEmhPk1TLGtl8Q0TwyucZbzTsfUe1ibUfNV0JrMMC/trained_model.tar",
67
+
68
+ // foundation
69
+ // replicateLora: "https://pbxt.replicate.delivery/VHU109Irgh6EPJrZ7aVScvadYDqXhlL3izfEAjfhs8Cvz0hRA/trained_model.tar",
70
+
71
+ size: "768x320", // "1024x512", // "512x512" // "320x768"
72
+
73
+ steps: 40,
74
  })
75
  setAssetUrl(newAssetUrl)
76
  } catch (err) {
 
143
  ? `bg-sky-100 text-sky-500 border-transparent`
144
  : `bg-sky-200 text-sky-600 selection:bg-sky-200`,
145
  `text-left`,
146
+ `text-xl leading-10 px-6 h-16 pt-1`,
147
+ `selection:bg-sky-800 selection:text-sky-200`
148
  )}
149
  value={promptDraft}
150
  onChange={e => setPromptDraft(e.target.value)}
src/app/server/actions/animation.ts CHANGED
@@ -1,42 +1,38 @@
1
  "use server"
2
 
3
- import Replicate from "replicate"
4
 
5
- import { generateSeed } from "@/lib/generateSeed"
6
- import { ImageInferenceSize } from "@/types"
7
- import { sleep } from "@/lib/sleep"
8
 
9
  const videoEngine = `${process.env.VIDEO_ENGINE || ""}`
10
 
11
- const officialApi = `${process.env.VIDEO_HOTSHOT_XL_API_OFFICIAL || ""}`
12
  const nodeApi = `${process.env.VIDEO_HOTSHOT_XL_API_NODE || ""}`
13
- const gradioApi = `${process.env.VIDEO_HOTSHOT_XL_API_GRADIO || ""}`
14
-
15
- const replicateToken = `${process.env.AUTH_REPLICATE_API_TOKEN || ""}`
16
- const replicateModel = `${process.env.VIDEO_HOTSHOT_XL_API_REPLICATE_MODEL || ""}`
17
- const replicateModelVersion = `${process.env.VIDEO_HOTSHOT_XL_API_REPLICATE_MODEL_VERSION || ""}`
18
 
19
  export async function generateAnimation({
20
- prompt,
21
- size,
22
- lora,
23
- }: {
24
- prompt: string
25
- size: ImageInferenceSize
26
- lora: string
27
- }): Promise<string> {
28
- if (!prompt?.length) {
 
29
  throw new Error(`prompt is too short!`)
30
  }
31
 
32
- // pimp themy prompt
33
- prompt = [
 
34
  "beautiful",
35
- prompt,
36
  "hd"
37
  ].join(", ")
38
 
39
- const negativePrompt = [
 
40
  "cropped",
41
  "dark",
42
  "underexposed",
@@ -45,76 +41,20 @@ export async function generateAnimation({
45
  "watermarked",
46
  ].join(", ")
47
 
48
- const [width, height] = size.split("x").map(x => Number(x))
49
-
50
  try {
51
 
52
  if (videoEngine === "VIDEO_HOTSHOT_XL_API_REPLICATE") {
53
- if (!replicateToken) {
54
- throw new Error(`you need to configure your AUTH_REPLICATE_API_TOKEN in order to use the REPLICATE rendering engine`)
55
- }
56
- if (!replicateModel) {
57
- throw new Error(`you need to configure your RENDERING_REPLICATE_API_MODEL in order to use the REPLICATE rendering engine`)
58
- }
59
-
60
- if (!replicateModelVersion) {
61
- throw new Error(`you need to configure your REPLICATE_API_MODEL_VERSION in order to use the REPLICATE rendering engine`)
62
- }
63
- const replicate = new Replicate({ auth: replicateToken })
64
-
65
-
66
- // console.log("Calling replicate..")
67
- const seed = generateSeed()
68
- const prediction = await replicate.predictions.create({
69
- version: replicateModelVersion,
70
- input: {
71
- prompt,
72
- negative_prompt: negativePrompt,
73
- hf_lora_url: lora,
74
- width,
75
- height,
76
-
77
- // those are used to create an upsampling or downsampling
78
- // original_width: width,
79
- // original_height: height,
80
- // target_width: width,
81
- // target_height: height,
82
-
83
- video_length: 8, // nb frames
84
- video_duration: 1000, // video duration in ms
85
- // seed
86
- }
87
  })
88
 
89
- // console.log("prediction:", prediction)
90
-
91
- // no need to reply straight away as images take time to generate, this isn't instantaneous
92
- // also our friends at Replicate won't like it if we spam them with requests
93
- await sleep(5000)
94
-
95
-
96
- const res = await fetch(`https://api.replicate.com/v1/predictions/${prediction.id}`, {
97
- method: "GET",
98
- headers: {
99
- Authorization: `Token ${replicateToken}`,
100
- },
101
- cache: 'no-store',
102
- // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
103
- // next: { revalidate: 1 }
104
- })
105
-
106
- // Recommendation: handle errors
107
- if (res.status !== 200) {
108
- // This will activate the closest `error.js` Error Boundary
109
- throw new Error('Failed to fetch data')
110
- }
111
-
112
- const response = (await res.json()) as any
113
- const error = `${response?.error || ""}`
114
- if (error) {
115
- throw new Error(error)
116
- }
117
- return `${response?.output || ""}`
118
  } else if (videoEngine === "VIDEO_HOTSHOT_XL_API_NODE") {
119
  // TODO: support other API to avoid duplicate work?
120
  // (are the other API supporting custom LoRAs?)
@@ -126,8 +66,8 @@ export async function generateAnimation({
126
  // Authorization: `Bearer ${token}`,
127
  },
128
  body: JSON.stringify({
129
- prompt,
130
- lora,
131
  size,
132
  }),
133
  cache: "no-store",
@@ -146,36 +86,16 @@ export async function generateAnimation({
146
 
147
  return content
148
  } else if (videoEngine === "VIDEO_HOTSHOT_XL_API_GRADIO") {
149
- const res = await fetch(gradioApi + (gradioApi.endsWith("/") ? "" : "/") + "api/predict", {
150
- method: "POST",
151
- headers: {
152
- "Content-Type": "application/json",
153
- // access isn't secured for now, the free lunch is open
154
- // Authorization: `Bearer ${token}`,
155
- },
156
- body: JSON.stringify({
157
- fn_index: 0,
158
- data: [
159
- prompt,
160
- lora,
161
- size,
162
- ],
163
- }),
164
- cache: "no-store",
165
- // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
166
- // next: { revalidate: 1 }
167
  })
168
-
169
- const { data } = await res.json()
170
-
171
- // Recommendation: handle errors
172
- if (res.status !== 200) {
173
- // This will activate the closest `error.js` Error Boundary
174
- throw new Error('Failed to fetch data')
175
- }
176
- // console.log("data:", data.slice(0, 50))
177
-
178
- return data
179
  } else {
180
  throw new Error(`not implemented yet!`)
181
  }
 
1
  "use server"
2
 
3
+ import { VideoOptions } from "@/types"
4
 
5
+ import { generateVideoWithGradioAPI } from "./generateWithGradioApi"
6
+ import { generateVideoWithReplicateAPI } from "./generateWithReplicateAPI"
 
7
 
8
  const videoEngine = `${process.env.VIDEO_ENGINE || ""}`
9
 
10
+ // const officialApi = `${process.env.VIDEO_HOTSHOT_XL_API_OFFICIAL || ""}`
11
  const nodeApi = `${process.env.VIDEO_HOTSHOT_XL_API_NODE || ""}`
 
 
 
 
 
12
 
13
  export async function generateAnimation({
14
+ positivePrompt = "",
15
+ negativePrompt = "",
16
+ size = "512x512",
17
+ huggingFaceLora,
18
+ replicateLora,
19
+ nbFrames = 8,
20
+ duration = 1000,
21
+ steps = 30,
22
+ }: VideoOptions): Promise<string> {
23
+ if (!positivePrompt?.length) {
24
  throw new Error(`prompt is too short!`)
25
  }
26
 
27
+ // pimp the prompt
28
+ positivePrompt = [
29
+ positivePrompt,
30
  "beautiful",
 
31
  "hd"
32
  ].join(", ")
33
 
34
+ negativePrompt = [
35
+ negativePrompt,
36
  "cropped",
37
  "dark",
38
  "underexposed",
 
41
  "watermarked",
42
  ].join(", ")
43
 
 
 
44
  try {
45
 
46
  if (videoEngine === "VIDEO_HOTSHOT_XL_API_REPLICATE") {
47
+ return generateVideoWithReplicateAPI({
48
+ positivePrompt,
49
+ negativePrompt,
50
+ size,
51
+ huggingFaceLora,
52
+ replicateLora,
53
+ nbFrames,
54
+ duration,
55
+ steps,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
56
  })
57
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
58
  } else if (videoEngine === "VIDEO_HOTSHOT_XL_API_NODE") {
59
  // TODO: support other API to avoid duplicate work?
60
  // (are the other API supporting custom LoRAs?)
 
66
  // Authorization: `Bearer ${token}`,
67
  },
68
  body: JSON.stringify({
69
+ prompt: positivePrompt,
70
+ lora: huggingFaceLora,
71
  size,
72
  }),
73
  cache: "no-store",
 
86
 
87
  return content
88
  } else if (videoEngine === "VIDEO_HOTSHOT_XL_API_GRADIO") {
89
+ return generateVideoWithGradioAPI({
90
+ positivePrompt,
91
+ negativePrompt,
92
+ size,
93
+ huggingFaceLora,
94
+ replicateLora,
95
+ nbFrames,
96
+ duration,
97
+ steps,
 
 
 
 
 
 
 
 
 
98
  })
 
 
 
 
 
 
 
 
 
 
 
99
  } else {
100
  throw new Error(`not implemented yet!`)
101
  }
src/app/server/actions/generateWithGradioApi.ts ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { generateSeed } from "@/lib/generateSeed"
2
+ import { VideoOptions } from "@/types"
3
+
4
+ const gradioApi = `${process.env.VIDEO_HOTSHOT_XL_API_GRADIO || ""}`
5
+ const accessToken = `${process.env.AUTH_HOTSHOT_XL_API_GRADIO_ACCESS_TOKEN || ""}`
6
+
7
+ export async function generateVideoWithGradioAPI({
8
+ positivePrompt = "",
9
+ negativePrompt = "",
10
+ size = "512x512",
11
+ huggingFaceLora,
12
+ // replicateLora, // not supported yet
13
+ nbFrames = 8,
14
+ duration = 1000,
15
+ steps = 30,
16
+ }: VideoOptions): Promise<string> {
17
+ const res = await fetch(gradioApi + (gradioApi.endsWith("/") ? "" : "/") + "api/predict", {
18
+ method: "POST",
19
+ headers: {
20
+ "Content-Type": "application/json",
21
+ // Authorization: `Bearer ${token}`,
22
+ },
23
+ body: JSON.stringify({
24
+ fn_index: 0,
25
+ data: [
26
+ accessToken,
27
+ positivePrompt,
28
+ negativePrompt,
29
+ huggingFaceLora,
30
+ size,
31
+ generateSeed(),
32
+ steps,
33
+ nbFrames,
34
+ duration,
35
+ ],
36
+ }),
37
+ cache: "no-store",
38
+ // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
39
+ // next: { revalidate: 1 }
40
+ })
41
+
42
+ const { data } = await res.json()
43
+
44
+ // Recommendation: handle errors
45
+ if (res.status !== 200) {
46
+ // This will activate the closest `error.js` Error Boundary
47
+ throw new Error('Failed to fetch data')
48
+ }
49
+ // console.log("data:", data.slice(0, 50))
50
+
51
+ return data
52
+ }
src/app/server/actions/generateWithReplicateAPI.ts ADDED
@@ -0,0 +1,114 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import Replicate from "replicate"
2
+
3
+ import { generateSeed } from "@/lib/generateSeed"
4
+ import { VideoOptions } from "@/types"
5
+ import { sleep } from "@/lib/sleep"
6
+
7
+ const replicateToken = `${process.env.AUTH_REPLICATE_API_TOKEN || ""}`
8
+ const replicateModel = `${process.env.VIDEO_HOTSHOT_XL_API_REPLICATE_MODEL || ""}`
9
+ const replicateModelVersion = `${process.env.VIDEO_HOTSHOT_XL_API_REPLICATE_MODEL_VERSION || ""}`
10
+
11
+ export async function generateVideoWithReplicateAPI({
12
+ positivePrompt = "",
13
+ negativePrompt = "",
14
+ size = "512x512",
15
+ huggingFaceLora,
16
+ replicateLora,
17
+ steps = 30
18
+ }: VideoOptions): Promise<string> {
19
+ if (!replicateToken) {
20
+ throw new Error(`you need to configure your AUTH_REPLICATE_API_TOKEN in order to use the REPLICATE rendering engine`)
21
+ }
22
+ if (!replicateModel) {
23
+ throw new Error(`you need to configure your RENDERING_REPLICATE_API_MODEL in order to use the REPLICATE rendering engine`)
24
+ }
25
+
26
+ if (!replicateModelVersion) {
27
+ throw new Error(`you need to configure your REPLICATE_API_MODEL_VERSION in order to use the REPLICATE rendering engine`)
28
+ }
29
+ const replicate = new Replicate({ auth: replicateToken })
30
+
31
+ const [width, height] = size.split("x").map(x => Number(x))
32
+
33
+ // console.log("Calling replicate..")
34
+ const seed = generateSeed()
35
+ const prediction = await replicate.predictions.create({
36
+ version: replicateModelVersion,
37
+ input: {
38
+ prompt,
39
+ negative_prompt: negativePrompt,
40
+
41
+ // this is not a URL but a model name
42
+ hf_lora_url: replicateLora ? undefined : huggingFaceLora,
43
+
44
+ // this is a URL to the .tar (we can get it from the "trainings" page)
45
+ replicate_weights_url: huggingFaceLora ? undefined : replicateLora,
46
+
47
+ width,
48
+ height,
49
+
50
+ // those are used to create an upsampling or downsampling
51
+ // original_width: width,
52
+ // original_height: height,
53
+ // target_width: width,
54
+ // target_height: height,
55
+
56
+ video_length: 8, // nb frames
57
+ video_duration: 1000, // video duration in ms
58
+
59
+ seed
60
+ }
61
+ })
62
+
63
+ // console.log("prediction:", prediction)
64
+
65
+ // response times are random on Replicate, so this sleep doesn't really help us
66
+ // we should of course do it like on the AI Comic Factory (use the async loading/polling)
67
+ // but on the other hand if it takes 2 minutes to answer, it's not really useful either
68
+ await sleep(50000)
69
+
70
+ try {
71
+ const res = await fetch(`https://api.replicate.com/v1/predictions/${prediction.id}`, {
72
+ method: "GET",
73
+ headers: {
74
+ Authorization: `Token ${replicateToken}`,
75
+ },
76
+ cache: 'no-store',
77
+ })
78
+
79
+ if (res.status !== 200) {
80
+ throw new Error("failed")
81
+ }
82
+
83
+ const response = (await res.json()) as any
84
+ const error = `${response?.error || ""}`
85
+ if (error) {
86
+ throw new Error(error)
87
+ }
88
+ return `${response?.output || ""}`
89
+ } catch (err) {
90
+ // OK let's give replicate a second chance
91
+ await sleep(10000)
92
+
93
+ const res = await fetch(`https://api.replicate.com/v1/predictions/${prediction.id}`, {
94
+ method: "GET",
95
+ headers: {
96
+ Authorization: `Token ${replicateToken}`,
97
+ },
98
+ cache: 'no-store',
99
+ // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
100
+ // next: { revalidate: 1 }
101
+ })
102
+
103
+ if (res.status !== 200) {
104
+ throw new Error('Failed to fetch data')
105
+ }
106
+
107
+ const response = (await res.json()) as any
108
+ const error = `${response?.error || ""}`
109
+ if (error) {
110
+ throw new Error(error)
111
+ }
112
+ return `${response?.output || ""}`
113
+ }
114
+ }
src/types.ts CHANGED
@@ -1,11 +1,3 @@
1
- export type ImageInferenceSize =
2
- | '320x768'
3
- | '384x672'
4
- | '416x608'
5
- | '512x512'
6
- | '608x416'
7
- | '672x384'
8
- | '768x320'
9
 
10
  export type ProjectionMode = 'cartesian' | 'spherical'
11
 
@@ -273,3 +265,38 @@ export type CurrentPanel =
273
  | "join"
274
  | "play"
275
  | "results"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
 
2
  export type ProjectionMode = 'cartesian' | 'spherical'
3
 
 
265
  | "join"
266
  | "play"
267
  | "results"
268
+
269
+ // vendor-specific types
270
+
271
+ export type HotshotImageInferenceSize =
272
+ | '320x768'
273
+ | '384x672'
274
+ | '416x608'
275
+ | '512x512'
276
+ | '608x416'
277
+ | '672x384'
278
+ | '768x320'
279
+ | '1024x1024' // custom ratio - this isn't supported / supposed to work properly
280
+ | '1024x512' // custom panoramic ratio - this isn't supported / supposed to work properly
281
+ | '1024x576' // movie ratio (16:9) this isn't supported / supposed to work properly
282
+ | '576x1024' // tiktok ratio (9:16) this isn't supported / supposed to work properly
283
+
284
+ export type VideoOptions = {
285
+ positivePrompt: string
286
+
287
+ negativePrompt?: string
288
+
289
+ size?: HotshotImageInferenceSize
290
+
291
+ /**
292
+ * Must be a model *name*
293
+ */
294
+ huggingFaceLora?: string
295
+
296
+ replicateLora?: string
297
+
298
+ nbFrames?: number // FPS (eg. 8)
299
+ duration?: number // in milliseconds
300
+
301
+ steps?: number
302
+ }