clusterize musicgen for today's release

package-lock.json

@@ -8,9 +8,9 @@
       "name": "@aitube/website",
       "version": "0.0.0",
       "dependencies": {
-        "@aitube/clap": "0.0.16",
-        "@aitube/client": "0.0.24",
-        "@aitube/engine": "0.0.6",
+        "@aitube/clap": "0.0.17",
+        "@aitube/client": "0.0.25",
+        "@aitube/engine": "0.0.7",
         "@huggingface/hub": "0.12.3-oauth",
         "@huggingface/inference": "^2.6.7",
         "@jcoreio/async-throttle": "^1.6.0",
@@ -118,9 +118,9 @@
       }
     },
     "node_modules/@aitube/clap": {
-      "version": "0.0.16",
-      "resolved": "https://registry.npmjs.org/@aitube/clap/-/clap-0.0.16.tgz",
-      "integrity": "sha512-EcBu4gbc8Kx58rkMmLjh0uyCWVR4cSgWqALlQ50C1GbHRg/r81gDET6faWeos14ZGrtg4B0CJv8WUTIDE54pjg==",
+      "version": "0.0.17",
+      "resolved": "https://registry.npmjs.org/@aitube/clap/-/clap-0.0.17.tgz",
+      "integrity": "sha512-g/jjePX2Hz9Eo4hk+rxd6FRwoy2Hx9sadGLgN9yWSm7dGHhr9B/DVv8eLjFabu7jgW0zvZZ1FHvlsNAby4Pr/Q==",
       "dependencies": {
         "pure-uuid": "^1.8.1",
         "yaml": "^2.4.1"
@@ -130,22 +130,22 @@
       }
     },
     "node_modules/@aitube/client": {
-      "version": "0.0.24",
-      "resolved": "https://registry.npmjs.org/@aitube/client/-/client-0.0.24.tgz",
-      "integrity": "sha512-9J3PhVabyc/aOcB5j7wF5Fxb4VORB6aoHTTY6Y+ciFi96zp1YcFeYBMmBpIDq/6KqucIJFtANtZDPoy5a/j1Og==",
+      "version": "0.0.25",
+      "resolved": "https://registry.npmjs.org/@aitube/client/-/client-0.0.25.tgz",
+      "integrity": "sha512-gX5eJOKiigVY3xK1NcsStruUuWMQbj4o4XHTceZhUyKCgHDldC0Y15mvIWVabCtEW5FFebdmhH0EFeg+PBMCsg==",
       "dependencies": {
         "query-string": "^9.0.0"
       },
       "peerDependencies": {
-        "@aitube/clap": "0.0.16"
+        "@aitube/clap": "0.0.17"
       }
     },
     "node_modules/@aitube/engine": {
-      "version": "0.0.6",
-      "resolved": "https://registry.npmjs.org/@aitube/engine/-/engine-0.0.6.tgz",
-      "integrity": "sha512-Li+yE6MCIvD32Kyn43I7GHyQkmxtc/1/lrt3xPKMuFrWuc2FP9JR1ZYRlKqodmWqrSmhWCRBOt3Z6f62QpEvIQ==",
+      "version": "0.0.7",
+      "resolved": "https://registry.npmjs.org/@aitube/engine/-/engine-0.0.7.tgz",
+      "integrity": "sha512-Bu3MhQ7DelO+K/lU82MGSu1ksf3IUi3L9q1E2WfdBh4bkI0Kq8hH+OgUFKp38e5t1zhwjY2lJPNCbAJau5RcBA==",
       "peerDependencies": {
-        "@aitube/clap": "0.0.16"
+        "@aitube/clap": "0.0.17"
       }
     },
     "node_modules/@alloc/quick-lru": {
@@ -6081,9 +6081,9 @@
       }
     },
     "node_modules/jose": {
-      "version": "5.2.4",
-      "resolved": "https://registry.npmjs.org/jose/-/jose-5.2.4.tgz",
-      "integrity": "sha512-6ScbIk2WWCeXkmzF6bRPmEuaqy1m8SbsRFMa/FLrSCkGIhj8OLVG/IH+XHVmNMx/KUo8cVWEE6oKR4dJ+S0Rkg==",
+      "version": "5.3.0",
+      "resolved": "https://registry.npmjs.org/jose/-/jose-5.3.0.tgz",
+      "integrity": "sha512-IChe9AtAE79ru084ow8jzkN2lNrG3Ntfiv65Cvj9uOCE2m5LNsdHG+9EbxWxAoWRF9TgDOqLN5jm08++owDVRg==",
       "funding": {
         "url": "https://github.com/sponsors/panva"
       }

package.json

@@ -10,9 +10,9 @@
     "lint": "next lint"
   },
   "dependencies": {
-    "@aitube/clap": "0.0.16",
-    "@aitube/client": "0.0.24",
-    "@aitube/engine": "0.0.6",
+    "@aitube/clap": "0.0.17",
+    "@aitube/client": "0.0.25",
+    "@aitube/engine": "0.0.7",
     "@huggingface/hub": "0.12.3-oauth",
     "@huggingface/inference": "^2.6.7",
     "@jcoreio/async-throttle": "^1.6.0",

src/app/api/generators/music/generateMusicAsBase64.ts

@@ -1,72 +0,0 @@
-import { sleep } from "@/lib/utils/sleep"
-import { generateMusicWithMusicgen } from "./generateMusicWithMusicgen"
-
-// apparently if we ask to generate like 4 minutes of audio, it crashes
-const maxAudioDurationInSec = 120
-
-// generate music
-// this may generate multiple tracks (one after another)
-// if the durationInSec parameter is larger than the max audio duration
-export async function generateMusicAsBase64({
-    prompt,
-    durationInSec,
-    hd = false,
-  }: {
-    prompt: string
-    durationInSec: number
-
-    // use diffusion (increases quality, but requires more RAM)
-    hd?: boolean
-  }): Promise<string[]> {
-  
-  const musicPrompt = prompt || ""
- 
-  if (durationInSec < 1 || !musicPrompt) { return [] }
-  
-  if (durationInSec > maxAudioDurationInSec) {
-    const halfTheDuration = Math.round(durationInSec / 2)
-  
-    // no! we shouldn't generate them in parallel
-    // or at least, no now, because we only have ONE music server!
-    // const chunks = await Promise.all([
-    //  generateMusic({ video, durationInSec: halfTheDuration })
-    //])
-    // return chunks.reduce((acc, tracks) => ([...acc, ...tracks]), [])
-
-    // instead, let's play it safe and generate them one after another
-    let chunks: string[] = []
-    const partA = await generateMusicAsBase64({ prompt, hd, durationInSec: halfTheDuration })
-    if (partA) { chunks = chunks.concat(partA) }
-
-    const partB = await generateMusicAsBase64({ prompt, hd, durationInSec: halfTheDuration })
-    if (partB) { chunks = chunks.concat(partB) }
-
-    return [...partA, ...partB]
-  }
-
-  let musicTracks: string[] = []
-
-  const musicParams = {
-    prompt: musicPrompt,
-    durationInSec,
-    hd,
-  }
-  try {
-    console.log(`  |- generating ${durationInSec} seconds of music..`)
-    const musicTrack = await generateMusicWithMusicgen(musicParams)
-    if (!musicTrack?.length) { throw new Error("audio is too short to be valid!")}
-    musicTracks.push(musicTrack)
-  } catch (err) {
-    try {
-      await sleep(4000)
-      const musicTrack = await generateMusicWithMusicgen(musicParams)
-      if (!musicTrack?.length) { throw new Error("audio is too short to be valid!")}
-      musicTracks.push(musicTrack)
-    } catch (err2) {
-      console.error(`  |- failed to generate the music (yes, we retried after a delay)`)
-    }
-  }
-
-
-  return musicTracks
-}

src/app/api/v1/create/index.ts

@@ -131,49 +131,59 @@ Output: `
     clap.segments.push(newSegment({
       track: 0,
       startTimeInMs: currentElapsedTimeInMs,
+      endTimeInMs: currentElapsedTimeInMs + defaultSegmentDurationInMs,
       assetDurationInMs: defaultSegmentDurationInMs,
       category: ClapSegmentCategory.VIDEO,
       prompt: image,
       outputType: ClapOutputType.VIDEO,
+      status: "to_generate",
     }))
 
     clap.segments.push(newSegment({
       track: 1,
       startTimeInMs: currentElapsedTimeInMs,
+      endTimeInMs: currentElapsedTimeInMs + defaultSegmentDurationInMs,
       assetDurationInMs: defaultSegmentDurationInMs,
       category: ClapSegmentCategory.STORYBOARD,
       prompt: image,
       outputType: ClapOutputType.IMAGE,
+      status: "to_generate",
     }))
 
     clap.segments.push(newSegment({
       track: 2,
       startTimeInMs: currentElapsedTimeInMs,
+      endTimeInMs: currentElapsedTimeInMs + defaultSegmentDurationInMs,
       assetDurationInMs: defaultSegmentDurationInMs,
       category: ClapSegmentCategory.INTERFACE,
       prompt: comment,
       // assetUrl: `data:text/plain;base64,${btoa(comment)}`,
       assetUrl: comment,
       outputType: ClapOutputType.TEXT,
+      status: "to_generate",
     }))
 
     clap.segments.push(newSegment({
       track: 3,
       startTimeInMs: currentElapsedTimeInMs,
+      endTimeInMs: currentElapsedTimeInMs + defaultSegmentDurationInMs,
       assetDurationInMs: defaultSegmentDurationInMs,
       category: ClapSegmentCategory.DIALOGUE,
       prompt: voice,
       outputType: ClapOutputType.AUDIO,
+      status: "to_generate",
     }))
 
     // the presence of a camera is mandatory
     clap.segments.push(newSegment({
       track: 4,
       startTimeInMs: currentElapsedTimeInMs,
+      endTimeInMs: currentElapsedTimeInMs + defaultSegmentDurationInMs,
       assetDurationInMs: defaultSegmentDurationInMs,
       category: ClapSegmentCategory.CAMERA,
       prompt: "video",
       outputType: ClapOutputType.TEXT,
+      status: "to_generate",
     }))
 
     currentElapsedTimeInMs += defaultSegmentDurationInMs
@@ -187,6 +197,21 @@ Output: `
       prompt,
       latentStory: await clapToLatentStory(clap)
     })
+    const musicPrompt = musicPrompts.at(0)
+    if (!musicPrompt) { throw new Error(`not enough music prompts`) }
+
+    console.log("musicPrompt:", musicPrompt)
+
+    clap.segments.push(newSegment({
+      track: 5,
+      startTimeInMs: 0,
+      endTimeInMs: currentElapsedTimeInMs,
+      assetDurationInMs: currentElapsedTimeInMs,
+      category: ClapSegmentCategory.MUSIC,
+      prompt: musicPrompt,
+      outputType: ClapOutputType.AUDIO,
+      status: "to_generate",
+    }))
   } catch (err) {
     console.error(`[api/v1/create] failed to generate music prompts`)
     musicPrompts.push("lofi hiphop loop")

src/app/api/v1/edit/entities/generateEntityPrompts.ts

@@ -37,7 +37,7 @@ export async function generateEntityPrompts({
 
   if (!latentStory.length) { throw new Error(`please provide a story`) }
 
-  console.log("generateEntityPrompts(): latentStory:", latentStory)
+  // console.log("generateEntityPrompts(): latentStory:", latentStory)
 
   const userPrompt = `The input story is about: ${prompt}.
 

src/app/api/v1/edit/music/cluster.ts

@@ -0,0 +1,44 @@
+import { sleep } from "@/lib/utils/sleep"
+import { ClusterMachine } from "../../types"
+
+export const nbClusterMachines = 3
+// make sure the machines are running!!
+
+// https://huggingface.co/spaces/jbilcke-hf/ai-tube-model-musicgen-1/settings
+// https://huggingface.co/spaces/jbilcke-hf/ai-tube-model-musicgen-2/settings
+// https://huggingface.co/spaces/jbilcke-hf/ai-tube-model-musicgen-3/settings
+
+// we maintain a global cluster state
+
+export const clusterMachines: ClusterMachine[] = []
+for (let i = 0; i < nbClusterMachines; i++) {
+  clusterMachines.push({
+    id: i,
+    url: `https://jbilcke-hf-ai-tube-model-musicgen-${i + 1}.hf.space`,
+    busy: false
+  })
+}
+
+export async function getClusterMachine(maxWaitTimeInMs: number = 10000): Promise<ClusterMachine> {
+  let clusterMachine: ClusterMachine | undefined = undefined
+  let timeSpentWaitingInMs = 0
+  const intervalInMs = 500
+  
+  while (true) {
+    clusterMachine = clusterMachines.find(m => !m.busy)
+    if (clusterMachine) { break }
+    if (timeSpentWaitingInMs > maxWaitTimeInMs) { break }
+    await sleep(intervalInMs)
+  }
+
+  if (!clusterMachine) {
+    throw new Error(`failed to find a cluster machine within ${maxWaitTimeInMs/10} seconds`)
+  }
+
+  // change the global state
+  clusterMachine.busy = true
+
+  return clusterMachine
+}
+
+export const token = `${process.env.MICROSERVICE_API_SECRET_TOKEN || ""}`

src/app/api/v1/edit/music/generateMusic.ts

@@ -13,7 +13,7 @@ import { getSpeechBackgroundAudioPrompt } from "@aitube/engine"
 
 import { generateSpeechWithParlerTTS } from "@/app/api/generators/speech/generateVoiceWithParlerTTS"
 import { getMediaInfo } from "@/app/api/utils/getMediaInfo"
-import { generateMusicWithMusicgen } from "@/app/api/generators/music/generateMusicWithMusicgen"
+import { generateMusicWithMusicgen } from "@/app/api/v1/edit/music/generateMusicWithMusicgen"
 
 export async function generateMusic({
   musicSegment,
@@ -32,6 +32,14 @@ export async function generateMusic({
     console.log(`generateMusic(): music segment is empty, so skipping music generation.`)
     return
   }
+
+    // for now we do something very basic
+
+  if (musicSegment.status === "completed") {
+    console.log(`generateMusic(): music segment is already generated, skipping doing it twice.`)
+    return
+  }
+  
   // for now we do something very basic
   const prompt = musicSegment.prompt
   if (!prompt) {
@@ -39,28 +47,47 @@ export async function generateMusic({
     return
   }
 
+  const durationInSec = 10 // musicSegment.assetDurationInMs / 1000
+
+  console.log(`generateMusic(): generating a music with:\n  duration: ${durationInSec} sec\n  prompt: ${prompt}`)
+
   const assetUrl = await generateMusicWithMusicgen({
     prompt,
-    durationInSec: 10,
+    durationInSec,
     hd: false,
     debug: true,
     neverThrow: true,
   })
 
+
   if (!assetUrl || assetUrl?.length < 30) {
-    console.log(`generateMusic(): generated assetUrl is empty, so music generation failed.`)
+    console.log(`generateMusic(): the generated assetUrl is empty, so music generation failed.`)
+    return
+  }
+
+  const { durationInMs, hasAudio } = await getMediaInfo(assetUrl)
+  
+  if (!hasAudio) {
+    console.log(`generateMusic(): the generated music waveform is silent, so music generation failed.`)
     return
   }
 
+  const newProperties: Partial<ClapSegment> = {
+    assetUrl,
+    assetDurationInMs: durationInMs,
+    outputGain: 1.0,
+    status: "completed"
+  }
+
   if (mode !== ClapCompletionMode.FULL) {
     console.log(`generateMusic(): adding music to a new clap file`)
     newerClap.segments.push(newSegment({
       ...musicSegment,
-      assetUrl,
+      ...newProperties,
     }))
   } else {
     console.log(`generateMusic(): overwriting the music inside the existing clap file`)
-    // this will replace the existing clap (normally)
-    musicSegment.assetUrl = assetUrl
+    // this will update the existing clap (normally)
+    Object.assign(musicSegment, newProperties)
   }
 }

src/app/api/v1/edit/music/generateMusicPrompt.ts

@@ -26,7 +26,7 @@ export async function generateMusicPrompts({
 
   if (!latentStory.length) { throw new Error(`please provide a story`) }
 
-  console.log("generateMusicPrompts(): latentStory:", latentStory)
+  // console.log("generateMusicPrompts(): latentStory:", latentStory)
 
   const userPrompt = `The input story is about: ${prompt}.
 

src/app/api/{generators → v1/edit}/music/generateMusicWithMusicgen.ts

@@ -1,7 +1,6 @@
 import { addBase64Header } from "@/lib/data/addBase64Header"
-
-import { tryApiCalls } from "../../utils/tryApiCall"
 import { MusicGenerationParams } from "./types"
+import { getClusterMachine } from "./cluster"
 
 const gradioSpaceApiUrl = `https://jbilcke-hf-ai-tube-model-musicgen.hf.space`
 const huggingFaceSpace = "jbilcke-hf/ai-tube-model-musicgen"
@@ -18,16 +17,21 @@ export async function generateMusicWithMusicgen({
   neverThrow = false,
 }: MusicGenerationParams): Promise<string> {
 
-  const actualFunction = async () => {
+  if (!prompt?.length) {
+    throw new Error(`prompt is too short!`)
+  }
+
+  const machine = await getClusterMachine()
 
-    const res = await fetch(gradioSpaceApiUrl + (gradioSpaceApiUrl.endsWith("/") ? "" : "/") + "api/predict", {
+  try {
+    const res = await fetch(machine.url + (machine.url.endsWith("/") ? "" : "/") + "api/predict", {
       method: "POST",
       headers: {
         "Content-Type": "application/json",
         // Authorization: `Bearer ${token}`,
       },
       body: JSON.stringify({
-        fn_index: 0, // <- important!
+        fn_index: 1, // <- important!
         data: [
           microserviceApiKey, // string  in 'Secret Token' Textbox component		
           "facebook/musicgen-stereo-large", // string  in 'Model' Radio component		
@@ -55,8 +59,7 @@ export async function generateMusicWithMusicgen({
     if (res.status !== 200) {
       throw new Error('Failed to fetch data')
     }
-
- 
+  
     const { data } = await res.json()
 
     // console.log("data:", data)
@@ -66,34 +69,17 @@ export async function generateMusicWithMusicgen({
       throw new Error(`Failed to fetch data (status: ${res.status})`)
     }
     // console.log("data:", data.slice(0, 50))
-  
+
     if (!data[0]) {
       throw new Error(`the returned music was empty`)
     }
-  
-    console.log("data:", data[0].slice(0, 60))
-    return addBase64Header(data[0] as string, "mp3")
-  }
-
-  try {
-    if (!prompt?.length) {
-      throw new Error(`prompt is too short!`)
-    }
 
-    const result = await tryApiCalls({
-      func: actualFunction,
-      huggingFaceSpace,
-      debug,
-      failureMessage: "failed to generate the music"
-    })
-
-    return result
+    // console.log("data:", data[0].slice(0, 60))
+    return addBase64Header(data[0] as string, "mp3")
   } catch (err) {
-    if (neverThrow) {
-      console.error(`generateVoiceWithMusicgen():`, err)
-      return ""
-    } else {
-      throw err
-    }
+    throw err
+  } finally {
+    // important: we need to free up the machine!
+    machine.busy = false
   }
 }

src/app/api/v1/edit/music/systemPrompt.ts

@@ -3,11 +3,16 @@ You are a backend API engine, designed to generate music prompt output from a st
 
 ## Prompting guidelines
 
+Be concise! don't say things like "The track should have an cheerful vibe.." instead just add "cheerful vibe".
+Avoid concepts that don't translate well to music (eg use "mysterious" instead of "investigative")
+
 To create a music prompt, you need to combine styles with moods, plus a few other things.
+
 1. Please choose a base style among those categories: "Hip Hop and Rap track", "Classic track", "Jazz track", "Electronic and dance track", "Rock'n'Roll track", "Funk track", "Dubstep track", "Afrobeats", "Orchestral track", "Pop track", "Reggae track", "Metal track", "Country track", "Blues track", "Soul track", "R'n'B track", "Disco track", "Trap track", "Ambient track", "Lofi track", "Chill track", etc.
 2. Then choose a vibe: "with an happy vibe", "with a sad vibe", "with an angry vibe", "with a chill vibe", "with a romantic vibe", "with an epic vibe", "with an energetic vibe", "with a dreamy vibe", "with a mysterious vibe", "with a relaxing vibe", "with a dark vibe", "with an upbeat vibe", "with a motivational vibe", "with an inspiring vibe", "with a nostalgic vibe", "with a groovy vibe", "with a cheerful vibe", "with a melancholic vibe", "with a hopeful vibe", etc.
 3. build up a coherent description eg.: "80s pop track with bassy drums and synth", "90s rock song with loud guitars and heavy drums", "a light and cheerly EDM track, with syncopated drums, aery pads, and strong emotions bpm: 130", "A cheerful country song with acoustic guitars", "lofi slow bpm electro chill with organic samples" etc.
 
+
 ## Example of input/output
 
 Given the following input story, provided as YAML:

src/app/api/v1/export/route.ts

@@ -23,6 +23,7 @@ export async function POST(req: NextRequest, res: NextResponse) {
   // console.log("[api/v1/export] sending blob to ai-tube-clap-exporter.hf.space")
 
   const result = await fetch(
+    // `http://localhost:7860?f=${format}`,
     `https://jbilcke-hf-ai-tube-clap-exporter.hf.space?f=${format}`,
     { method: "POST", body: await req.blob() }
   )

src/app/api/v1/render/cluster.ts

@@ -1,10 +1,6 @@
 import { sleep } from "@/lib/utils/sleep"
+import { ClusterMachine } from "../types"
 
-export type ClusterMachine = {
-  id: number
-  url: string 
-  busy: boolean
-}
 
 export const nbClusterMachines = 3
 // make sure the machines are running!!

src/app/api/v1/types.ts

@@ -1,5 +1,11 @@
 import { ClapSegmentCategory } from "@aitube/clap"
 
+export type ClusterMachine = {
+  id: number
+  url: string 
+  busy: boolean
+}
+
 export type LatentEntity = {
   name: string
   category: ClapSegmentCategory