yep, it works

src/app/api/generators/search/searchResultToMediaInfo.ts

@@ -87,6 +87,7 @@ export async function searchResultToMediaInfo(searchResult: LatentSearchResult):
     prompt: searchResult.thumbnail,
     negativePrompt: "",
     nbFrames: 1,
+    nbFPS: 1,
     nbSteps: 4,
     width: 1024,
     height: 576,

src/app/api/providers/videochain/renderWithVideoChain.ts

@@ -11,6 +11,7 @@ export async function newRender({
   prompt,
   negativePrompt,
   nbFrames,
+  nbFPS,
   nbSteps,
   width,
   height,
@@ -21,6 +22,7 @@ export async function newRender({
   prompt: string
   negativePrompt: string
   nbFrames: number
+  nbFPS: number
   nbSteps: number
   width: number
   height: number
@@ -64,6 +66,7 @@ export async function newRender({
           negativePrompt,
           // nbFrames: 8 and nbSteps: 15 --> ~10 sec generation
           nbFrames, // when nbFrames is 1, we will only generate static images
+          nbFPS,
           nbSteps, // 20 = fast, 30 = better, 50 = best
           width,
           height,
@@ -72,8 +75,11 @@ export async function newRender({
           segmentation: "disabled", // one day we will remove this param, to make it automatic
           upscalingFactor: 1, // let's disable upscaling right now
           turbo, // always use turbo mode (it's for images only anyway)
+
           // also what could be done iw that we could use the width and height to control this
           cache: "ignore", // shouldRenewCache ? "renew" : "use",
+
+          // this is the "blocking mode", which help us having to poll the server
           wait: true,
         } as Partial<RenderRequest>),
         cache: 'no-store',

src/app/api/resolvers/image/route.ts

@@ -4,6 +4,8 @@ import queryString from "query-string"
 import { newRender, getRender } from "../../providers/videochain/renderWithVideoChain"
 import { generateSeed } from "@/lib/utils/generateSeed"
 import { sleep } from "@/lib/utils/sleep"
+import { getNegativePrompt, getPositivePrompt } from "../../utils/imagePrompts"
+import { getContentType } from "@/lib/data/getContentType"
 
 export async function GET(req: NextRequest) {
 
@@ -18,12 +20,21 @@ let prompt = ""
     return NextResponse.json({ error: 'no prompt provided' }, { status: 400 });
   }
 
+  let format = "binary"
+  try {
+    const f = decodeURIComponent(query?.f?.toString() || "").trim()
+    if (f === "json" || f === "binary") { format = f }
+  } catch (err) {}
+  
   // console.log("calling await newRender")
+  prompt = getPositivePrompt(prompt)
+  const negativePrompt = getNegativePrompt()
 
   let render = await newRender({
-    prompt: `${prompt}, cinematic, photo, sublime, pro quality, sharp, crisp, beautiful, impressive, amazing, high quality, 4K`,
-    negativePrompt: "logo, text, ui, hud, interface, buttons, ad, signature, copyright, blurry, cropped, bad quality",
+    prompt,
+    negativePrompt,
     nbFrames: 1,
+    nbFPS: 1,
     nbSteps: 8,
     width: 1024,
     height: 576,
@@ -36,11 +47,23 @@ let prompt = ""
 
   while (attempts-- > 0) {
     if (render.status === "completed") {
-      return NextResponse.json(render, {
-        status: 200,
-        statusText: "OK",
-      })
-
+      if (format === "json") {
+        return NextResponse.json(render, {
+          status: 200,
+          statusText: "OK",
+        })
+       } else {
+        const contentType = getContentType(render.assetUrl)
+        const base64String = render.assetUrl.split(";base64,").pop() || ""
+        const data = Buffer.from(base64String, "base64")
+        const headers = new Headers()
+        headers.set('Content-Type', contentType)
+        return new NextResponse(data, {
+          status: 200,
+          statusText: "OK",
+          headers
+        })
+      }
     }
 
     if (render.status === "error") {
@@ -50,9 +73,9 @@ let prompt = ""
       })
     }
 
-    await sleep(1000) // minimum wait time
+    await sleep(2000) // minimum wait time
 
-    console.log("asking getRender")
+    // console.log("asking getRender")
     render = await getRender(render.renderId)
   }
 

src/app/api/resolvers/video/route.ts

@@ -4,44 +4,101 @@ import queryString from "query-string"
 import { newRender, getRender } from "../../providers/videochain/renderWithVideoChain"
 import { generateSeed } from "@/lib/utils/generateSeed"
 import { sleep } from "@/lib/utils/sleep"
+import { getNegativePrompt, getPositivePrompt } from "../../utils/imagePrompts"
 import { getContentType } from "@/lib/data/getContentType"
 
 export async function GET(req: NextRequest) {
 
-const qs = queryString.parseUrl(req.url || "")
-const query = (qs || {}).query
+  const qs = queryString.parseUrl(req.url || "")
+  const query = (qs || {}).query
 
-let prompt = ""
+
+  let prompt = ""
   try {
     prompt = decodeURIComponent(query?.p?.toString() || "").trim()
   } catch (err) {}
+
   if (!prompt) {
     return NextResponse.json({ error: 'no prompt provided' }, { status: 400 });
   }
 
+  let format = "binary"
+  try {
+    const f = decodeURIComponent(query?.f?.toString() || "").trim()
+    if (f === "json" || f === "binary") { format = f }
+  } catch (err) {}
+  
+  prompt = getPositivePrompt(prompt)
+  const negativePrompt = getNegativePrompt()
+
   // console.log("calling await newRender")
 
   let render = await newRender({
-    prompt: `${prompt}, cinematic, photo, sublime, pro quality, sharp, crisp, beautiful, impressive, amazing, high quality, 4K`,
-    negativePrompt: "logo, text, ui, hud, interface, buttons, ad, signature, copyright, blurry, cropped, bad quality",
-    nbFrames: 1,
-    nbSteps: 8,
-    width: 1024,
-    height: 576,
-    turbo: true,
+    prompt,
+    negativePrompt,
+    
+    // ATTENTION: changing those will slow things to 5-6s of loading time (compared to 3-4s)
+    // and with no real visible change
+
+    nbFrames: 20, // apparently the model can only do 2 seconds at 10, so be it
+
+    nbFPS: 10,
+
+    // possibles values are 1, 2, 4, and 8
+    // but I don't see much improvements with 8 to be honest
+    // the best one seems to be 4
+    nbSteps: 4,
+
+    // this corresponds roughly to 16:9
+    // which is the aspect ratio video used by AiTube
+
+    // unfortunately, this is too compute intensive, so we have to take half of that
+    // width: 1024,
+    // height: 576,
+
+    // IMPORTANT: since we use the tailwind class aspect-video,
+    // you cannot use use anything here!
+    // this must be aligned with whatever you choose in the frontend UI
+    //
+    // if you don't do this:
+    // - that's pixel waste, you are rendering some area for nothing (and on this project each pixel is a precious nanosecond)
+    // - clicks won't be aligned with the video, so segmentation will be off
+    // eg you cannot use 1024x512 or 512x256, because that's not aspect-video
+    // (you would have to create an entry in the tailwind config to do that properly)
+    //
+    // that's not the only constraint: you also need to respect this:
+    // `height` and `width` have to be divisible by 8 (use 32 to be safe)
+    // width: 512,
+    // height: 288,
+    width: 456, // 512,
+    height: 256, // 288,
+
+    turbo: true, // without much effect for videos as of now, as we only supports turbo (AnimateDiff Lightning)
     shouldRenewCache: true,
     seed: generateSeed()
   })
 
-  let attempts = 20
+  let attempts = 10
 
   while (attempts-- > 0) {
     if (render.status === "completed") {
-      return NextResponse.json(render, {
-        status: 200,
-        statusText: "OK",
-      })
-
+      if (format === "json") {
+        return NextResponse.json(render, {
+          status: 200,
+          statusText: "OK",
+        })
+       } else {
+        const contentType = getContentType(render.assetUrl)
+        const base64String = render.assetUrl.split(";base64,").pop() || ""
+        const data = Buffer.from(base64String, "base64")
+        const headers = new Headers()
+        headers.set('Content-Type', contentType)
+        return new NextResponse(data, {
+          status: 200,
+          statusText: "OK",
+          headers
+        })
+      }
     }
 
     if (render.status === "error") {
@@ -51,9 +108,9 @@ let prompt = ""
       })
     }
 
-    await sleep(2000) // minimum wait time
+    await sleep(1000) // minimum wait time
 
-    console.log("asking getRender")
+    // console.log("asking getRender")
     render = await getRender(render.renderId)
   }
 

src/app/api/utils/imagePrompts.ts

@@ -0,0 +1,57 @@
+// should we really have default prompts in here?
+// I think they should probably be defined at the applicative software layer (ie. in the client)
+
+export function addWordsIfNotPartOfThePrompt(prompt: string = "", words: string[] = []): string {
+  const promptWords = prompt.split(",").map(w => w.trim().toLocaleLowerCase())
+
+  return [
+    prompt,
+    // we add our keywords, unless they are already part of the prompt
+    ...words.filter(w => !promptWords.includes(w.toLocaleLowerCase()))
+  ].join(", ")
+}
+
+ export function getPositivePrompt(prompt: string = "", triggerWord = "") {
+  return addWordsIfNotPartOfThePrompt(prompt, [
+    triggerWord,
+    "sublime",
+    "pro quality",
+    "sharp",
+    "crisp",
+    "beautiful",
+    "impressive",
+    "amazing",
+    "4K",
+    "hd"
+  ])
+}
+
+export function getNegativePrompt(prompt: string = "") {
+  return addWordsIfNotPartOfThePrompt(prompt, [
+    "cropped",
+    // "underexposed", // <-- can be a desired style
+    // "overexposed", // <-- can be a desired style
+    "logo",
+    "hud",
+    "ui",
+    "censored",
+    "blurry",
+    "watermark",
+    "watermarked",
+    "copyright",
+    "extra digit",
+    "fewer digits",
+    "bad fingers",
+    "bad quality",
+    "worst quality",
+    "low quality",
+    "low resolution",
+    "glitch", // <-- keep or not? could be a desired style?
+    // "deformed",
+    // "mutated",
+    // "ugly",
+    // "disfigured",
+    // "3D render", // <-- keep or not? could be a desired style?
+    "signature"
+  ])
+}

src/components/interface/latent-engine/components/content-layer/index.tsx

@@ -26,7 +26,7 @@ export const ContentLayer = forwardRef(function ContentLayer({
       ref={ref}
       onClick={onClick}
       >
-      <div className="h-full aspect-video">
+      <div className="h-full aspect-video opacity-60">
         {children}
       </div>
     </div>

src/components/interface/latent-engine/core/engine.tsx

@@ -31,6 +31,9 @@ function LatentEngine({
   const setVideoElement = useLatentEngine(s => s.setVideoElement)
   const setSegmentationElement = useLatentEngine(s => s.setSegmentationElement)
 
+  const simulationVideoPlaybackFPS = useLatentEngine(s => s.simulationVideoPlaybackFPS)
+  const simulationRenderingTimeFPS = useLatentEngine(s => s.simulationRenderingTimeFPS)
+
   const streamType = useLatentEngine(s => s.streamType)
   const isStatic = useLatentEngine(s => s.isStatic)
   const isLive = useLatentEngine(s => s.isLive)
@@ -254,15 +257,17 @@ function LatentEngine({
 
             {/* right-side buttons */}
             <div className={cn(`
-            flex flex-none
+            flex flex-none flex-row space-x-2
             items-center justify-center
-             w-14 h-full
+             w-32 h-full
             `)}>
               {/*
 
               TODO: put a fullscreen button (and mode) here
 
              */}
+             <div className="mono text-xs text-center">playback: {Math.round(simulationVideoPlaybackFPS * 100) / 100} FPS</div>
+             <div className="mono text-xs text-center">rendering: {Math.round(simulationRenderingTimeFPS * 100) / 100} FPS</div>
             </div>
           </div>
         </div>

src/components/interface/latent-engine/core/types.ts

@@ -53,6 +53,11 @@ export type LatentEngineStore = {
 
   simulationPromise?: Promise<void>
   simulationPending: boolean // used as a "lock"
+  simulationStartedAt: number
+  simulationEndedAt: number
+  simulationDurationInMs: number
+  simulationVideoPlaybackFPS: number
+  simulationRenderingTimeFPS: number
 
   renderingIntervalId: NodeJS.Timeout | string | number | undefined
   renderingIntervalDelayInMs: number

src/components/interface/latent-engine/resolvers/image/generateImage.ts

@@ -1,15 +1,7 @@
-import { RenderedScene } from "@/types/general"
-
 export async function generateImage(prompt: string): Promise<string> {
   const requestUri = `/api/resolvers/image?p=${encodeURIComponent(prompt)}`
-
   const res = await fetch(requestUri)
-
-  const scene = (await res.json()) as RenderedScene
-
-  if (scene.error || scene.status !== "completed") {
-    throw new Error(scene.error)
-  }
-
-  return scene.assetUrl
+  const blob = await res.blob()
+  const url = URL.createObjectURL(blob)
+  return url
 }

src/components/interface/latent-engine/resolvers/video/generateVideo.ts

@@ -1,17 +1,8 @@
-import { RenderedScene } from "@/types/general"
-
 export async function generateVideo(prompt: string): Promise<string> {
   const requestUri = `/api/resolvers/video?p=${encodeURIComponent(prompt)}`
-
-  // console.log(`generateVideo: calling ${requestUri}`)
-
   const res = await fetch(requestUri)
+  const blob = await res.blob()
+  const url = URL.createObjectURL(blob)
+  return url
 
-  const scene = (await res.json()) as RenderedScene
-
-  if (scene.error || scene.status !== "completed") {
-    throw new Error(scene.error)
-  }
-
-  return scene.assetUrl
 }

src/components/interface/latent-engine/resolvers/video/index.tsx

@@ -23,6 +23,18 @@ export async function resolve(segment: ClapSegment, clap: ClapProject): Promise<
   // note: the latent-video class is not used for styling, but to grab the component
   // from JS when we need to segment etc
   return (
-    <img className="latent-video object-cover" src={assetUrl} />
+    <video
+      loop
+      className="latent-video object-cover h-full"
+      playsInline
+
+      // muted needs to be enabled for iOS to properly autoplay
+      muted
+      autoPlay
+
+      // we hide the controls
+      // controls
+      src={assetUrl}>
+    </video>
   )
 }

src/components/interface/latent-engine/store/useLatentEngine.ts

@@ -39,6 +39,11 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
 
   simulationPromise: undefined,
   simulationPending: false,
+  simulationStartedAt: performance.now(),
+  simulationEndedAt: performance.now(),
+  simulationDurationInMs: 0,
+  simulationVideoPlaybackFPS: 0,
+  simulationRenderingTimeFPS: 0,
 
   renderingIntervalId: undefined,
   renderingIntervalDelayInMs: 2000, // 2 sec
@@ -174,7 +179,7 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
       console.log("onClickOnSegmentationLayer")
     }
     // TODO use the videoElement if this is is video!
-    if (!imageElement) { return }
+    if (!videoElement) { return }
 
     const box = event.currentTarget.getBoundingClientRect()
 
@@ -183,17 +188,17 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
 
     const x = px / box.width
     const y = py / box.height
-    console.log(`onClickOnSegmentationLayer: user clicked on `, { x, y, px, py, box, imageElement })
+    console.log(`onClickOnSegmentationLayer: user clicked on `, { x, y, px, py, box, videoElement })
 
     const fn = async () => {
-      const results: InteractiveSegmenterResult = await segmentFrame(imageElement, x, y)
+      const results: InteractiveSegmenterResult = await segmentFrame(videoElement, x, y)
       get().processClickOnSegment(results)
     }
     fn()
   },
   
   togglePlayPause: (): boolean => {
-    const { isLoaded, isPlaying, renderingIntervalId } = get()
+    const { isLoaded, isPlaying, renderingIntervalId, videoElement } = get()
     if (!isLoaded) { return false }
     
     const newValue = !isPlaying
@@ -201,11 +206,25 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
     clearInterval(renderingIntervalId)
 
     if (newValue) {
+      if (videoElement) {
+        try {
+          videoElement.play()
+        } catch (err) {
+          console.error(`togglePlayPause: failed to start the video (${err})`)
+        }
+      }
       set({
         isPlaying: true,
         renderingIntervalId: setTimeout(() => { get().runRenderingLoop() }, 0)
       })
     } else {
+      if (videoElement) {
+        try {
+          videoElement.pause()
+        } catch (err) {
+          console.error(`togglePlayPause: failed to pause the video (${err})`)
+        }
+      }
       set({ isPlaying: false })
     }
 
@@ -264,6 +283,7 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
 
     set({
       simulationPending: true,
+      simulationStartedAt: performance.now(),
     })
 
     try {
@@ -284,7 +304,7 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
       }
       */
 
-      await sleep(500)
+      // await sleep(500)
 
       // note: since we are asynchronous, we need to regularly check if
       // the user asked to pause the system or no
@@ -292,6 +312,8 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
         // console.log(`runSimulationLoop: rendering video content layer..`)
         // we only grab the first one
     
+
+
         const videoLayer = (await resolveSegments(clap, "video", 1)).at(0)
 
         if (get().isPlaying) {
@@ -302,6 +324,12 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
 
           const { videoElement, imageElement, segmentationElement } = get()
 
+          if (videoElement) {
+            // yes, it is a very a dirty trick
+            // yes, it will look back
+            videoElement.defaultPlaybackRate = 0.5
+          }
+
           const canvas = drawSegmentation({
              // no mask means this will effectively clear the canvas
             canvas: segmentationElement,
@@ -309,7 +337,7 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
           })
           
 
-          console.log(`runSimulationLoop: rendered video content layer`)
+          // console.log(`runSimulationLoop: rendered video content layer`)
         }
       }
 
@@ -319,7 +347,7 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
 
     try {
       if (get().isPlaying) {
-        console.log(`runSimulationLoop: rendering UI layer..`)
+        // console.log(`runSimulationLoop: rendering UI layer..`)
 
         // note: for now we only display one element, to avoid handing a list of html elements
         const interfaceLayer = (await resolveSegments(clap, "interface", 1)).at(0)
@@ -328,15 +356,30 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
             interfaceLayer
           })
 
-          console.log(`runSimulationLoop: rendered UI layer`)
+          // console.log(`runSimulationLoop: rendered UI layer`)
         }
       }
     } catch (err) {
       console.error(`runSimulationLoop failed to render UI layer ${err}`)
     }
 
+    const simulationEndedAt = performance.now()
+    const simulationDurationInMs = simulationEndedAt - get().simulationStartedAt 
+    const simulationDurationInSec =simulationDurationInMs / 1000
+
+    // I've counted the frames manually, and we indeed have, in term of pure video playback,
+    // 10 fps divided by 2 (the 0.5 playback factor)
+    const videoFPS = 10
+    const videoDurationInSec = 1
+    const videoPlaybackSpeed = 0.5
+    const simulationVideoPlaybackFPS = videoDurationInSec * videoFPS * videoPlaybackSpeed
+    const simulationRenderingTimeFPS = (videoDurationInSec * videoFPS) / simulationDurationInSec
     set({
       simulationPending: false,
+      simulationEndedAt,
+      simulationDurationInMs,
+      simulationVideoPlaybackFPS,
+      simulationRenderingTimeFPS,
     })
   },
 

src/components/interface/track-card/index.tsx

@@ -118,6 +118,9 @@ export function TrackCard({
                 // prevent iOS from attempting to open the video in full screen, which is annoying
                 playsInline
 
+                // muted needs to be enabled for iOS to properly autoplay
+                muted
+
                 ref={ref}
                 src={media.assetUrlHd || media.assetUrl}
                 className={cn(

src/lib/clap/getMockClap.ts

@@ -4,10 +4,11 @@ import { ClapProject } from "./types"
 
 let defaultSegmentDurationInMs = 2000
 
-// const demoPrompt = "closeup of Queen angelfish, bokeh"
-// const demoPrompt = "portrait of a man tv news anchor, pierre-jean-hyves, serious, bokeh"
-// const demoPrompt = "screenshot from Call of Duty, FPS game, nextgen, videogame screenshot, unreal engine, raytracing"
-const demoPrompt = "screenshot from a flight simulator, nextgen, videogame screenshot, unreal engine, raytracing"
+let demoPrompt = "closeup of Queen angelfish, bokeh"
+// demoPrompt = "portrait of a man tv news anchor, pierre-jean-hyves, serious, bokeh"
+// demoPrompt = "screenshot from Call of Duty, FPS game, nextgen, videogame screenshot, unreal engine, raytracing"
+demoPrompt = "screenshot from a flight simulator, nextgen, videogame screenshot, unreal engine, raytracing"
+demoPrompt = "screenshot from fallout3, fallout4, wasteland, 3rd person RPG, nextgen, videogame screenshot, unreal engine, raytracing"
 
 export function getMockClap({
   prompt = demoPrompt,
@@ -66,15 +67,31 @@ export function getMockClap({
 
   currentElapsedTimeInMs += currentSegmentDurationInMs
   */
-    
-  clap.segments.push(newSegment({
-    startTimeInMs: currentElapsedTimeInMs,
-    endTimeInMs: currentSegmentDurationInMs,
-    category: "video",
-    prompt,
-    label: "demo",
-    outputType: "video",
-  }))
+  
+  // this is just for us, to quickly switch between video or image
+  const generationMode: "IMAGE" | "VIDEO" =
+  "VIDEO"
+  //"IMAGE"
+
+  if (generationMode === "VIDEO") {
+    clap.segments.push(newSegment({
+      startTimeInMs: currentElapsedTimeInMs,
+      endTimeInMs: currentSegmentDurationInMs,
+      category: "video",
+      prompt,
+      label: "demo",
+      outputType: "video",
+    }))
+  } else {
+    clap.segments.push(newSegment({
+      startTimeInMs: currentElapsedTimeInMs,
+      endTimeInMs: currentSegmentDurationInMs,
+      category: "storyboard",
+      prompt,
+      label: "demo",
+      outputType: "image",
+    }))
+  }
 
   return clap
 }

src/types/general.ts

@@ -21,12 +21,8 @@ export interface RenderRequest {
   // actionnables are names of things like "chest", "key", "tree", "chair" etc
   actionnables: string[]
 
-  // note: this is the number of frames for Zeroscope,
-  // which is currently configured to only output 3 seconds, so:
-  // nbFrames=8 -> 1 sec
-  // nbFrames=16 -> 2 sec
-  // nbFrames=24 -> 3 sec
-  nbFrames: number // min: 1, max: 24
+  nbFrames: number
+  nbFPS: number
 
   nbSteps: number // min: 1, max: 50