upgrade

.nvmrc

@@ -1 +1 @@
-v20.9.0
+v20.17.0

package.json

@@ -9,8 +9,8 @@
     "lint": "next lint"
   },
   "dependencies": {
-    "@aitube/clap": "0.1.2",
-    "@aitube/client": "0.1.2",
+    "@aitube/clap": "0.2.4",
+    "@aitube/client": "0.2.4-3",
     "@huggingface/hub": "^0.15.1",
     "@radix-ui/react-accordion": "^1.1.2",
     "@radix-ui/react-avatar": "^1.0.4",
@@ -46,7 +46,7 @@
     "jimp": "^0.22.12",
     "jose": "^5.2.4",
     "lucide-react": "^0.334.0",
-    "next": "^14.2.3",
+    "next": "^14.2.7",
     "next-themes": "^0.2.1",
     "postcss": "8.4.38",
     "qs": "^6.12.1",

src/app/main.tsx

@@ -3,7 +3,7 @@
 import React from "react"
 import { IoMdPhonePortrait } from "react-icons/io"
 import { GiRollingDices } from "react-icons/gi"
-import { ClapMediaOrientation } from "@aitube/clap"
+import { ClapImageRatio } from "@aitube/clap"
 
 import { Card, CardContent, CardHeader } from "@/components/ui/card"
 import { Button } from "@/components/ui/button"
@@ -28,7 +28,7 @@ import { useStore } from "./store"
 export function Main() {
   const { storyPromptDraft, setStoryPromptDraft, promptDraftRef } = useStoryPromptDraft()
   const  { isBusy } = useIsBusy()
-  const { orientation, toggleOrientation } = useOrientation()
+  const { imageRatio, toggleOrientation } = useOrientation()
   const { handleCreateStory, handleExtendStory } = useProcessors()
   useQueryStringParams()
 
@@ -276,7 +276,7 @@ export function Main() {
                           >
                             <div className={cn(
                               `transition-all duration-200 ease-in-out`,
-                              orientation === ClapMediaOrientation.LANDSCAPE ? `rotate-90` : `rotate-0`
+                              imageRatio === ClapImageRatio.LANDSCAPE ? `rotate-90` : `rotate-0`
                             )}>
                               <IoMdPhonePortrait size={24} />
                             </div>

src/app/server/aitube/createClap.ts

@@ -3,8 +3,8 @@
 
 import { Ratelimit } from "@upstash/ratelimit"
 import { Redis } from "@upstash/redis"
-import { ClapProject, ClapMediaOrientation } from "@aitube/clap"
-import { createClap as apiCreateClap } from "@aitube/client"
+import { ClapProject, ClapImageRatio } from "@aitube/clap"
+import { createClap as apiCreateClap } from "@aitube/api-client"
 
 import { getToken } from "./getToken"
 import { RESOLUTION_LONG, RESOLUTION_SHORT, MAX_PROMPT_LENGTH_IN_CHARS } from "../config"
@@ -14,11 +14,11 @@ const rateLimit = getRateLimit()
 
 export async function createClap({
   prompt = "",
-  orientation = ClapMediaOrientation.PORTRAIT,
+  imageRatio = ClapImageRatio.PORTRAIT,
   turbo = false,
 }: {
   prompt: string
-  orientation?: ClapMediaOrientation
+  imageRatio?: ClapImageRatio
   turbo?: boolean
 }): Promise<ClapProject> {
 
@@ -42,8 +42,8 @@ export async function createClap({
   const clap: ClapProject = await apiCreateClap({
     prompt: prompt.slice(0, MAX_PROMPT_LENGTH_IN_CHARS),
 
-    height: orientation === ClapMediaOrientation.PORTRAIT ? RESOLUTION_LONG : RESOLUTION_SHORT,
-    width: orientation === ClapMediaOrientation.PORTRAIT ? RESOLUTION_SHORT : RESOLUTION_LONG,
+    height: imageRatio === ClapImageRatio.PORTRAIT ? RESOLUTION_LONG : RESOLUTION_SHORT,
+    width: imageRatio === ClapImageRatio.PORTRAIT ? RESOLUTION_SHORT : RESOLUTION_LONG,
 
     turbo,
 

src/app/server/aitube/editClapDialogues.ts

@@ -1,7 +1,7 @@
 "use server"
 
 import { ClapProject, ClapCompletionMode } from "@aitube/clap"
-import { editClapDialogues as apiEditClapDialogues } from "@aitube/client"
+import { editClapDialogues as apiEditClapDialogues } from "@aitube/api-client"
 
 import { getToken } from "./getToken"
 import { Workaround } from "./types"

src/app/server/aitube/editClapEntities.ts

@@ -1,7 +1,7 @@
 "use server"
 
 import { ClapProject, ClapCompletionMode } from "@aitube/clap"
-import { editClapEntities as apiEditClapEntities, ClapEntityPrompt } from "@aitube/client"
+import { editClapEntities as apiEditClapEntities, ClapEntityPrompt } from "@aitube/api-client"
 
 import { getToken } from "./getToken"
 import { Workaround } from "./types"

src/app/server/aitube/editClapMusic.ts

@@ -1,7 +1,7 @@
 "use server"
 
 import { ClapProject, ClapCompletionMode } from "@aitube/clap"
-import { editClapMusic as apiEditClapMusic } from "@aitube/client"
+import { editClapMusic as apiEditClapMusic } from "@aitube/api-client"
 
 import { getToken } from "./getToken"
 import { Workaround } from "./types"

src/app/server/aitube/editClapSounds.ts

@@ -1,7 +1,7 @@
 "use server"
 
 import { ClapProject, ClapCompletionMode } from "@aitube/clap"
-import { editClapSounds as apiEditClapSounds } from "@aitube/client"
+import { editClapSounds as apiEditClapSounds } from "@aitube/api-client"
 
 import { getToken } from "./getToken"
 import { Workaround } from "./types"

src/app/server/aitube/editClapStory.ts

@@ -1,7 +1,7 @@
 "use server"
 
 import { ClapProject, ClapCompletionMode } from "@aitube/clap"
-import { editClapStory as apiEditClapStory } from "@aitube/client"
+import { editClapStory as apiEditClapStory } from "@aitube/api-client"
 
 import { getToken } from "./getToken"
 import { Workaround } from "./types"

src/app/server/aitube/editClapStoryboards.ts

@@ -1,7 +1,7 @@
 "use server"
 
 import { ClapProject, ClapCompletionMode } from "@aitube/clap"
-import { editClapStoryboards as apiEditClapStoryboards } from "@aitube/client"
+import { editClapStoryboards as apiEditClapStoryboards } from "@aitube/api-client"
 
 import { getToken } from "./getToken"
 import { Workaround } from "./types"

src/app/server/aitube/editClapVideos.ts

@@ -1,7 +1,7 @@
 "use server"
 
 import { ClapProject, ClapCompletionMode } from "@aitube/clap"
-import { editClapVideos as apiEditClapVideos } from "@aitube/client"
+import { editClapVideos as apiEditClapVideos } from "@aitube/api-client"
 
 import { getToken } from "./getToken"
 import { Workaround } from "./types"

src/app/server/aitube/exportClapToVideo.ts

@@ -1,7 +1,7 @@
 "use server"
 
 import { ClapProject } from "@aitube/clap"
-import { exportClapToVideo as apiExportClapToVideo } from "@aitube/client"
+import { exportClapToVideo as apiExportClapToVideo } from "@aitube/api-client"
 
 import { getToken } from "./getToken"
 import { removeFinalVideos } from "@/lib/utils/removeFinalVideos"

src/app/store.ts

@@ -1,10 +1,10 @@
 "use client"
 
-import { ClapProject, parseClap, serializeClap, ClapMediaOrientation, parseMediaOrientation, ClapSegmentCategory, newSegment, getClapAssetSourceType, ClapSegmentStatus } from "@aitube/clap"
+import { ClapProject, parseClap, serializeClap, ClapImageRatio, parseImageRatio, ClapSegmentCategory, newSegment, getClapAssetSourceType, ClapSegmentStatus } from "@aitube/clap"
 import { create } from "zustand"
 
 import { GenerationStage, GlobalStatus, TaskStatus } from "@/types"
-import { getVideoOrientation } from "@/lib/utils/getVideoOrientation"
+import { getImageRatio } from "@/lib/utils/getImageRatio"
 
 import { RESOLUTION_LONG, RESOLUTION_SHORT } from "./server/config"
 import { putTextInTextAreaElement } from "@/lib/utils/putTextInTextAreaElement"
@@ -15,9 +15,9 @@ export const useStore = create<{
   mainCharacterVoice: string
   storyPrompt: string
 
-  // the desired orientation for the next video
-  // but this won't impact the actual orientation of the fake device container
-  orientation: ClapMediaOrientation
+  // the desired imageRatio for the next video
+  // but this won't impact the actual imageRatio of the fake device container
+  imageRatio: ClapImageRatio
 
   status: GlobalStatus
   stage: GenerationStage
@@ -43,16 +43,16 @@ export const useStore = create<{
 
   currentVideo: string
 
-  // orientation of the currently loaded video (which can be different from `orientation`)
-  // it will impact the actual orientation of the fake device container
-  currentVideoOrientation: ClapMediaOrientation
+  // imageRatio of the currently loaded video (which can be different from `imageRatio`)
+  // it will impact the actual imageRatio of the fake device container
+  currentImageRatio: ClapImageRatio
   progress: number
   error: string
   showAuthWall: boolean
   setShowAuthWall: (showAuthWall: boolean) => void
   toggleOrientation: () => void
-  setOrientation: (orientation: ClapMediaOrientation) => void
-  setCurrentVideoOrientation: (currentVideoOrientation: ClapMediaOrientation) => void
+  setImageRatio: (imageRatio: ClapImageRatio) => void
+  setCurrentVideoOrientation: (currentImageRatio: ClapImageRatio) => void
   setMainCharacterImage: (mainCharacterImage: string) => void
   setMainCharacterVoice: (mainCharacterVoice: string) => void
   setStoryPrompt: (storyPrompt: string) => void
@@ -70,7 +70,7 @@ export const useStore = create<{
   setSkeletonClap: (fullClap?: ClapProject) => void
   setFullClap: (fullClap?: ClapProject) => void
 
-  // note: this will preload the video, and compute the orientation too
+  // note: this will preload the video, and compute the imageRatio too
   setCurrentVideo: (currentVideo: string) => Promise<void>
 
   setProgress: (progress: number) => void
@@ -86,7 +86,7 @@ export const useStore = create<{
   mainCharacterVoice: "",
   storyPromptDraft: defaultPrompt,
   storyPrompt: "",
-  orientation: ClapMediaOrientation.PORTRAIT,
+  imageRatio: ClapImageRatio.PORTRAIT,
   status: "idle",
   stage: "idle",
   statusMessage: "",
@@ -103,42 +103,42 @@ export const useStore = create<{
   skeletonClap: undefined,
   fullClap: undefined,
   currentVideo: "",
-  currentVideoOrientation: ClapMediaOrientation.PORTRAIT,
+  currentImageRatio: ClapImageRatio.PORTRAIT,
   progress: 0,
   error: "",
   showAuthWall: false,
   setShowAuthWall: (showAuthWall: boolean) => { set({ showAuthWall }) },
   toggleOrientation: () => {
-    const { orientation: previousOrientation, currentVideoOrientation, currentVideo } = get()
-    const orientation =
-      previousOrientation === ClapMediaOrientation.LANDSCAPE
-      ? ClapMediaOrientation.PORTRAIT
-      : ClapMediaOrientation.LANDSCAPE
+    const { imageRatio: previousOrientation, currentImageRatio, currentVideo } = get()
+    const imageRatio =
+      previousOrientation === ClapImageRatio.LANDSCAPE
+      ? ClapImageRatio.PORTRAIT
+      : ClapImageRatio.LANDSCAPE
 
     set({
-      orientation,
+      imageRatio,
 
-      // we normally don't touch the currentVideoOrientation since it will already contain a video
-      currentVideoOrientation:
+      // we normally don't touch the currentImageRatio since it will already contain a video
+      currentImageRatio:
         currentVideo
-        ? currentVideoOrientation
-        : orientation
+        ? currentImageRatio
+        : imageRatio
     })
   },
-  setOrientation: (orientation: ClapMediaOrientation) => {
-    const { currentVideoOrientation, currentVideo } = get()
+  setImageRatio: (imageRatio: ClapImageRatio) => {
+    const { currentImageRatio, currentVideo } = get()
 
     set({
-      orientation,
+      imageRatio,
 
-      // we normally don't touch the currentVideoOrientation since it will already contain a video
-      currentVideoOrientation:
+      // we normally don't touch the currentImageRatio since it will already contain a video
+      currentImageRatio:
         currentVideo
-        ? currentVideoOrientation
-        : orientation
+        ? currentImageRatio
+        : imageRatio
     })
   },
-  setCurrentVideoOrientation: (currentVideoOrientation: ClapMediaOrientation) => { set({ currentVideoOrientation }) },
+  setCurrentVideoOrientation: (currentImageRatio: ClapImageRatio) => { set({ currentImageRatio }) },
   setMainCharacterImage: (mainCharacterImage: string) => { set({ mainCharacterImage }) },
   setMainCharacterVoice: (mainCharacterVoice: string) => { set({ mainCharacterVoice }) },
   setStoryPrompt: (storyPrompt: string) => { set({ storyPrompt }) },
@@ -231,18 +231,18 @@ export const useStore = create<{
     set({
       currentVideo,
     })
-    const { currentVideoOrientation } = get()
-    let orientation: ClapMediaOrientation = currentVideoOrientation
+    const { currentImageRatio } = get()
+    let imageRatio: ClapImageRatio = currentImageRatio
     try {
-      let newOrientation = await getVideoOrientation(currentVideo)
+      let newOrientation = await getImageRatio(currentVideo)
       if (newOrientation) {
-        orientation = newOrientation
+        imageRatio = newOrientation
       }
     } catch (err) {
-      console.error(`failed to get the media orientation`)
+      console.error(`failed to get the media imageRatio`)
     }
     set({
-      currentVideoOrientation: orientation
+      currentImageRatio: imageRatio
     })
 
     // get().syncStatusAndStageState()
@@ -359,10 +359,10 @@ export const useStore = create<{
       storyPrompt
     )
 
-    const orientation = parseMediaOrientation(fullClap.meta.orientation)
+    const imageRatio = parseImageRatio(fullClap.meta.imageRatio)
 
-    fullClap.meta.height = orientation === ClapMediaOrientation.LANDSCAPE ? RESOLUTION_SHORT : RESOLUTION_LONG
-    fullClap.meta.width = orientation === ClapMediaOrientation.PORTRAIT ? RESOLUTION_SHORT : RESOLUTION_LONG
+    fullClap.meta.height = imageRatio === ClapImageRatio.LANDSCAPE ? RESOLUTION_SHORT : RESOLUTION_LONG
+    fullClap.meta.width = imageRatio === ClapImageRatio.PORTRAIT ? RESOLUTION_SHORT : RESOLUTION_LONG
 
     const embeddedFinalVideoAssetUrl = fullClap.segments.filter(s =>
       s.category === ClapSegmentCategory.VIDEO &&
@@ -374,9 +374,9 @@ export const useStore = create<{
     set({
       fullClap,
       storyPrompt,
-      orientation,
+      imageRatio,
       currentVideo: embeddedFinalVideoAssetUrl || get().currentVideo,
-      currentVideoOrientation: orientation,
+      currentImageRatio: imageRatio,
     })
 
     return {

src/lib/hooks/useOrientation.ts

@@ -1,21 +1,21 @@
 import { useStore } from "@/app/store"
-import { ClapMediaOrientation } from "@aitube/clap"
+import { ClapImageRatio } from "@aitube/clap"
 
 export function useOrientation() {
-  const orientation = useStore(s => s.orientation)
-  const setOrientation = useStore(s => s.setOrientation)
-  const currentVideoOrientation = useStore(s => s.currentVideoOrientation)
+  const imageRatio = useStore(s => s.imageRatio)
+  const setImageRatio = useStore(s => s.setImageRatio)
+  const currentImageRatio = useStore(s => s.currentImageRatio)
   const toggleOrientation = useStore(s => s.toggleOrientation)
-  // note: we are interested in the *current* video orientation,
-  // not the requested video orientation requested for the next video
-  const isLandscape = currentVideoOrientation === ClapMediaOrientation.LANDSCAPE
-  const isPortrait = currentVideoOrientation === ClapMediaOrientation.PORTRAIT
-  const isSquare = currentVideoOrientation === ClapMediaOrientation.SQUARE
+  // note: we are interested in the *current* video imageRatio,
+  // not the requested video imageRatio requested for the next video
+  const isLandscape = currentImageRatio === ClapImageRatio.LANDSCAPE
+  const isPortrait = currentImageRatio === ClapImageRatio.PORTRAIT
+  const isSquare = currentImageRatio === ClapImageRatio.SQUARE
 
   return {
-    orientation,
-    setOrientation,
-    currentVideoOrientation,
+    imageRatio,
+    setImageRatio,
+    currentImageRatio,
     toggleOrientation,
     isLandscape,
     isPortrait,

src/lib/hooks/useProcessors.ts

@@ -74,7 +74,7 @@ export function useProcessors() {
 
       clap = await createClap({
         prompt: promptDraftRef.current,
-        orientation: useStore.getState().orientation,
+        imageRatio: useStore.getState().imageRatio,
 
         turbo: false,
       })
@@ -264,22 +264,22 @@ export function useProcessors() {
       if (!clap) { throw new Error(`failed to edit the storyboards`) }
 
       // const fusion = 
-      console.log(`generateStoryboards(): received storyboards = `, clap)
+      console.log(`generateStoryboards(): received storyboard images = `, clap)
 
       setImageGenerationStatus("finished")
-      console.log("---------------- GENERATED STORYBOARDS ----------------")
+      console.log("---------------- GENERATED STORYBOARD IMAGES ----------------")
       clap.segments
-        .filter(s => s.category === ClapSegmentCategory.STORYBOARD)
+        .filter(s => s.category === ClapSegmentCategory.IMAGE)
         .forEach((s, i) => {
           if (s.status === ClapSegmentStatus.COMPLETED && s.assetUrl) {
             // console.log(`  [${i}] storyboard: ${s.prompt}`)
             logImage(s.assetUrl, 0.35)
           } else {
-            console.log(`  [${i}] failed to generate storyboard`)
+            console.log(`  [${i}] failed to generate storyboard images`)
           }
           // console.log(`------------------`)
         })
-      console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.STORYBOARD), [
+      console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.IMAGE), [
         'endTimeInMs',
         'prompt',
         'assetUrl'
@@ -376,14 +376,14 @@ export function useProcessors() {
   }
   
   const injectCharacters = async (clap: ClapProject): Promise<void> => {
-    const storyboards = clap.segments.filter(s => s.category === ClapSegmentCategory.STORYBOARD)
+    const storyboards = clap.segments.filter(s => s.category === ClapSegmentCategory.IMAGE)
 
     let mainCharacter = clap.entities.at(0)
 
     // let's do something basic for now: we only support 1 entity (character)
     // and we apply it to *all* the storyboards (we can always improve this later)
     if (mainCharacter) {
-      console.log(`injectCharacters(): we use the clap's main character's face on all storyboards`)
+      console.log(`injectCharacters(): we use the clap's main character's face on all storyboard images`)
       storyboards.forEach(storyboard => { storyboard.entityId = mainCharacter!.id })
       logImage(mainCharacter.imageId, 0.35)
     } else if (mainCharacterImage) {

src/lib/hooks/useProgressTimer.ts

@@ -1,10 +1,12 @@
+"use client"
+
 import { useStore } from "@/app/store"
 import { useEffect, useRef } from "react"
 import { useIsBusy } from "./useIsBusy"
 
 export function useProgressTimer() {
   const runningRef = useRef(false)
-  const timerRef = useRef<NodeJS.Timeout>()
+  const timerRef = useRef<Timer>()
 
   const progress = useStore(s => s.progress)
   const stage = useStore(s => s.stage)

src/lib/hooks/useQueryStringParams.ts

@@ -1,6 +1,6 @@
 import { useEffect } from "react"
 import { useSearchParams } from "next/navigation"
-import { ClapMediaOrientation } from "@aitube/clap"
+import { ClapImageRatio } from "@aitube/clap"
 
 import { useStore } from "@/app/store"
 
@@ -13,24 +13,24 @@ export function useQueryStringParams() {
   const { busyRef } = useIsBusy()
   const { handleCreateStory } = useProcessors()
   
-  const setOrientation = useStore(s => s.setOrientation)
+  const setImageRatio = useStore(s => s.setImageRatio)
   // this is how we support query string parameters
   // ?prompt=hello <- set a default prompt
   // ?prompt=hello&autorun=true <- automatically run the app
-  // ?orientation=landscape <- can be "landscape" or "portrait" (default)
+  // ?imageRatio=landscape <- can be "landscape" or "portrait" (default)
   const searchParams = useSearchParams()
   const queryStringPrompt = (searchParams?.get('prompt') as string) || ""
   const queryStringAutorun = (searchParams?.get('autorun') as string) || ""
-  const queryStringOrientation = (searchParams?.get('orientation') as string) || ""
+  const queryStringOrientation = (searchParams?.get('imageRatio') as string) || ""
 
   useEffect(() => {
     if (queryStringOrientation?.length > 1) {
-      console.log(`orientation = "${queryStringOrientation}"`)
-      const orientation =
+      console.log(`imageRatio = "${queryStringOrientation}"`)
+      const imageRatio =
         queryStringOrientation.trim().toLowerCase() === "landscape"
-        ? ClapMediaOrientation.LANDSCAPE
-        : ClapMediaOrientation.PORTRAIT
-      setOrientation(orientation)
+        ? ClapImageRatio.LANDSCAPE
+        : ClapImageRatio.PORTRAIT
+      setImageRatio(imageRatio)
     }
     if (queryStringPrompt?.length > 1) {
       console.log(`prompt = "${queryStringPrompt}"`)

src/lib/utils/{getVideoOrientation.ts → getImageRatio.ts}

@@ -1,19 +1,19 @@
-import { ClapMediaOrientation } from "@aitube/clap"
+import { ClapImageRatio } from "@aitube/clap"
 
 /**
- * Determine the video orientation from a video URL (data-uri or hosted)
+ * Determine the video imageRatio from a video URL (data-uri or hosted)
  * 
  * @param url 
  * @returns 
  */
-export async function getVideoOrientation(url: string): Promise<ClapMediaOrientation> {
-  return new Promise<ClapMediaOrientation>(resolve => {
+export async function getImageRatio(url: string): Promise<ClapImageRatio> {
+  return new Promise<ClapImageRatio>(resolve => {
     const video = document.createElement('video')
     video.addEventListener( "loadedmetadata", function () {
       resolve(
-        this.videoHeight < this.videoWidth ? ClapMediaOrientation.LANDSCAPE :
-        this.videoHeight > this.videoWidth ? ClapMediaOrientation.PORTRAIT :
-        ClapMediaOrientation.SQUARE
+        this.videoHeight < this.videoWidth ? ClapImageRatio.LANDSCAPE :
+        this.videoHeight > this.videoWidth ? ClapImageRatio.PORTRAIT :
+        ClapImageRatio.SQUARE
       )
     }, false)
     video.src = url