"use client" import React, { useEffect, useRef, useTransition } from 'react' import { IoMdPhonePortrait } from 'react-icons/io' import { ClapProject, ClapMediaOrientation, ClapSegmentCategory, updateClap } from '@aitube/clap' import Image from 'next/image' import { useFilePicker } from 'use-file-picker' import { DeviceFrameset } from 'react-device-frameset' import 'react-device-frameset/styles/marvel-devices.min.css' import { Card, CardContent, CardHeader } from '@/components/ui/card' import { Button } from '@/components/ui/button' import { Toaster } from '@/components/ui/sonner' import { TextareaField } from '@/components/form/textarea-field' import { cn } from '@/lib/utils/cn' import { createClap } from './server/aitube/createClap' import { editClapEntities } from './server/aitube/editClapEntities' import { editClapDialogues } from './server/aitube/editClapDialogues' import { editClapStoryboards } from './server/aitube/editClapStoryboards' import { editClapMusic } from './server/aitube/editClapMusic' import { editClapVideos } from './server/aitube/editClapVideos' import { exportClapToVideo } from './server/aitube/exportClapToVideo' import { useStore } from './store' import HFLogo from "./hf-logo.svg" import { fileToBase64 } from '@/lib/base64/fileToBase64' import { Input } from '@/components/ui/input' import { Field } from '@/components/form/field' import { Label } from '@/components/form/label' import { getParam } from '@/lib/utils/getParam' import { GenerationStage } from '@/types' import { FileContent } from 'use-file-picker/dist/interfaces' export function Main() { const [_isPending, startTransition] = useTransition() const storyPromptDraft = useStore(s => s.storyPromptDraft) const promptDraft = useRef("") promptDraft.current = storyPromptDraft const storyPrompt = useStore(s => s.storyPrompt) const mainCharacterImage = useStore(s => s.mainCharacterImage) const mainCharacterVoice = useStore(s => s.mainCharacterVoice) const orientation = useStore(s => s.orientation) const status = useStore(s => s.status) const parseGenerationStatus = useStore(s => s.parseGenerationStatus) const storyGenerationStatus = useStore(s => s.storyGenerationStatus) const assetGenerationStatus = useStore(s => s.assetGenerationStatus) const musicGenerationStatus = useStore(s => s.musicGenerationStatus) const voiceGenerationStatus = useStore(s => s.voiceGenerationStatus) const imageGenerationStatus = useStore(s => s.imageGenerationStatus) const videoGenerationStatus = useStore(s => s.videoGenerationStatus) const finalGenerationStatus = useStore(s => s.finalGenerationStatus) const currentClap = useStore(s => s.currentClap) const currentVideo = useStore(s => s.currentVideo) const currentVideoOrientation = useStore(s => s.currentVideoOrientation) const setStoryPromptDraft = useStore(s => s.setStoryPromptDraft) const setStoryPrompt = useStore(s => s.setStoryPrompt) const setMainCharacterImage = useStore(s => s.setMainCharacterImage) const setMainCharacterVoice = useStore(s => s.setMainCharacterVoice) const setStatus = useStore(s => s.setStatus) const toggleOrientation = useStore(s => s.toggleOrientation) const error = useStore(s => s.error) const setError = useStore(s => s.setError) const setParseGenerationStatus = useStore(s => s.setParseGenerationStatus) const setStoryGenerationStatus = useStore(s => s.setStoryGenerationStatus) const setAssetGenerationStatus = useStore(s => s.setAssetGenerationStatus) const setMusicGenerationStatus = useStore(s => s.setMusicGenerationStatus) const setVoiceGenerationStatus = useStore(s => s.setVoiceGenerationStatus) const setImageGenerationStatus = useStore(s => s.setImageGenerationStatus) const setVideoGenerationStatus = useStore(s => s.setVideoGenerationStatus) const setFinalGenerationStatus = useStore(s => s.setFinalGenerationStatus) const setCurrentClap = useStore(s => s.setCurrentClap) const setCurrentVideo = useStore(s => s.setCurrentVideo) const progress = useStore(s => s.progress) const setProgress = useStore(s => s.setProgress) const saveClap = useStore(s => s.saveClap) const loadClap = useStore(s => s.loadClap) const canSeeBetaFeatures = true // getParam("beta", false) const isBusy = useStore(s => s.isBusy) const importStory = async (fileData: FileContent): Promise => { if (!fileData?.name) { throw new Error(`invalid file (missing file name)`) } const { setStatus, setProgress, setParseGenerationStatus, } = useStore.getState() let clap: ClapProject | undefined = undefined setParseGenerationStatus("generating") try { const blob = new Blob([fileData.content]) clap = await loadClap(blob, fileData.name) if (!clap) { throw new Error(`failed to load the clap file`) } setParseGenerationStatus("finished") setCurrentClap(clap) return clap } catch (err) { console.error("failed to load the Clap file:", err) setParseGenerationStatus("error") throw err } } const generateStory = async (): Promise => { let clap: ClapProject | undefined = undefined try { setProgress(0) setStatus("generating") setStoryGenerationStatus("generating") setStoryPrompt(promptDraft.current) clap = await createClap({ prompt: promptDraft.current, orientation: useStore.getState().orientation, turbo: true, }) if (!clap) { throw new Error(`failed to create the clap`) } if (clap.segments.length <= 1) { throw new Error(`failed to generate more than one segments`) } console.log(`handleSubmit(): received a clap = `, clap) setCurrentClap(clap) setStoryGenerationStatus("finished") console.log("-------- GENERATED STORY --------") console.table(clap.segments, [ // 'startTimeInMs', 'endTimeInMs', // 'track', 'category', 'prompt' ]) return clap } catch (err) { setStoryGenerationStatus("error") throw err } } const generateEntities = async (clap: ClapProject): Promise => { try { // setProgress(20) setAssetGenerationStatus("generating") clap = await editClapEntities({ clap, // generating entities requires a "smart" LLM turbo: false, // turbo: true, }) if (!clap) { throw new Error(`failed to edit the entities`) } console.log(`handleSubmit(): received a clap with entities = `, clap) setCurrentClap(clap) setAssetGenerationStatus("finished") console.log("-------- GENERATED ENTITIES --------") console.table(clap.entities, [ 'category', 'label', 'imagePrompt', 'appearance' ]) return clap } catch (err) { setAssetGenerationStatus("error") throw err } } const generateMusic = async (clap: ClapProject): Promise => { try { // setProgress(30) setMusicGenerationStatus("generating") clap = await editClapMusic({ clap, turbo: true }).then(r => r.promise) if (!clap) { throw new Error(`failed to edit the music`) } console.log(`handleSubmit(): received a clap with music = `, clap) setCurrentClap(clap) setMusicGenerationStatus("finished") console.log("-------- GENERATED MUSIC --------") console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.MUSIC), [ 'endTimeInMs', 'prompt', 'entityId', ]) return clap } catch (err) { setMusicGenerationStatus("error") throw err } } const generateStoryboards = async (clap: ClapProject): Promise => { try { // setProgress(40) setImageGenerationStatus("generating") clap = await editClapStoryboards({ clap, // the turbo is mandatory here, // since this uses a model with character consistency, // which is not the case for the non-turbo one turbo: true }).then(r => r.promise) if (!clap) { throw new Error(`failed to edit the storyboards`) } // const fusion = console.log(`handleSubmit(): received a clap with images = `, clap) setCurrentClap(clap) setImageGenerationStatus("finished") console.log("-------- GENERATED STORYBOARDS --------") console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.STORYBOARD), [ 'endTimeInMs', 'prompt', 'assetUrl' ]) return clap } catch (err) { setImageGenerationStatus("error") throw err } } const generateVideos = async (clap: ClapProject): Promise => { try { // setProgress(50) setVideoGenerationStatus("generating") clap = await editClapVideos({ clap, turbo: true }).then(r => r.promise) if (!clap) { throw new Error(`failed to edit the videos`) } console.log(`handleSubmit(): received a clap with videos = `, clap) setCurrentClap(clap) setVideoGenerationStatus("finished") console.log("-------- GENERATED VIDEOS --------") console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.VIDEO), [ 'endTimeInMs', 'prompt', 'entityId', ]) return clap } catch (err) { setVideoGenerationStatus("error") throw err } } const generateDialogues = async (clap: ClapProject): Promise => { try { // setProgress(70) setVoiceGenerationStatus("generating") clap = await editClapDialogues({ clap, turbo: true }) if (!clap) { throw new Error(`failed to edit the dialogues`) } console.log(`handleSubmit(): received a clap with dialogues = `, clap) setCurrentClap(clap) setVoiceGenerationStatus("finished") console.log("-------- GENERATED DIALOGUES --------") console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.DIALOGUE), [ 'endTimeInMs', 'prompt', 'entityId', ]) return clap } catch (err) { setVoiceGenerationStatus("error") throw err } } const generateFinalVideo = async (clap: ClapProject): Promise => { let assetUrl = "" try { // setProgress(85) setFinalGenerationStatus("generating") assetUrl = await exportClapToVideo({ clap, turbo: true }) console.log(`handleSubmit(): received a video: ${assetUrl.slice(0, 60)}...`) setFinalGenerationStatus("finished") setCurrentVideo(assetUrl) return assetUrl } catch (err) { setFinalGenerationStatus("error") throw err } } const handleSubmit = async () => { startTransition(async () => { console.log(`handleSubmit(): generating a clap using prompt = "${promptDraft.current}" `) try { let clap = await generateStory() const claps = await Promise.all([ generateMusic(clap), generateVideos(clap) ]) console.log("finished processing the 2 tasks in parallel") for (const newerClap of claps) { clap = await updateClap(clap, newerClap, { overwriteMeta: false, inlineReplace: true, }) } /* clap = await claps.reduce(async (existingClap, newerClap) => updateClap(existingClap, newerClap, { overwriteMeta: false, inlineReplace: true, }) , Promise.resolve(clap) */ // We can't have consistent characters with video (yet) // clap = await generateEntities(clap) /* if (mainCharacterImage) { console.log("handleSubmit(): User specified a main character image") // various strategies here, for instance we can assume that the first character is the main character, // or maybe a more reliable way is to count the number of occurrences. // there is a risk of misgendering, so ideally we should add some kind of UI to do this, // such as a list of characters. } */ // let's skip storyboards for now // clap = await generateStoryboards(clap) // clap = await generateVideos(clap) // clap = await generateDialogues(clap) await generateFinalVideo(clap) setStatus("finished") setError("") } catch (err) { console.error(`failed to generate: `, err) setStatus("error") setError(`${err}`) } }) } const { openFilePicker, filesContent } = useFilePicker({ accept: '.clap', readAs: "ArrayBuffer" }) const fileData = filesContent[0] useEffect(() => { const fn = async () => { if (!fileData?.name) { return } const { setStatus, setProgress } = useStore.getState() setProgress(0) setStatus("generating") try { let clap = await importStory(fileData) const claps = await Promise.all([ generateMusic(clap), generateVideos(clap) ]) // console.log("finished processing the 2 tasks in parallel") for (const newerClap of claps) { clap = await updateClap(clap, newerClap, { overwriteMeta: false, inlineReplace: true, }) } await generateFinalVideo(clap) setStatus("finished") setProgress(100) setError("") } catch (err) { console.error(`failed to import: `, err) setStatus("error") setError(`${err}`) } } fn() }, [fileData?.name]) // note: we are interested in the *current* video orientation, // not the requested video orientation requested for the next video const isLandscape = currentVideoOrientation === ClapMediaOrientation.LANDSCAPE const isPortrait = currentVideoOrientation === ClapMediaOrientation.PORTRAIT const isSquare = currentVideoOrientation === ClapMediaOrientation.SQUARE const runningRef = useRef(false) const timerRef = useRef() const timerFn = async () => { const { isBusy, progress, stage } = useStore.getState() clearTimeout(timerRef.current) if (!isBusy || stage === "idle") { return } /* console.log("progress function:", { stage, delay: progressDelayInMsPerStage[stage], progress, }) */ useStore.setState({ // progress: Math.min(maxProgressPerStage[stage], progress + 1) progress: Math.min(100, progress + 1) }) // timerRef.current = setTimeout(timerFn, progressDelayInMsPerStage[stage]) timerRef.current = setTimeout(timerFn, 800) } useEffect(() => { timerFn() clearTimeout(timerRef.current) if (!isBusy) { return } timerRef.current = setTimeout(timerFn, 0) }, [isBusy]) return (
AI
Stories Factory

Make video stories using AI ✨

{/* LEFT MENU BUTTONS + MAIN PROMPT INPUT */}
{/* TODO: To finish by Julian a bit later
) => { if (e.target.files && e.target.files.length > 0) { const file = e.target.files[0]; const newImageBase64 = await fileToBase64(file) setMainCharacterImage(newImageBase64) } }} accept="image/*" />
*/} {/* MAIN PROMPT INPUT */}
{ setStoryPromptDraft(e.target.value) promptDraft.current = e.target.value }} placeholder="Yesterday I was at my favorite pizza place and.." inputClassName=" transition-all duration-200 ease-in-out h-32 md:h-56 lg:h-64 " disabled={isBusy} value={storyPromptDraft} /> {/* END OF MAIN PROMPT INPUT */}
{/* END OF LEFT MENU BUTTONS + MAIN PROMPT INPUT */}
{/* ACTION BAR */}
{/* */}
{canSeeBetaFeatures ? :
} {canSeeBetaFeatures ? :
}
{/* ORIENTATION SWITCH */}
toggleOrientation()}>
Orientation:
{/* END OF ORIENTATION SWITCH */}
{/* END OF ACTION BAR */}
{isBusy ?

{progress}%

{isBusy ? ( storyGenerationStatus === "generating" ? "Writing story.." : parseGenerationStatus === "generating" ? "Loading the project.." : assetGenerationStatus === "generating" ? "Casting characters.." : musicGenerationStatus === "generating" ? "Producing music.." : imageGenerationStatus === "generating" ? "Creating storyboards.." : videoGenerationStatus === "generating" ? "Filming shots.." : voiceGenerationStatus === "generating" ? "Recording dialogues.." : finalGenerationStatus === "generating" ? "Assembling final cut.." : "Please wait.." ) : status === "error" ? {error || ""} :   // to prevent layout changes }

: currentVideo ?
Powered by Hugging Face Hugging Face
); }