Spaces:

kenken999
/

fastapi_django_main_live

Running on Zero

App Files Files Community

kenken999 commited on Aug 22

Commit

967ac9e

•

1 Parent(s): cfecc3f

ｆだ

Browse files

Files changed (2) hide show

staticfiles/livetest copy.html +632 -0
staticfiles/livetest.html +2 -2

staticfiles/livetest copy.html ADDED Viewed

	@@ -0,0 +1,632 @@

+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width">
+    <title>Sample Chat AI VTuber</title>
+    <link rel="stylesheet" href="aivtuber.css">
+    <script src="./aivtuber.js"></script>
+    <title>Live2Dサンプル</title>
+    <!-- Live2D -->
+  <script src="./live2dcubismcore.js"></script>
+  <script src="//cdn.jsdelivr.net/gh/dylanNew/live2d/webgl/Live2D/lib/live2d.min.js"></script>
+  <!-- PixiJS -->
+  <script src="//cdnjs.cloudflare.com/ajax/libs/pixi.js/5.1.3/pixi.min.js"></script>
+  <script src="//cdn.jsdelivr.net/npm/pixi-live2d-display/dist/index.min.js"></script>
+  <!-- Kalidokit -->
+  <script src="//cdn.jsdelivr.net/npm/kalidokit@1.1/dist/kalidokit.umd.js"></script>
+</head>
+<body>
+<video id="my-video" autoplay></video>
+<script>
+let _debug = 0;
+    'use strict';
+    function speaks(input) {
+    var text = input;//document.getElementById("text").value;
+    var speech = new SpeechSynthesisUtterance(text);
+    speech.lang = "ja-JP";  // 日本語に設定
+    window.speechSynthesis.speak(speech);
+}
+function speak(input) {
+    var text = input;//document.getElementById("text").value;
+    var speech = new SpeechSynthesisUtterance(text);
+    // 声のリストから男性の声を選択
+    var voices = window.speechSynthesis.getVoices();
+    for (var i = 0; i < voices.length; i++) {
+        if (voices[i].name.includes("Google 日本語") || voices[i].name.includes("Male")) {
+            speech.voice = voices[i];
+            break;
+        }
+    }
+    // 音声の設定
+    speech.lang = "ja-JP"; // 日本語の設定
+    speech.pitch = 1; // ピッチ（声の高さ）
+    speech.rate = 1;  // 読み上げ速度
+    speech.volume = 1; // 音量
+    // 音声の再生
+    window.speechSynthesis.speak(speech);
+}
+class MiiboAvatar {
+constructor(config) {
+    this.container = config.container;
+    this.speechToTextOptions = config.option.speech_to_text;
+    this.miiboOptions = config.option.miibo;
+    this.didOptions = config.option.d_id;
+    this.initialize();
+}
+initialize() {
+    //alert("38");
+    const RTCPeerConnection = window.RTCPeerConnection || window.webkitRTCPeerConnection || window.mozRTCPeerConnection;
+    this.videoElement = document.getElementById(this.container);
+    this.videoElement.setAttribute('playsinline', '');
+    this.playIdleVideo();
+    this.createNewStream();
+    this.rec = new webkitSpeechRecognition()
+    this.speaching = false;
+    this.processing = false;
+    this.streams = [];
+}
+async createNewStream() {
+    if(_debug==1)
+        alert("53")
+    try {
+        this.stopAllStreams();
+        this.closePC();
+        let presenter = {"source_url": this.didOptions.presenter.image_url}
+        const sessionResponse = await this.fetchWithRetries(`https://api.d-id.com/${this.didOptions.service}/streams`, {
+            method: 'POST',
+            headers: {
+                Authorization: `Basic ${this.didOptions.key}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify(presenter),
+        });
+        const { id: newStreamId, offer, ice_servers: iceServers, session_id: newSessionId } = await sessionResponse.json();
+        this.streamId = newStreamId;
+        this.sessionId = newSessionId;
+        try {
+            this.sessionClientAnswer = await this.createPeerConnection(offer, iceServers);
+        } catch (e) {
+            console.log('Error creating peer connection:', e);
+            this.stopAllStreams();
+            this.closePC();
+            return;
+        }
+        const sdpResponse = await fetch(`https://api.d-id.com/${this.didOptions.service}/streams/${this.streamId}/sdp`, {
+            method: 'POST',
+            headers: {
+                Authorization: `Basic ${this.didOptions.key}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                answer: this.sessionClientAnswer,
+                session_id: this.sessionId,
+            }),
+        });
+        // Handle sdpResponse if needed
+    } catch (error) {
+        console.log('Error creating new stream:', error);
+        // Handle error
+    }
+}
+speechRecogInit() {
+    this.audioContext = new (window.AudioContext || window.webkitAudioContext)()
+    if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
+        navigator.mediaDevices.getUserMedia({audio: true}).then((stream) => {
+            var input = this.audioContext.createMediaStreamSource(stream)
+            this.audioContext.resume()
+            this.recorder = new Recorder(input)
+        })
+    }
+}
+startRecording() {
+    this.recorder && this.recorder.record();
+}
+stopRecording() {
+    this.playLoadingVideo();
+    this.recorder && this.recorder.stop();
+    this.audioRecognize();
+    this.recorder.clear();
+}
+audioRecognize() {
+    this.recorder && this.recorder.exportWAV((blob) => {
+        let reader = new FileReader()
+        reader.onload = () => {
+            let result = new Uint8Array(reader.result)
+            let data = {
+                "config": {
+                    "encoding": "LINEAR16",
+                    "languageCode": "ja-JP",
+                    "alternativeLanguageCodes": ["en-US"],//,"cmn-CN","ko-KR"],
+                    "audio_channel_count": 2
+                },
+                "audio": {
+                    "content": this.arrayBufferToBase64(result)
+                }
+            }
+            fetch('https://speech.googleapis.com/v1/speech:recognize?key=' + this.speechToTextOptions.api_key, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json; charset=utf-8'
+                },
+                body: JSON.stringify(data)
+            }).then((response) => {
+                return response.text()
+            }).then((text) => {
+                let result_json = JSON.parse(text)
+                text = result_json.results[0].alternatives[0].transcript;
+                this.languageCode = result_json.results[0].languageCode;
+                //alert("154 ask")
+                this.ask(text)
+            })
+        }
+        reader.readAsArrayBuffer(blob)
+    })
+}
+// Chrome Only
+autoRecognizeMessage(message) {
+    //handleLiveComment("test",message)
+    //live2d
+    if(_debug==1)
+        alert("start word test")
+    this.rec.continuous = false
+    this.rec.interimResults = false
+    this.rec.lang = 'ja-JP'
+    this.processing = true
+        this.playLoadingVideo();
+        this.rec.stop()
+        //alert("172 start================================= "+message)
+        let transcript  = message
+        //alert("transcript"+transcript)
+        this.ask(transcript);
+        //for (var i = e.resultIndex; i < e.results.length; i++) {
+         //   if (!e.results[i].isFinal) continue
+   //
+     //       const { transcript } = e.results[i][0]
+       //     this.ask(transcript);
+        //}
+    //this.rec.onresult = (e) => {
+    //}
+    this.rec.onend = () => { this.autoRecognizeRestart() }
+    this.rec.start()
+}
+// Chrome Only
+autoRecognize() {
+    this.rec.continuous = false
+    this.rec.interimResults = false
+    this.rec.lang = 'ja-JP'
+    this.rec.onresult = (e) => {
+        this.processing = true
+        this.playLoadingVideo();
+        this.rec.stop()
+        if(_debug==1)
+            alert("start")
+        for (var i = e.resultIndex; i < e.results.length; i++) {
+            if (!e.results[i].isFinal) continue
+            const { transcript } = e.results[i][0]
+            this.ask(transcript);
+        }
+    }
+    this.rec.onend = () => { this.autoRecognizeRestart() }
+    this.rec.start()
+}
+autoRecognizeRestart() {
+    if (this.processing) {
+        setTimeout(() => {this.autoRecognizeRestart()}, 1000)
+    } else {
+        this.rec.start()
+    }
+}
+arrayBufferToBase64(buffer) {
+    let binary = ''
+    let bytes = new Uint8Array(buffer);
+    let len = bytes.byteLength
+    for (let i = 0; i < len; i++) {
+        binary += String.fromCharCode(bytes[i])
+    }
+    return window.btoa(binary)
+}
+ask(message) {
+    //youtubeからの質問
+    if(_debug==1)
+        alert("203 ask ========================== "+message);
+    //ask(start);
+    this.getMiiboResponse(message);
+}
+async getMiiboResponse(utterance) {
+    if(_debug==1)
+        alert("209 getMiiboResponse"+utterance)
+    const params = {
+        api_key: this.miiboOptions.api_key,
+        agent_id: this.miiboOptions.agent_id,
+        uid: this.miiboOptions.user_id,
+        stream: true,
+        utterance: utterance
+    };
+    try {
+        const res = await fetch("https://api-mebo.dev/api", {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify(params),
+        });
+        const reader = res.body.getReader();
+        const decoder = new TextDecoder();
+        let output = "";
+        let sentences = [];
+        let current_index = 0;
+        const read = async () => {
+            const { done, value } = await reader.read();
+            if (done) return;
+            let dataString = decoder.decode(value).split("\n").filter(x => x != "");
+            try {
+                let responseData = JSON.parse(dataString[dataString.length - 1]);
+                output = responseData.bestResponse.utterance.split("\n").filter(x => x.trim() != "").join("\n");
+                sentences = output.replace(/[。、\.\,\!\?！？]/,".").split(".")
+                if (this.didOptions.priority == "speed" && current_index == 0 && current_index + 1 < sentences.length) {
+                    this.startTalk(sentences[current_index++])
+                }
+            } catch(e) {
+                console.log(e);
+            }
+            return read();
+        };
+        await read();
+        reader.releaseLock();
+        this.startTalk(sentences.slice(current_index).join("。"));
+    } catch(error) {
+        console.log("Error fetching AI response: ", error);
+    }
+}
+async startTalk(input) {
+    if(_debug==1)
+        alert("start streaming id＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝"+input)
+    if (this.peerConnection?.signalingState === 'stable' || this.peerConnection?.iceConnectionState === 'connected') {
+        const gender = this.didOptions.presenter.gender;
+        let voice_id = this.didOptions.presenter.voice_id || "";
+        if (voice_id == "") {
+            switch (this.languageCode) {
+                case "en-us":
+                    voice_id = gender == "male" ? "en-US-GuyNeural" : "en-US-AriaNeural"
+                    break;
+                case "ko-kr":
+                    voice_id = gender == "male" ? "ko-KR-InJoonNeural" : "ko-KR-YuJinNeural"
+                    break;
+                case "cmn-CN":
+                    voice_id = gender == "male" ? "zh-CN-YunjianNeural" : "zh-CN-XiaohanNeural"
+                    break;
+                default:
+                    voice_id = gender == "male" ? "ja-JP-KeitaNeural" : "ja-JP-NanamiNeural"
+            }
+        }
+        alert("start tolak 330")
+        speak(input)
+        const requestOptions = {
+            method: 'POST',
+            headers: {
+                Authorization: `Basic ${this.didOptions.key}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                script: {
+                    type: "text",
+                    subtitles: false,
+                    provider: {
+                        type: "microsoft",
+                        voice_id: voice_id
+                    },
+                    ssml: false,
+                    input: input
+                },
+                config: {
+                    fluent: false,
+                    pad_audio: 0,
+                    align_driver: false,
+                    stitch: false,
+                    auto_match: false,
+                    sharpen: false,
+                    normalization_factor: 0
+                },
+                session_id: this.sessionId,
+            }),
+        };
+        if (this.didOptions.service === 'clips') {
+            requestOptions.body.background = { color: '#FFFFFF' };
+        }
+        try {
+            //const playResponse = await this.fetchWithRetries(`https://api.d-id.com/${this.didOptions.service}/streams/${this.streamId}`, requestOptions);
+            //
+            startTyping({
+        el: "#aiResponseUtterance",
+        string: input,
+        speed: 50
+    });
+            // Handle response if needed
+        } catch (error) {
+            console.error('Error starting talk:', error);
+            // Handle error
+        }
+    }
+}
+async destoryTalk(input) {
+    try {
+        await fetch(`https://api.d-id.com/${this.didOptions.service}/streams/${this.streamId}`, {
+            method: 'DELETE',
+            headers: {
+                Authorization: `Basic ${this.didOptions.key}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({ session_id: this.sessionId }),
+        });
+        await this.stopAllStreams();
+        await this.closePC();
+    } catch (error) {
+        console.error('Error destroying talk:', error);
+        // Handle error
+    }
+}
+onIceCandidate(event) {
+    if (event.candidate) {
+        const { candidate, sdpMid, sdpMLineIndex } = event.candidate;
+        fetch(`https://api.d-id.com/${this.didOptions.service}/streams/${this.streamId}/ice`, {
+            method: 'POST',
+            headers: {
+                Authorization: `Basic ${this.didOptions.key}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                candidate,
+                sdpMid,
+                sdpMLineIndex,
+                session_id: this.sessionId,
+            }),
+        }).catch((error) => {
+            console.error('Error sending ICE candidate:', error);
+            // Handle error
+        });
+    }
+}
+onIceConnectionStateChange() {
+   if (this.peerConnection.iceConnectionState === 'failed' || this.peerConnection.iceConnectionState === 'closed') {
+       this.stopAllStreams();
+       this.closePC();
+   }
+}
+onTrack(event) {
+    if (!event.track || !event.streams || event.streams.length === 0) return;
+    this.statsIntervalId = setInterval(async () => {
+        const stats = await this.peerConnection.getStats(event.track);
+        stats.forEach((report) => {
+            if (report.type === 'inbound-rtp' && report.mediaType === 'video') {
+                const videoStatusChanged = this.videoIsPlaying !== report.bytesReceived > this.lastBytesReceived;
+                if (videoStatusChanged) {
+                    this.videoIsPlaying = report.bytesReceived > this.lastBytesReceived;
+                    this.onVideoStatusChange(this.videoIsPlaying, event.streams[0]);
+                }
+                this.lastBytesReceived = report.bytesReceived;
+            }
+        });
+    }, 100);
+}
+onVideoStatusChange(videoIsPlaying, stream) {
+    let status;
+    if (videoIsPlaying) {
+        status = 'streaming';
+        const remoteStream = stream;
+        this.streams.push(remoteStream);
+        this.checkSpeaching();
+    } else {
+        status = 'empty';
+        this.speaching = false;
+        this.processing = false;
+        this.playIdleVideo();
+    }
+}
+checkSpeaching() {
+    if (this.speaching) {
+        setTimeout(() => {this.checkSpeaching()}, 20)
+    } else {
+        this.setVideoElement(this.streams.shift());
+    }
+}
+async createPeerConnection(offer, iceServers) {
+    if (!this.peerConnection) {
+        this.peerConnection = new RTCPeerConnection({ iceServers });
+        this.peerConnection.addEventListener('icecandidate', this.onIceCandidate.bind(this), true);
+        this.peerConnection.addEventListener('iceconnectionstatechange', this.onIceConnectionStateChange.bind(this), true);
+        this.peerConnection.addEventListener('track', this.onTrack.bind(this), true);
+    }
+    await this.peerConnection.setRemoteDescription(offer);
+    const sessionClientAnswer = await this.peerConnection.createAnswer();
+    await this.peerConnection.setLocalDescription(sessionClientAnswer);
+    return sessionClientAnswer;
+}
+setVideoElement(stream) {
+    if (!stream) return;
+    this.videoElement.srcObject = stream;
+    this.videoElement.loop = false;
+    // safari hotfix
+    if (this.videoElement.paused) {
+        this.videoElement
+            .play()
+            .then((_) => {})
+            .catch((e) => {});
+    }
+}
+playIdleVideo() {
+    this.videoElement.srcObject = undefined;
+    this.videoElement.src = this.didOptions.presenter.idle_movie;
+    this.videoElement.loop = true;
+}
+playLoadingVideo() {
+    this.videoElement.srcObject = undefined;
+    this.videoElement.src = this.didOptions.presenter.loading_movie;
+    this.videoElement.loop = false;
+}
+stopAllStreams() {
+    if (this.videoElement.srcObject) {
+        this.videoElement.srcObject.getTracks().forEach((track) => track.stop());
+        this.videoElement.srcObject = null;
+    }
+}
+closePC(pc = this.peerConnection) {
+    if (!pc) return;
+    pc.close();
+    pc.removeEventListener('icecandidate', this.onIceCandidate.bind(this), true);
+    pc.removeEventListener('iceconnectionstatechange', this.onIceConnectionStateChange.bind(this), true);
+    pc.removeEventListener('track', this.onTrack.bind(this), true);
+    clearInterval(this.statsIntervalId);
+    if (pc === this.peerConnection) {
+        this.peerConnection = null;
+    }
+}
+async fetchWithRetries(url, options, retries = 1) {
+    const maxRetryCount = 3;
+    const maxDelaySec = 4;
+    try {
+        return await fetch(url, options);
+    } catch (err) {
+        if (retries <= maxRetryCount) {
+            const delay = Math.min(Math.pow(2, retries) / 4 + Math.random(), maxDelaySec) * 1000;
+            await new Promise((resolve) => setTimeout(resolve, delay));
+            return this.fetchWithRetries(url, options, retries + 1);
+        } else {
+            throw new Error(`Max retries exceeded. error: ${err}`);
+        }
+        }
+    }
+}
+    const miiboAvatar = new MiiboAvatar({
+        container: "my-video",
+        option: {
+            miibo: {
+                api_key: "9206bb9d-be50-46b9-8ba2-4441fee3fe7b190cb511a0c78",
+                agent_id: "3cfb2749-a1fa-4a5b-a2c8-eb17aee77808190cb500dd1357",
+                user_id: "user",
+            },
+            d_id: {
+                key: "bWl5YXRha2VuOTk3QGdtYWlsLmNvbQ:m3WhwvxpDS6nrr4YdW9wX",
+                service: "talks",
+                priority: "cost",
+                presenter:{
+                    gender: "male",
+                    image_url: "https://kenken999-fastapi-django-main-live.hf.space/static/sugi.jpg",
+                    idle_movie: "https://kenken999-fastapi-django-main-live.hf.space/static/miiboi.mp4",
+                    loading_movie: "https://kenken999-fastapi-django-main-live.hf.space/static/miibo.mp4",
+                }
+            }
+        }
+    })
+    miiboAvatar.autoRecognize();
+    const onClickSends = () => {
+    //alert("start send ============================ ")
+    let utterance = document.querySelector("#utterances");
+    //alert(utterances.value)
+    miiboAvatar.autoRecognizeMessage(utterance.value)
+    //handleLiveComment(utterance.value, '匿名');
+    //onClickSend
+    utterances.value = "";
+}
+</script>
+<canvas id="my-live2d"></canvas>
+<script src="live2d.js"></script>
+<script src="index.js"></script>
+    <div id="vtubers">
+        <!--
+        <iframe width="auto" height="315" src="https://www.youtube.com/embed/9BSgttQDyOw?si=ax2F4oAi2h9f9eGT" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>
+        /-->
+        <!--<img id="charaImg" src="chara.png" width="auto" height="400" />/-->
+    </div>
+    <div id="aiResponse" class="aiResponseBox">
+        <p class="ai-response" id="aiResponseUtterance"></p>
+    </div>
+    <div class="bottomBox">
+        <p id="userComment"></p>
+        <button id="startLiveButton" onclick="startLive();">LIVE開始</button>
+        <div id="submit_forms">
+            <input type="text" id="utterances" />
+            <button id="sendButton" onclick="onClickSends();">送信</button>
+        </div>
+    </div>
+</body>
+</html>

staticfiles/livetest.html CHANGED Viewed

@@ -387,8 +387,8 @@ async startTalk(input) {
         }
         try {
-            //const playResponse = await this.fetchWithRetries(`https://api.d-id.com/${this.didOptions.service}/streams/${this.streamId}`, requestOptions);
-            //
             startTyping({
         el: "#aiResponseUtterance",
         string: input,

         }
         try {
+            const playResponse = await this.fetchWithRetries(`https://api.d-id.com/${this.didOptions.service}/streams/${this.streamId}`, requestOptions);
             startTyping({
         el: "#aiResponseUtterance",
         string: input,