[Streaming] Pre-launch cleanup of streaming-react-app and seamless_server (#170)

seamless_server/app_pubsub.py

@@ -622,7 +622,6 @@ async def configure_stream(sid, config):
     await emit_server_state_update()
 
     return {"status": "ok", "message": "server_ready"}
-    # await sio.emit("server_ready", None, to=sid)
 
 
 # The config here is a partial config, meaning it may not contain all the config values -- only the ones the user
@@ -636,14 +635,10 @@ async def set_dynamic_config(
 ):
     session_data = await get_session_data(sid)
 
-    # client_id = None
     member = None
-    # room = None
 
     if session_data:
-        # client_id = session_data.get("client_id")
         member = session_data.get("member_object")
-        # room = session_data.get("room_object")
 
     if member:
         new_dynamic_config = {
@@ -661,8 +656,6 @@ async def set_dynamic_config(
 @sio.event
 @catch_and_log_exceptions_for_sio_event_handlers
 async def incoming_audio(sid, blob):
-    # logger.info(f"[event: incoming_audio] {sid}")
-
     session_data = await get_session_data(sid)
 
     client_id = None
@@ -721,9 +714,11 @@ async def incoming_audio(sid, blob):
         blob, dynamic_config=member.transcoder_dynamic_config
     )
 
-    # TODO: What we have below is NOT a good way to do this, because instead of sending output when its ready we're
-    # sending it when new input comes in, which means it's just sitting around. This is a temporary hack until
-    # we figure out a better architecture for awaiting transcoder output and sending it to the client.
+    # Send back any available model output
+    # NOTE: In theory it would make sense remove this from the incoming_audio handler and
+    # handle this in a dedicated thread that checks for output and sends it right away,
+    # but in practice for our limited demo use cases this approach didn't add noticeable
+    # latency, so we're keeping it simple for now.
     events = get_transcoder_output_events(member.transcoder)
     logger.debug(f"[incoming_audio] transcoder output events: {len(events)}")
 

seamless_server/src/connection_tracker.py

@@ -1,64 +0,0 @@
-from logging import Logger
-import time
-
-
-class StreamingConnectionInfo:
-    def __init__(self, address, active_connections, latest_message_received_timestamp):
-        self.address = address
-        self.active_connections = active_connections
-        self.latest_message_received_timestamp = latest_message_received_timestamp
-
-    def __repr__(self):
-        return str(self)
-
-    def __str__(self):
-        return str(
-            {
-                "address": self.address,
-                "active_connections": self.active_connections,
-                "latest_message_received_timestamp": self.latest_message_received_timestamp,
-            }
-        )
-
-
-class ConnectionTracker:
-    def __init__(self, logger: Logger):
-        self.connections = dict()
-        self.logger = logger
-
-    def __str__(self):
-        return str(self.connections)
-
-    def add_connection(self, address):
-        if address not in self.connections:
-            self.connections[address] = StreamingConnectionInfo(address, 1, time.time())
-        else:
-            self.connections[address].active_connections += 1
-            self.connections[address].latest_message_received_timestamp = time.time()
-
-    def log_recent_message(self, address):
-        if address in self.connections:
-            self.connections[address].latest_message_received_timestamp = time.time()
-        else:
-            self.logger.warning(
-                f"Address {address} not found in connection tracker when attempting to log recent message"
-            )
-
-    def remove_connection(self, address):
-        if address in self.connections:
-            self.connections[address].active_connections -= 1
-            if self.connections[address].active_connections < 0:
-                self.logger.warning(
-                    f"Address {address} has negative active connections ({self.connections[address].active_connections})"
-                )
-            if self.connections[address].active_connections <= 0:
-                del self.connections[address]
-        else:
-            self.logger.warning(
-                f"Address {address} not found in connection tracker when attempting to remove it"
-            )
-
-    def get_active_connection_count(self):
-        return sum(
-            [connection.active_connections for connection in self.connections.values()]
-        )

streaming-react-app/README.md

@@ -2,64 +2,13 @@
 
 ## Getting Started
 
-- `yarn run dev` - Run the app with a development server that supports hot module reloading
+This project uses the [Yarn Package Manager](https://yarnpkg.com/).
 
-## URL Parameters
-
-You can provide URL parameters in order to change the behavior of the app. Those are documented in [URLParams.ts](src/URLParams.ts).
-
-# Vite Information: React + TypeScript + Vite
-
-This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
-
-Currently, two official plugins are available:
-
-- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
-- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
-
-## Expanding the ESLint configuration
-
-If you are developing a production application, we recommend updating the configuration to enable type aware lint rules:
-
-- Configure the top-level `parserOptions` property like this:
-
-```js
-   parserOptions: {
-    ecmaVersion: 'latest',
-    sourceType: 'module',
-    project: ['./tsconfig.json', './tsconfig.node.json'],
-    tsconfigRootDir: __dirname,
-   },
-```
+1. `yarn` - Install project dependencies
+2. `yarn run dev` - Run the app with a development server that supports hot module reloading
 
-- Replace `plugin:@typescript-eslint/recommended` to `plugin:@typescript-eslint/recommended-type-checked` or `plugin:@typescript-eslint/strict-type-checked`
-- Optionally add `plugin:@typescript-eslint/stylistic-type-checked`
-- Install [eslint-plugin-react](https://github.com/jsx-eslint/eslint-plugin-react) and add `plugin:react/recommended` & `plugin:react/jsx-runtime` to the `extends` list
+NOTE: You will either need to provide the server URL via environment variable (you can use the `.env` file for this) or via a url param when you load the react app (example: `http://localhost:5173/?serverURL=localhost:8000`)
 
-# To Deploy to AWS
-
-1. Acquire AWS credentials (not needed if already on an EC2 instance with permissions)
-
-On your local mac use the following command.
-
-```
-eval $(corp_cloud aws get-creds 790537050551)
-```
-
-2. Deploy to AWS
-
-Build the react and copy the contents of [dist](dist) folder to s3 bucket and then invalidate the cloudfront (CDN) cache. Note step 2 has been automated using `yarn deploy_dev`
-
-To deploy to the (old) seamless-vc s3 bucket:
-
-```
-yarn build:dev_vc
-yarn deploy_dev_vc
-```
-
-To deploy to the (new) seamless-vr terraform-based s3 bucket:
+## URL Parameters
 
-```
-yarn build:dev_vr
-yarn deploy_dev_vr
-```
+You can provide URL parameters in order to change the behavior of the app. Those are documented in [URLParams.ts](src/URLParams.ts).

streaming-react-app/package.json

@@ -6,16 +6,12 @@
   "scripts": {
     "dev": "vite --host --strictPort",
     "build": "vite build",
-    "build:dev_vr": "yarn build --mode deploy_dev_vr",
-    "build:dev_vc": "yarn build --mode deploy_dev_vc",
     "preview": "vite preview",
     "clean:node-modules": "rm -rf node_modules/",
     "ts-check": "tsc --noEmit",
     "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
     "prettier-check": "cd ../ && yarn run prettier-base --check streaming-react-app",
-    "signal": "concurrently --names \"TS,LINT,PRETTIER\" -c \"bgBlack.bold,bgRed.bold,bgCyan.bold\" \"yarn run ts-check\" \"yarn run lint\" \"yarn run prettier-check\"",
-    "deploy_dev_vr": "aws s3 sync dist/ s3://dev-seamless-vr-www && aws cloudfront create-invalidation --distribution-id E38ZTD4R79FGYF --paths \"/*\"",
-    "deploy_dev_vc": "aws s3 sync dist/ s3://seamless-vc.dev.metademolab.com && aws cloudfront create-invalidation --distribution-id E29D1W2ORBP77G --paths \"/*\""
+    "signal": "concurrently --names \"TS,LINT,PRETTIER\" -c \"bgBlack.bold,bgRed.bold,bgCyan.bold\" \"yarn run ts-check\" \"yarn run lint\" \"yarn run prettier-check\""
   },
   "dependencies": {
     "@emotion/react": "11.11.1",

streaming-react-app/src/RoomConfig.tsx

@@ -74,7 +74,6 @@ export default function RoomConfig({
       const lockServerValidated: string | null =
         lockServer && roles['speaker'] ? lockServerName : null;
 
-      // TODO: Show error state if roomID isn't valid
       setJoinInProgress(true);
 
       const configObject: JoinRoomConfig = {

streaming-react-app/src/SeamlessLogo.tsx

@@ -1,33 +0,0 @@
-function SeamlessLogo() {
-  return (
-    <svg
-      width="24"
-      height="24"
-      viewBox="0 0 24 24"
-      fill="none"
-      xmlns="http://www.w3.org/2000/svg">
-      <circle cx="12" cy="12" r="12" fill="#1C2B33" />
-      <rect x="7" y="9" width="2" height="6" rx="1" fill="white" />
-      <rect
-        x="15"
-        y="9"
-        width="2"
-        height="6"
-        rx="1"
-        fill="white"
-        fill-opacity="0.5"
-      />
-      <rect
-        x="11"
-        y="6"
-        width="2"
-        height="12"
-        rx="1"
-        fill="white"
-        fill-opacity="0.5"
-      />
-    </svg>
-  );
-}
-
-export default SeamlessLogo;

streaming-react-app/src/StreamingInterface.css

@@ -47,16 +47,10 @@
 
 .translation-text-container-sra {
   background-color: #f8f8f8;
-  /* flex-grow: 1; make it expand to fill the available space */
   padding-top: 12px;
   padding-bottom: 4px;
 }
 
-.translation-text-sra {
-  /* overflow-y: scroll; */
-  /* max-height: 500px; */
-}
-
 .text-chunk-sra {
   margin-bottom: 12px;
 }

streaming-react-app/src/StreamingInterface.tsx

@@ -98,7 +98,6 @@ async function requestDisplayMediaAudioStream(
 ) {
   const stream = await navigator.mediaDevices.getDisplayMedia({
     audio: {...config, channelCount: 1},
-    // selfBrowserSurface: false, // don't allow the user to select the current tab as the source
   });
   console.debug(
     '[requestDisplayMediaAudioStream] stream created with settings:',
@@ -183,8 +182,6 @@ export default function StreamingInterface() {
   );
 
   const [receivedData, setReceivedData] = useState<Array<ServerTextData>>([]);
-  // const [translationSentencesAnimated, setTranslationSentencesAnimated] =
-  //   useState<TranslationSentences>([]);
   const [
     translationSentencesAnimatedIndex,
     setTranslationSentencesAnimatedIndex,
@@ -261,7 +258,6 @@ export default function StreamingInterface() {
         if (prevAgent?.name !== newAgent?.name) {
           setTargetLang(newAgent?.targetLangs[0] ?? null);
           setEnableExpressive(null);
-          // setOutputMode(newAgent.modalities[0]);
         }
         return newAgent;
       });
@@ -310,7 +306,6 @@ export default function StreamingInterface() {
         event: 'config',
         rate: sampleRate,
         model_name: modelName,
-        // source_language: inputLang,
         debug: serverDebugFlag,
         // synchronous processing isn't implemented on the v2 pubsub server, so hardcode this to true
         async_processing: true,
@@ -390,11 +385,12 @@ export default function StreamingInterface() {
     const mediaStreamSource = audioContext.createMediaStreamSource(stream);
     setInputStreamSource(mediaStreamSource);
     /**
-     * NOTE: This currently uses a deprecated way of processing the audio (createScriptProcessor).
+     * NOTE: This currently uses a deprecated way of processing the audio (createScriptProcessor), but
+     * which is easy and convenient for our purposes.
      *
      * Documentation for the deprecated way of doing it is here: https://developer.mozilla.org/en-US/docs/Web/API/BaseAudioContext/createScriptProcessor
      *
-     * This should be migrated to something like this SO answer: https://stackoverflow.com/a/65448287
+     * In an ideal world this would be migrated to something like this SO answer: https://stackoverflow.com/a/65448287
      */
     const scriptProcessor = audioContext.createScriptProcessor(16384, 1, 1);
     setScriptNodeProcessor(scriptProcessor);
@@ -408,7 +404,6 @@ export default function StreamingInterface() {
         console.warn('[onaudioprocess] socket is null in onaudioprocess');
         return;
       }
-      // console.debug('[onaudioprocess] event', event);
 
       if (mutedRef.current) {
         // We still want to send audio to the server when we're muted to ensure we
@@ -475,10 +470,6 @@ export default function StreamingInterface() {
       inputStreamSource.disconnect(scriptNodeProcessor);
       scriptNodeProcessor.disconnect(audioContext.destination);
 
-      // From: https://stackoverflow.com/questions/65447236/scriptnode-onaudioprocess-is-deprecated-any-alternative
-      // do we also need this??
-      // recorder?.stop();
-
       // Release the mic input so we stop showing the red recording icon in the browser
       inputStream?.getTracks().forEach((track) => track.stop());
     }
@@ -518,7 +509,6 @@ export default function StreamingInterface() {
     }
 
     const onRoomStateUpdate = (roomState: RoomState) => {
-      // console.log('[event: room_state_update]', roomState);
       setRoomState(roomState);
     };
 
@@ -632,11 +622,9 @@ export default function StreamingInterface() {
   useEffect(() => {
     const onScroll = () => {
       if (isScrolledToDocumentBottom(SCROLLED_TO_BOTTOM_THRESHOLD_PX)) {
-        // console.debug('scrolled to bottom!');
         isScrolledToBottomRef.current = true;
         return;
       }
-      // console.debug('NOT scrolled to bottom!');
       isScrolledToBottomRef.current = false;
       return;
     };
@@ -712,7 +700,6 @@ export default function StreamingInterface() {
         valueLabelDisplay="auto"
         value={gain}
         onChange={(_event: Event, newValue: number | number[]) => {
-          // console.log({event, newValue});
           if (typeof newValue === 'number') {
             const scaledGain = getGainScaledValue(newValue);
             // We want the actual gain node to use the scaled value
@@ -1002,7 +989,7 @@ export default function StreamingInterface() {
                           }
                           label="Noise Suppression (Browser)"
                         />
-                        <FormControlLabel
+                                                <FormControlLabel
                           control={
                             <Checkbox
                               checked={

streaming-react-app/src/createBufferedSpeechPlayer.ts

@@ -100,8 +100,7 @@ export default function createBufferedSpeechPlayer({
     );
 
     source.connect(gainNode);
-    // the gain node is already connected to audioContext.destination
-    // source.connect(audioContext.destination);
+
     const startTime = new Date().getTime();
     source.start();
     currentPlayingBufferSource = source;
@@ -118,9 +117,6 @@ export default function createBufferedSpeechPlayer({
       debug()?.playedAudio(startTime, endTime, buffer);
       currentPlayingBufferSource = null;
 
-      // TODO: should we disconnect source from gain node here?
-      // source.disconnect(gainNode);
-
       // We don't set isPlaying = false here because we are attempting to continue playing. It will get set to false if there are no more buffers to play
       playNextBuffer();
     };

streaming-react-app/src/generateNewRoomID.ts

@@ -54,7 +54,3 @@ export function getSequentialRoomIDForTestingGenerator(): () => string {
     return result;
   };
 }
-
-// const generator = getSequentialRoomIDForTestingGenerator();
-
-// Array.from({length: 200}, () => console.log(generator()));

streaming-react-app/src/getTranslationSentencesFromReceivedData.ts

@@ -6,7 +6,6 @@ export default function getTranslationSentencesFromReceivedData(
   return receivedData
     .reduce(
       (acc, data) => {
-        // TODO: Add special handling if the payload starts/ends with an apostrophe?
         const newAcc = [
           ...acc.slice(0, -1),
           acc[acc.length - 1].trim() + ' ' + data.payload,

streaming-react-app/src/main.tsx

@@ -1,7 +1,6 @@
 import React from 'react';
 import ReactDOM from 'react-dom/client';
 import App from './App.tsx';
-import './index.css';
 
 ReactDOM.createRoot(document.getElementById('root')!).render(
   <React.StrictMode>

streaming-react-app/vite.config.ts

@@ -9,6 +9,11 @@ import react from '@vitejs/plugin-react';
 // https://vitejs.dev/config/
 export default defineConfig(({ command }) => {
   let define = {};
+  if (command === 'serve') {
+    define = {
+      global: {},
+    };
+  }
   return {
     plugins: [react()],
     define: define,