jbilcke-hf HF staff commited on
Commit
e123fec
β€’
1 Parent(s): 6525751

working on improvements

Browse files
README.md CHANGED
@@ -1,6 +1,6 @@
1
  ---
2
  title: FacePoke
3
- emoji: πŸ’¬
4
  colorFrom: yellow
5
  colorTo: red
6
  sdk: docker
@@ -115,6 +115,14 @@ The project structure is organized as follows:
115
  - `src/`: TypeScript source files.
116
  - `public/`: Static assets and built files.
117
 
 
 
 
 
 
 
 
 
118
  ## Contributing
119
 
120
  Contributions to FacePoke are welcome! Please read our [Contributing Guidelines](CONTRIBUTING.md) for details on how to submit pull requests, report issues, or request features.
 
1
  ---
2
  title: FacePoke
3
+ emoji: πŸ™‚β€β†”οΈπŸ‘ˆ
4
  colorFrom: yellow
5
  colorTo: red
6
  sdk: docker
 
115
  - `src/`: TypeScript source files.
116
  - `public/`: Static assets and built files.
117
 
118
+ ### Increasing the framerate
119
+
120
+ I am testing various things to increase the framerate.
121
+
122
+ One project is to only transmit the modified head, instead of the whole image.
123
+
124
+ Another one is to automatically adapt to the server and network speed.
125
+
126
  ## Contributing
127
 
128
  Contributions to FacePoke are welcome! Please read our [Contributing Guidelines](CONTRIBUTING.md) for details on how to submit pull requests, report issues, or request features.
app.py CHANGED
@@ -9,6 +9,8 @@ import sys
9
  import asyncio
10
  from aiohttp import web, WSMsgType
11
  import json
 
 
12
  import uuid
13
  import logging
14
  import os
@@ -18,16 +20,18 @@ import base64
18
  import io
19
 
20
  from PIL import Image
 
 
21
  import pillow_avif
22
 
23
  # Configure logging
24
- logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
25
  logger = logging.getLogger(__name__)
26
 
27
  # Set asyncio logger to DEBUG level
28
- logging.getLogger("asyncio").setLevel(logging.DEBUG)
29
 
30
- logger.debug(f"Python version: {sys.version}")
31
 
32
  # SIGSEGV handler
33
  def SIGSEGV_signal_arises(signalNum, stack):
@@ -43,89 +47,51 @@ from engine import Engine, base64_data_uri_to_PIL_Image
43
  DATA_ROOT = os.environ.get('DATA_ROOT', '/tmp/data')
44
  MODELS_DIR = os.path.join(DATA_ROOT, "models")
45
 
46
- image_cache: Dict[str, Image.Image] = {}
 
 
 
 
 
 
 
 
 
47
 
48
  async def websocket_handler(request: web.Request) -> web.WebSocketResponse:
49
- """
50
- Handle WebSocket connections for the FacePoke application.
51
-
52
- Args:
53
- request (web.Request): The incoming request object.
54
-
55
- Returns:
56
- web.WebSocketResponse: The WebSocket response object.
57
- """
58
  ws = web.WebSocketResponse()
59
  await ws.prepare(request)
 
60
  try:
61
  #logger.info("New WebSocket connection established")
62
-
63
  while True:
64
  msg = await ws.receive()
65
 
66
- if msg.type == WSMsgType.TEXT:
67
- data = json.loads(msg.data)
68
-
69
- # let's not log user requests, they are heavy
70
- #logger.debug(f"Received message: {data}")
71
-
72
- if data['type'] == 'modify_image':
73
- uuid = data.get('uuid')
74
- if not uuid:
75
- logger.warning("Received message without UUID")
76
 
77
- await handle_modify_image(request, ws, data, uuid)
 
 
 
 
78
 
 
 
 
 
79
 
80
- elif msg.type in (WSMsgType.CLOSE, WSMsgType.ERROR):
81
- #logger.warning(f"WebSocket connection closed: {msg.type}")
82
- break
 
83
 
84
  except Exception as e:
85
  logger.error(f"Error in websocket_handler: {str(e)}")
86
  logger.exception("Full traceback:")
87
  return ws
88
 
89
- async def handle_modify_image(request: web.Request, ws: web.WebSocketResponse, msg: Dict[str, Any], uuid: str):
90
- """
91
- Handle the 'modify_image' request.
92
-
93
- Args:
94
- request (web.Request): The incoming request object.
95
- ws (web.WebSocketResponse): The WebSocket response object.
96
- msg (Dict[str, Any]): The message containing the image or image_hash and modification parameters.
97
- uuid: A unique identifier for the request.
98
- """
99
- #logger.info("Received modify_image request")
100
- try:
101
- engine = request.app['engine']
102
- image_hash = msg.get('image_hash')
103
-
104
- if image_hash:
105
- image_or_hash = image_hash
106
- else:
107
- image_data = msg['image']
108
- image_or_hash = image_data
109
-
110
- modified_image_base64 = await engine.modify_image(image_or_hash, msg['params'])
111
-
112
- await ws.send_json({
113
- "type": "modified_image",
114
- "image": modified_image_base64,
115
- "image_hash": engine.get_image_hash(image_or_hash),
116
- "success": True,
117
- "uuid": uuid # Include the UUID in the response
118
- })
119
- #logger.info("Successfully sent modified image")
120
- except Exception as e:
121
- #logger.error(f"Error in modify_image: {str(e)}")
122
- await ws.send_json({
123
- "type": "modified_image",
124
- "success": False,
125
- "error": str(e),
126
- "uuid": uuid # Include the UUID even in error responses
127
- })
128
-
129
  async def index(request: web.Request) -> web.Response:
130
  """Serve the index.html file"""
131
  content = open(os.path.join(os.path.dirname(__file__), "public", "index.html"), "r").read()
 
9
  import asyncio
10
  from aiohttp import web, WSMsgType
11
  import json
12
+ from json import JSONEncoder
13
+ import numpy as np
14
  import uuid
15
  import logging
16
  import os
 
20
  import io
21
 
22
  from PIL import Image
23
+
24
+ # by popular demand, let's add support for avif
25
  import pillow_avif
26
 
27
  # Configure logging
28
+ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
29
  logger = logging.getLogger(__name__)
30
 
31
  # Set asyncio logger to DEBUG level
32
+ #logging.getLogger("asyncio").setLevel(logging.INFO)
33
 
34
+ #logger.debug(f"Python version: {sys.version}")
35
 
36
  # SIGSEGV handler
37
  def SIGSEGV_signal_arises(signalNum, stack):
 
47
  DATA_ROOT = os.environ.get('DATA_ROOT', '/tmp/data')
48
  MODELS_DIR = os.path.join(DATA_ROOT, "models")
49
 
50
+ class NumpyEncoder(json.JSONEncoder):
51
+ def default(self, obj):
52
+ if isinstance(obj, np.integer):
53
+ return int(obj)
54
+ elif isinstance(obj, np.floating):
55
+ return float(obj)
56
+ elif isinstance(obj, np.ndarray):
57
+ return obj.tolist()
58
+ else:
59
+ return super(NumpyEncoder, self).default(obj)
60
 
61
  async def websocket_handler(request: web.Request) -> web.WebSocketResponse:
 
 
 
 
 
 
 
 
 
62
  ws = web.WebSocketResponse()
63
  await ws.prepare(request)
64
+ engine = request.app['engine']
65
  try:
66
  #logger.info("New WebSocket connection established")
 
67
  while True:
68
  msg = await ws.receive()
69
 
70
+ if msg.type in (WSMsgType.CLOSE, WSMsgType.ERROR):
71
+ #logger.warning(f"WebSocket connection closed: {msg.type}")
72
+ break
 
 
 
 
 
 
 
73
 
74
+ try:
75
+ if msg.type == WSMsgType.BINARY:
76
+ res = await engine.load_image(msg.data)
77
+ json_res = json.dumps(res, cls=NumpyEncoder)
78
+ await ws.send_str(json_res)
79
 
80
+ elif msg.type == WSMsgType.TEXT:
81
+ data = json.loads(msg.data)
82
+ webp_bytes = engine.transform_image(data.get('hash'), data.get('params'))
83
+ await ws.send_bytes(webp_bytes)
84
 
85
+ except Exception as e:
86
+ logger.error(f"Error in engine: {str(e)}")
87
+ logger.exception("Full traceback:")
88
+ await ws.send_json({"error": str(e)})
89
 
90
  except Exception as e:
91
  logger.error(f"Error in websocket_handler: {str(e)}")
92
  logger.exception("Full traceback:")
93
  return ws
94
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
95
  async def index(request: web.Request) -> web.Response:
96
  """Serve the index.html file"""
97
  content = open(os.path.join(os.path.dirname(__file__), "public", "index.html"), "r").read()
client/src/app.tsx CHANGED
@@ -4,50 +4,41 @@ import { Download } from 'lucide-react';
4
  import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert';
5
  import { truncateFileName } from './lib/utils';
6
  import { useFaceLandmarkDetection } from './hooks/useFaceLandmarkDetection';
7
- import { PoweredBy } from './components/PoweredBy';
8
  import { Spinner } from './components/Spinner';
9
  import { useFacePokeAPI } from './hooks/useFacePokeAPI';
10
  import { Layout } from './layout';
11
  import { useMainStore } from './hooks/useMainStore';
12
- import { convertImageToBase64 } from './lib/convertImageToBase64';
13
 
14
  export function App() {
15
  const error = useMainStore(s => s.error);
16
  const setError = useMainStore(s => s.setError);
17
  const imageFile = useMainStore(s => s.imageFile);
18
  const setImageFile = useMainStore(s => s.setImageFile);
19
- const originalImage = useMainStore(s => s.originalImage);
20
- const setOriginalImage = useMainStore(s => s.setOriginalImage);
 
 
 
21
  const previewImage = useMainStore(s => s.previewImage);
22
- const setPreviewImage = useMainStore(s => s.setPreviewImage);
23
- const resetImage = useMainStore(s => s.resetImage);
24
- const setOriginalImageHash = useMainStore(s => s.setOriginalImageHash);
25
 
26
  const {
27
- status,
28
- setStatus,
29
  isDebugMode,
30
  setIsDebugMode,
31
  interruptMessage,
32
  } = useFacePokeAPI()
33
 
34
- // State for face detection
35
  const {
36
- canvasRef,
37
  canvasRefCallback,
38
- mediaPipeRef,
39
- faceLandmarks,
40
  isMediaPipeReady,
41
- blendShapes,
42
-
43
- setFaceLandmarks,
44
- setBlendShapes,
45
-
46
  handleMouseDown,
47
  handleMouseUp,
48
  handleMouseMove,
49
- handleMouseEnter,
50
- handleMouseLeave,
 
51
  currentOpacity
52
  } = useFaceLandmarkDetection()
53
 
@@ -55,37 +46,10 @@ export function App() {
55
  const videoRef = useRef<HTMLDivElement>(null);
56
 
57
  // Handle file change
58
- const handleFileChange = useCallback(async (event: React.ChangeEvent<HTMLInputElement>) => {
59
  const files = event.target.files;
60
- if (files && files[0]) {
61
- setImageFile(files[0]);
62
- setStatus(`File selected: ${truncateFileName(files[0].name, 16)}`);
63
-
64
- try {
65
- const image = await convertImageToBase64(files[0]);
66
- setPreviewImage(image);
67
- setOriginalImage(image);
68
- setOriginalImageHash('');
69
- } catch (err) {
70
- console.log(`failed to convert the image: `, err);
71
- setImageFile(null);
72
- setStatus('');
73
- setPreviewImage('');
74
- setOriginalImage('');
75
- setOriginalImageHash('');
76
- setFaceLandmarks([]);
77
- setBlendShapes([]);
78
- }
79
- } else {
80
- setImageFile(null);
81
- setStatus('');
82
- setPreviewImage('');
83
- setOriginalImage('');
84
- setOriginalImageHash('');
85
- setFaceLandmarks([]);
86
- setBlendShapes([]);
87
- }
88
- }, [isMediaPipeReady, setImageFile, setPreviewImage, setOriginalImage, setOriginalImageHash, setFaceLandmarks, setBlendShapes, setStatus]);
89
 
90
  const handleDownload = useCallback(() => {
91
  if (previewImage) {
@@ -139,7 +103,7 @@ export function App() {
139
  <div className="mb-5 relative">
140
  <div className="flex flex-row items-center justify-between w-full">
141
  <div className="flex items-center space-x-2">
142
- <div className="relative">
143
  <input
144
  id="imageInput"
145
  type="file"
@@ -155,7 +119,7 @@ export function App() {
155
  } focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-slate-500 shadow-xl`}
156
  >
157
  <Spinner />
158
- {imageFile ? truncateFileName(imageFile.name, 32) : (isMediaPipeReady ? 'Choose a portrait photo (.jpg, .png, .webp)' : 'Initializing...')}
159
  </label>
160
  </div>
161
  {previewImage && (
@@ -168,15 +132,38 @@ export function App() {
168
  </button>
169
  )}
170
  </div>
171
- {previewImage && <label className="mt-4 flex items-center">
172
- <input
173
- type="checkbox"
174
- checked={isDebugMode}
175
- onChange={(e) => setIsDebugMode(e.target.checked)}
176
- className="mr-2"
177
- />
178
- Show face landmarks on hover
179
- </label>}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
180
  </div>
181
  {previewImage && (
182
  <div className="mt-5 relative shadow-2xl rounded-xl overflow-hidden">
@@ -188,11 +175,12 @@ export function App() {
188
  <canvas
189
  ref={canvasRefCallback}
190
  className="absolute top-0 left-0 w-full h-full select-none"
191
- onMouseEnter={handleMouseEnter}
192
- onMouseLeave={handleMouseLeave}
193
  onMouseDown={handleMouseDown}
194
  onMouseUp={handleMouseUp}
195
  onMouseMove={handleMouseMove}
 
 
 
196
  style={{
197
  position: 'absolute',
198
  top: 0,
@@ -207,7 +195,7 @@ export function App() {
207
  )}
208
  {canDisplayBlendShapes && displayBlendShapes}
209
  </div>
210
- <PoweredBy />
211
  </Layout>
212
  );
213
  }
 
4
  import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert';
5
  import { truncateFileName } from './lib/utils';
6
  import { useFaceLandmarkDetection } from './hooks/useFaceLandmarkDetection';
7
+ import { About } from './components/About';
8
  import { Spinner } from './components/Spinner';
9
  import { useFacePokeAPI } from './hooks/useFacePokeAPI';
10
  import { Layout } from './layout';
11
  import { useMainStore } from './hooks/useMainStore';
 
12
 
13
  export function App() {
14
  const error = useMainStore(s => s.error);
15
  const setError = useMainStore(s => s.setError);
16
  const imageFile = useMainStore(s => s.imageFile);
17
  const setImageFile = useMainStore(s => s.setImageFile);
18
+ const isGazingAtCursor = useMainStore(s => s.isGazingAtCursor);
19
+ const setIsGazingAtCursor = useMainStore(s => s.setIsGazingAtCursor);
20
+ const isFollowingCursor = useMainStore(s => s.isFollowingCursor);
21
+ const setIsFollowingCursor = useMainStore(s => s.setIsFollowingCursor);
22
+
23
  const previewImage = useMainStore(s => s.previewImage);
24
+ const status = useMainStore(s => s.status);
25
+ const blendShapes = useMainStore(s => s.blendShapes);
 
26
 
27
  const {
 
 
28
  isDebugMode,
29
  setIsDebugMode,
30
  interruptMessage,
31
  } = useFacePokeAPI()
32
 
 
33
  const {
 
34
  canvasRefCallback,
 
 
35
  isMediaPipeReady,
 
 
 
 
 
36
  handleMouseDown,
37
  handleMouseUp,
38
  handleMouseMove,
39
+ handleTouchStart,
40
+ handleTouchMove,
41
+ handleTouchEnd,
42
  currentOpacity
43
  } = useFaceLandmarkDetection()
44
 
 
46
  const videoRef = useRef<HTMLDivElement>(null);
47
 
48
  // Handle file change
49
+ const handleFileChange = useCallback((event: React.ChangeEvent<HTMLInputElement>) => {
50
  const files = event.target.files;
51
+ setImageFile(files?.[0] || undefined)
52
+ }, [setImageFile]);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
53
 
54
  const handleDownload = useCallback(() => {
55
  if (previewImage) {
 
103
  <div className="mb-5 relative">
104
  <div className="flex flex-row items-center justify-between w-full">
105
  <div className="flex items-center space-x-2">
106
+ <div className="flex items-center justify-center">
107
  <input
108
  id="imageInput"
109
  type="file"
 
119
  } focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-slate-500 shadow-xl`}
120
  >
121
  <Spinner />
122
+ {imageFile ? truncateFileName(imageFile.name, 32) : (isMediaPipeReady ? 'Choose a portrait photo' : 'Initializing...')}
123
  </label>
124
  </div>
125
  {previewImage && (
 
132
  </button>
133
  )}
134
  </div>
135
+ {previewImage && <div className="flex items-center space-x-2">
136
+ {/* experimental features, not active yet */}
137
+ {/*
138
+ <label className="mt-4 flex items-center">
139
+ <input
140
+ type="checkbox"
141
+ checked={isGazingAtCursor}
142
+ onChange={(e) => setIsGazingAtCursor(!isGazingAtCursor)}
143
+ className="mr-2"
144
+ />
145
+ Autotrack eyes
146
+ </label>
147
+ <label className="mt-4 flex items-center">
148
+ <input
149
+ type="checkbox"
150
+ checked={isFollowingCursor}
151
+ onChange={(e) => setIsFollowingCursor(!isFollowingCursor)}
152
+ className="mr-2"
153
+ />
154
+ Autotrack head
155
+ </label>
156
+ */}
157
+ <label className="mt-4 flex items-center">
158
+ <input
159
+ type="checkbox"
160
+ checked={isDebugMode}
161
+ onChange={(e) => setIsDebugMode(e.target.checked)}
162
+ className="mr-2"
163
+ />
164
+ Show face markers
165
+ </label>
166
+ </div>}
167
  </div>
168
  {previewImage && (
169
  <div className="mt-5 relative shadow-2xl rounded-xl overflow-hidden">
 
175
  <canvas
176
  ref={canvasRefCallback}
177
  className="absolute top-0 left-0 w-full h-full select-none"
 
 
178
  onMouseDown={handleMouseDown}
179
  onMouseUp={handleMouseUp}
180
  onMouseMove={handleMouseMove}
181
+ onTouchStart={handleTouchStart}
182
+ onTouchMove={handleTouchMove}
183
+ onTouchEnd={handleTouchEnd}
184
  style={{
185
  position: 'absolute',
186
  top: 0,
 
195
  )}
196
  {canDisplayBlendShapes && displayBlendShapes}
197
  </div>
198
+ <About />
199
  </Layout>
200
  );
201
  }
client/src/components/{PoweredBy.tsx β†’ About.tsx} RENAMED
@@ -1,11 +1,11 @@
1
- export function PoweredBy() {
2
  return (
3
  <div className="flex flex-row items-center justify-center font-sans mt-4 w-full">
4
- {/*<span className="text-neutral-900 text-sm"
5
  style={{ textShadow: "rgb(255 255 255 / 80%) 0px 0px 2px" }}>
6
- Best hosted on
7
- </span>*/}
8
- <span className="mr-1">
9
  <img src="/hf-logo.svg" alt="Hugging Face" className="w-5 h-5" />
10
  </span>
11
  <span className="text-neutral-900 text-sm font-semibold"
 
1
+ export function About() {
2
  return (
3
  <div className="flex flex-row items-center justify-center font-sans mt-4 w-full">
4
+ <span className="text-neutral-900 text-sm"
5
  style={{ textShadow: "rgb(255 255 255 / 80%) 0px 0px 2px" }}>
6
+ Click and drag on the image.
7
+ </span>
8
+ <span className="ml-2 mr-1">
9
  <img src="/hf-logo.svg" alt="Hugging Face" className="w-5 h-5" />
10
  </span>
11
  <span className="text-neutral-900 text-sm font-semibold"
client/src/hooks/useFaceLandmarkDetection.tsx CHANGED
@@ -6,59 +6,36 @@ import { useMainStore } from './useMainStore';
6
  import useThrottledCallback from 'beautiful-react-hooks/useThrottledCallback';
7
 
8
  import { landmarkGroups, FACEMESH_LIPS, FACEMESH_LEFT_EYE, FACEMESH_LEFT_EYEBROW, FACEMESH_RIGHT_EYE, FACEMESH_RIGHT_EYEBROW, FACEMESH_FACE_OVAL } from './landmarks';
9
-
10
- // New types for improved type safety
11
- export type LandmarkGroup = 'lips' | 'leftEye' | 'leftEyebrow' | 'rightEye' | 'rightEyebrow' | 'faceOval' | 'background';
12
- export type LandmarkCenter = { x: number; y: number; z: number };
13
- export type ClosestLandmark = { group: LandmarkGroup; distance: number; vector: { x: number; y: number; z: number } };
14
-
15
- export type MediaPipeResources = {
16
- faceLandmarker: vision.FaceLandmarker | null;
17
- drawingUtils: vision.DrawingUtils | null;
18
- };
19
 
20
  export function useFaceLandmarkDetection() {
21
- const error = useMainStore(s => s.error);
22
  const setError = useMainStore(s => s.setError);
23
- const imageFile = useMainStore(s => s.imageFile);
24
- const setImageFile = useMainStore(s => s.setImageFile);
25
- const originalImage = useMainStore(s => s.originalImage);
26
- const originalImageHash = useMainStore(s => s.originalImageHash);
27
- const setOriginalImageHash = useMainStore(s => s.setOriginalImageHash);
28
  const previewImage = useMainStore(s => s.previewImage);
29
- const setPreviewImage = useMainStore(s => s.setPreviewImage);
30
- const resetImage = useMainStore(s => s.resetImage);
31
 
32
- ;(window as any).debugJuju = useMainStore;
33
  ////////////////////////////////////////////////////////////////////////
34
- // ok so apparently I cannot vary the latency, or else there is a bug
35
- // const averageLatency = useMainStore(s => s.averageLatency);
36
- const averageLatency = 220
 
37
  ////////////////////////////////////////////////////////////////////////
38
 
39
  // State for face detection
40
- const [faceLandmarks, setFaceLandmarks] = useState<vision.NormalizedLandmark[][]>([]);
41
  const [isMediaPipeReady, setIsMediaPipeReady] = useState(false);
42
  const [isDrawingUtilsReady, setIsDrawingUtilsReady] = useState(false);
43
- const [blendShapes, setBlendShapes] = useState<vision.Classifications[]>([]);
44
 
45
  // State for mouse interaction
46
  const [dragStart, setDragStart] = useState<{ x: number; y: number } | null>(null);
47
- const [dragEnd, setDragEnd] = useState<{ x: number; y: number } | null>(null);
48
 
49
  const [isDragging, setIsDragging] = useState(false);
50
- const [isWaitingForResponse, setIsWaitingForResponse] = useState(false);
51
  const dragStartRef = useRef<{ x: number; y: number } | null>(null);
52
- const currentMousePosRef = useRef<{ x: number; y: number } | null>(null);
53
- const lastModifiedImageHashRef = useRef<string | null>(null);
54
 
55
  const [currentLandmark, setCurrentLandmark] = useState<ClosestLandmark | null>(null);
56
  const [previousLandmark, setPreviousLandmark] = useState<ClosestLandmark | null>(null);
57
  const [currentOpacity, setCurrentOpacity] = useState(0);
58
  const [previousOpacity, setPreviousOpacity] = useState(0);
59
 
60
- const [isHovering, setIsHovering] = useState(false);
61
-
62
  // Refs
63
  const canvasRef = useRef<HTMLCanvasElement>(null);
64
  const mediaPipeRef = useRef<MediaPipeResources>({
@@ -222,6 +199,9 @@ export function useFaceLandmarkDetection() {
222
 
223
  // Detect face landmarks
224
  const detectFaceLandmarks = useCallback(async (imageDataUrl: string) => {
 
 
 
225
  // console.log('Attempting to detect face landmarks...');
226
  if (!isMediaPipeReady) {
227
  console.log('MediaPipe not ready. Skipping detection.');
@@ -247,6 +227,7 @@ export function useFaceLandmarkDetection() {
247
  setFaceLandmarks(faceLandmarkerResult.faceLandmarks);
248
  setBlendShapes(faceLandmarkerResult.faceBlendshapes || []);
249
 
 
250
  if (faceLandmarkerResult.faceLandmarks && faceLandmarkerResult.faceLandmarks[0]) {
251
  computeLandmarkCenters(faceLandmarkerResult.faceLandmarks[0]);
252
  }
@@ -352,279 +333,147 @@ export function useFaceLandmarkDetection() {
352
  detectFaceLandmarks(previewImage);
353
  }, [isMediaPipeReady, isDrawingUtilsReady, previewImage])
354
 
355
-
356
-
357
- const modifyImage = useCallback(({ landmark, vector }: {
358
- landmark: ClosestLandmark
359
- vector: { x: number; y: number; z: number }
360
- }) => {
361
-
362
- const {
363
- originalImage,
364
- originalImageHash,
365
- params: previousParams,
366
- setParams,
367
- setError
368
- } = useMainStore.getState()
369
-
370
-
371
- if (!originalImage) {
372
- console.error('Image file or facePoke not available');
373
- return;
374
- }
375
-
376
- const params = {
377
- ...previousParams
378
- }
379
-
380
- const minX = -0.50;
381
- const maxX = 0.50;
382
- const minY = -0.50;
383
- const maxY = 0.50;
384
-
385
- // Function to map a value from one range to another
386
- const mapRange = (value: number, inMin: number, inMax: number, outMin: number, outMax: number): number => {
387
- return Math.min(outMax, Math.max(outMin, ((value - inMin) * (outMax - outMin)) / (inMax - inMin) + outMin));
388
- };
389
-
390
- console.log("modifyImage:", {
391
- originalImage,
392
- originalImageHash,
393
- landmark,
394
- vector,
395
- minX,
396
- maxX,
397
- minY,
398
- maxY,
399
- })
400
-
401
- // Map landmarks to ImageModificationParams
402
- switch (landmark.group) {
403
- case 'leftEye':
404
- case 'rightEye':
405
- // eyebrow (min: -20, max: 5, default: 0)
406
- const eyesMin = -20
407
- const eyesMax = 5
408
- params.eyes = mapRange(-vector.y, minX, maxX, eyesMin, eyesMax);
409
-
410
- break;
411
- case 'leftEyebrow':
412
- case 'rightEyebrow':
413
- // moving the mouse vertically for the eyebrow
414
- // should make them up/down
415
- // eyebrow (min: -10, max: 15, default: 0)
416
- const eyebrowMin = -10
417
- const eyebrowMax = 15
418
- params.eyebrow = mapRange(-vector.y, minY, maxY, eyebrowMin, eyebrowMax);
419
-
420
- break;
421
- case 'lips':
422
- // aaa (min: -30, max: 120, default: 0)
423
- //const aaaMin = -30
424
- //const aaaMax = 120
425
- //params.aaa = mapRange(vector.x, minY, maxY, aaaMin, aaaMax);
426
-
427
- // eee (min: -20, max: 15, default: 0)
428
- const eeeMin = -20
429
- const eeeMax = 15
430
- params.eee = mapRange(-vector.y, minY, maxY, eeeMin, eeeMax);
431
-
432
-
433
- // woo (min: -20, max: 15, default: 0)
434
- const wooMin = -20
435
- const wooMax = 15
436
- params.woo = mapRange(-vector.x, minX, maxX, wooMin, wooMax);
437
-
438
- break;
439
- case 'faceOval':
440
- // displacing the face horizontally by moving the mouse on the X axis
441
- // should perform a yaw rotation
442
- // rotate_roll (min: -20, max: 20, default: 0)
443
- const rollMin = -40
444
- const rollMax = 40
445
-
446
- // note: we invert the axis here
447
- params.rotate_roll = mapRange(vector.x, minX, maxX, rollMin, rollMax);
448
- break;
449
-
450
- case 'background':
451
- // displacing the face horizontally by moving the mouse on the X axis
452
- // should perform a yaw rotation
453
- // rotate_yaw (min: -20, max: 20, default: 0)
454
- const yawMin = -40
455
- const yawMax = 40
456
-
457
- // note: we invert the axis here
458
- params.rotate_yaw = mapRange(-vector.x, minX, maxX, yawMin, yawMax);
459
-
460
- // displacing the face vertically by moving the mouse on the Y axis
461
- // should perform a pitch rotation
462
- // rotate_pitch (min: -20, max: 20, default: 0)
463
- const pitchMin = -40
464
- const pitchMax = 40
465
- params.rotate_pitch = mapRange(vector.y, minY, maxY, pitchMin, pitchMax);
466
- break;
467
- default:
468
- return
469
- }
470
-
471
- for (const [key, value] of Object.entries(params)) {
472
- if (isNaN(value as any) || !isFinite(value as any)) {
473
- console.log(`${key} is NaN, aborting`)
474
- return
475
- }
476
- }
477
- console.log(`PITCH=${params.rotate_pitch || 0}, YAW=${params.rotate_yaw || 0}, ROLL=${params.rotate_roll || 0}`);
478
-
479
- setParams(params)
480
- try {
481
- // For the first request or when the image file changes, send the full image
482
- if (!lastModifiedImageHashRef.current || lastModifiedImageHashRef.current !== originalImageHash) {
483
- lastModifiedImageHashRef.current = originalImageHash;
484
- facePoke.modifyImage(originalImage, null, params);
485
- } else {
486
- // For subsequent requests, send only the hash
487
- facePoke.modifyImage(null, lastModifiedImageHashRef.current, params);
488
- }
489
- } catch (error) {
490
- // console.error('Error modifying image:', error);
491
- setError('Failed to modify image');
492
- }
493
- }, []);
494
-
495
- // this is throttled by our average latency
496
  const modifyImageWithRateLimit = useThrottledCallback((params: {
497
  landmark: ClosestLandmark
498
  vector: { x: number; y: number; z: number }
 
499
  }) => {
500
- modifyImage(params);
501
- }, [modifyImage], averageLatency);
502
-
503
- const handleMouseEnter = useCallback(() => {
504
- setIsHovering(true);
505
- }, []);
506
 
507
- const handleMouseLeave = useCallback(() => {
508
- setIsHovering(false);
509
- }, []);
510
 
511
- // Update mouse event handlers
512
- const handleMouseDown = useCallback((event: React.MouseEvent<HTMLCanvasElement>) => {
513
  if (!canvasRef.current) return;
514
 
515
  const rect = canvasRef.current.getBoundingClientRect();
516
- const x = (event.clientX - rect.left) / rect.width;
517
- const y = (event.clientY - rect.top) / rect.height;
518
 
519
- const landmark = findClosestLandmark(x, y);
520
- console.log(`Mouse down on ${landmark.group}`);
521
  setActiveLandmark(landmark);
522
- setDragStart({ x, y });
523
- dragStartRef.current = { x, y };
524
  }, [findClosestLandmark, setActiveLandmark, setDragStart]);
525
 
526
- const handleMouseMove = useCallback((event: React.MouseEvent<HTMLCanvasElement>) => {
527
  if (!canvasRef.current) return;
528
 
529
  const rect = canvasRef.current.getBoundingClientRect();
530
- const x = (event.clientX - rect.left) / rect.width;
531
- const y = (event.clientY - rect.top) / rect.height;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
532
 
533
- // only send an API request to modify the image if we are actively dragging
534
  if (dragStart && dragStartRef.current) {
535
-
536
- const landmark = findClosestLandmark(x, y, currentLandmark?.group);
537
-
538
- console.log(`Dragging mouse (was over ${currentLandmark?.group || 'nothing'}, now over ${landmark.group})`);
539
-
540
- // Compute the vector from the landmark center to the current mouse position
541
  modifyImageWithRateLimit({
542
- landmark: currentLandmark || landmark, // this will still use the initially selected landmark
543
- vector: {
544
- x: x - landmarkCenters[landmark.group].x,
545
- y: y - landmarkCenters[landmark.group].y,
546
- z: 0 // Z is 0 as mouse interaction is 2D
547
- }
548
  });
549
- setIsDragging(true);
550
  } else {
551
- const landmark = findClosestLandmark(x, y);
552
-
553
- //console.log(`Moving mouse over ${landmark.group}`);
554
- // console.log(`Simple mouse move over ${landmark.group}`);
555
-
556
- // we need to be careful here, we don't want to change the active
557
- // landmark dynamically if we are busy dragging
558
-
559
  if (!currentLandmark || (currentLandmark?.group !== landmark?.group)) {
560
- // console.log("setting activeLandmark to ", landmark);
561
  setActiveLandmark(landmark);
562
  }
563
- setIsHovering(true); // Ensure hovering state is maintained during movement
 
 
 
 
564
  }
565
- }, [currentLandmark, dragStart, setIsHovering, setActiveLandmark, setIsDragging, modifyImageWithRateLimit, landmarkCenters]);
566
 
567
- const handleMouseUp = useCallback((event: React.MouseEvent<HTMLCanvasElement>) => {
568
  if (!canvasRef.current) return;
569
 
570
  const rect = canvasRef.current.getBoundingClientRect();
571
- const x = (event.clientX - rect.left) / rect.width;
572
- const y = (event.clientY - rect.top) / rect.height;
573
 
574
- // only send an API request to modify the image if we are actively dragging
575
  if (dragStart && dragStartRef.current) {
 
576
 
577
- const landmark = findClosestLandmark(x, y, currentLandmark?.group);
578
-
579
- console.log(`Mouse up (was over ${currentLandmark?.group || 'nothing'}, now over ${landmark.group})`);
580
-
581
- // Compute the vector from the landmark center to the current mouse position
582
  modifyImageWithRateLimit({
583
- landmark: currentLandmark || landmark, // this will still use the initially selected landmark
584
  vector: {
585
- x: x - landmarkCenters[landmark.group].x,
586
- y: y - landmarkCenters[landmark.group].y,
587
- z: 0 // Z is 0 as mouse interaction is 2D
588
- }
 
589
  });
590
  }
591
 
592
  setIsDragging(false);
593
  dragStartRef.current = null;
594
  setActiveLandmark(undefined);
595
- }, [currentLandmark, isDragging, modifyImageWithRateLimit, findClosestLandmark, setActiveLandmark, landmarkCenters, modifyImageWithRateLimit, setIsDragging]);
596
 
597
- useEffect(() => {
598
- facePoke.setOnModifiedImage((image: string, image_hash: string) => {
599
- if (image) {
600
- setPreviewImage(image);
601
- }
602
- setOriginalImageHash(image_hash);
603
- lastModifiedImageHashRef.current = image_hash;
604
- });
605
- }, [setPreviewImage, setOriginalImageHash]);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
606
 
607
  return {
608
  canvasRef,
609
  canvasRefCallback,
610
  mediaPipeRef,
611
- faceLandmarks,
612
  isMediaPipeReady,
613
  isDrawingUtilsReady,
614
- blendShapes,
615
-
616
- //dragStart,
617
- //setDragStart,
618
- //dragEnd,
619
- //setDragEnd,
620
- setFaceLandmarks,
621
- setBlendShapes,
622
 
623
  handleMouseDown,
624
  handleMouseUp,
625
  handleMouseMove,
626
- handleMouseEnter,
627
- handleMouseLeave,
 
628
 
629
  currentLandmark,
630
  currentOpacity,
 
6
  import useThrottledCallback from 'beautiful-react-hooks/useThrottledCallback';
7
 
8
  import { landmarkGroups, FACEMESH_LIPS, FACEMESH_LEFT_EYE, FACEMESH_LEFT_EYEBROW, FACEMESH_RIGHT_EYE, FACEMESH_RIGHT_EYEBROW, FACEMESH_FACE_OVAL } from './landmarks';
9
+ import type { ActionMode, ClosestLandmark, LandmarkCenter, LandmarkGroup, MediaPipeResources } from '@/types';
 
 
 
 
 
 
 
 
 
10
 
11
  export function useFaceLandmarkDetection() {
 
12
  const setError = useMainStore(s => s.setError);
 
 
 
 
 
13
  const previewImage = useMainStore(s => s.previewImage);
14
+ const handleServerResponse = useMainStore(s => s.handleServerResponse);
15
+ const faceLandmarks = useMainStore(s => s.faceLandmarks);
16
 
 
17
  ////////////////////////////////////////////////////////////////////////
18
+ // if we only send the face/square then we can use 138ms
19
+ // unfortunately it doesn't work well yet
20
+ // const throttleInMs = 138ms
21
+ const throttleInMs = 180
22
  ////////////////////////////////////////////////////////////////////////
23
 
24
  // State for face detection
 
25
  const [isMediaPipeReady, setIsMediaPipeReady] = useState(false);
26
  const [isDrawingUtilsReady, setIsDrawingUtilsReady] = useState(false);
 
27
 
28
  // State for mouse interaction
29
  const [dragStart, setDragStart] = useState<{ x: number; y: number } | null>(null);
 
30
 
31
  const [isDragging, setIsDragging] = useState(false);
 
32
  const dragStartRef = useRef<{ x: number; y: number } | null>(null);
 
 
33
 
34
  const [currentLandmark, setCurrentLandmark] = useState<ClosestLandmark | null>(null);
35
  const [previousLandmark, setPreviousLandmark] = useState<ClosestLandmark | null>(null);
36
  const [currentOpacity, setCurrentOpacity] = useState(0);
37
  const [previousOpacity, setPreviousOpacity] = useState(0);
38
 
 
 
39
  // Refs
40
  const canvasRef = useRef<HTMLCanvasElement>(null);
41
  const mediaPipeRef = useRef<MediaPipeResources>({
 
199
 
200
  // Detect face landmarks
201
  const detectFaceLandmarks = useCallback(async (imageDataUrl: string) => {
202
+ const { setFaceLandmarks,setBlendShapes } = useMainStore.getState();
203
+
204
+
205
  // console.log('Attempting to detect face landmarks...');
206
  if (!isMediaPipeReady) {
207
  console.log('MediaPipe not ready. Skipping detection.');
 
227
  setFaceLandmarks(faceLandmarkerResult.faceLandmarks);
228
  setBlendShapes(faceLandmarkerResult.faceBlendshapes || []);
229
 
230
+
231
  if (faceLandmarkerResult.faceLandmarks && faceLandmarkerResult.faceLandmarks[0]) {
232
  computeLandmarkCenters(faceLandmarkerResult.faceLandmarks[0]);
233
  }
 
333
  detectFaceLandmarks(previewImage);
334
  }, [isMediaPipeReady, isDrawingUtilsReady, previewImage])
335
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
336
  const modifyImageWithRateLimit = useThrottledCallback((params: {
337
  landmark: ClosestLandmark
338
  vector: { x: number; y: number; z: number }
339
+ mode: ActionMode
340
  }) => {
341
+ useMainStore.getState().modifyImage(params);
342
+ }, [], throttleInMs);
 
 
 
 
343
 
344
+ useEffect(() => {
345
+ facePoke.setOnServerResponse(handleServerResponse);
346
+ }, [handleServerResponse]);
347
 
348
+ const handleStart = useCallback((x: number, y: number, mode: ActionMode) => {
 
349
  if (!canvasRef.current) return;
350
 
351
  const rect = canvasRef.current.getBoundingClientRect();
352
+ const normalizedX = (x - rect.left) / rect.width;
353
+ const normalizedY = (y - rect.top) / rect.height;
354
 
355
+ const landmark = findClosestLandmark(normalizedX, normalizedY);
356
+ // console.log(`Interaction start on ${landmark.group}`);
357
  setActiveLandmark(landmark);
358
+ setDragStart({ x: normalizedX, y: normalizedY });
359
+ dragStartRef.current = { x: normalizedX, y: normalizedY };
360
  }, [findClosestLandmark, setActiveLandmark, setDragStart]);
361
 
362
+ const handleMove = useCallback((x: number, y: number, mode: ActionMode) => {
363
  if (!canvasRef.current) return;
364
 
365
  const rect = canvasRef.current.getBoundingClientRect();
366
+ const normalizedX = (x - rect.left) / rect.width;
367
+ const normalizedY = (y - rect.top) / rect.height;
368
+
369
+ const landmark = findClosestLandmark(
370
+ normalizedX,
371
+ normalizedY,
372
+ dragStart && dragStartRef.current ? currentLandmark?.group : undefined
373
+ );
374
+
375
+ const landmarkData = landmarkCenters[landmark?.group]
376
+ const vector = landmarkData ? {
377
+ x: normalizedX - landmarkData.x,
378
+ y: normalizedY - landmarkData.y,
379
+ z: 0
380
+ } : {
381
+ x: 0.5,
382
+ y: 0.5,
383
+ z: 0
384
+ }
385
 
 
386
  if (dragStart && dragStartRef.current) {
387
+ setIsDragging(true);
 
 
 
 
 
388
  modifyImageWithRateLimit({
389
+ landmark: currentLandmark || landmark,
390
+ vector,
391
+ mode
 
 
 
392
  });
 
393
  } else {
 
 
 
 
 
 
 
 
394
  if (!currentLandmark || (currentLandmark?.group !== landmark?.group)) {
 
395
  setActiveLandmark(landmark);
396
  }
397
+ modifyImageWithRateLimit({
398
+ landmark,
399
+ vector,
400
+ mode: 'HOVERING'
401
+ });
402
  }
403
+ }, [currentLandmark, dragStart, setActiveLandmark, setIsDragging, modifyImageWithRateLimit, landmarkCenters]);
404
 
405
+ const handleEnd = useCallback((x: number, y: number, mode: ActionMode) => {
406
  if (!canvasRef.current) return;
407
 
408
  const rect = canvasRef.current.getBoundingClientRect();
409
+ const normalizedX = (x - rect.left) / rect.width;
410
+ const normalizedY = (y - rect.top) / rect.height;
411
 
 
412
  if (dragStart && dragStartRef.current) {
413
+ const landmark = findClosestLandmark(normalizedX, normalizedY, currentLandmark?.group);
414
 
 
 
 
 
 
415
  modifyImageWithRateLimit({
416
+ landmark: currentLandmark || landmark,
417
  vector: {
418
+ x: normalizedX - landmarkCenters[landmark.group].x,
419
+ y: normalizedY - landmarkCenters[landmark.group].y,
420
+ z: 0
421
+ },
422
+ mode
423
  });
424
  }
425
 
426
  setIsDragging(false);
427
  dragStartRef.current = null;
428
  setActiveLandmark(undefined);
429
+ }, [currentLandmark, isDragging, modifyImageWithRateLimit, findClosestLandmark, setActiveLandmark, landmarkCenters, setIsDragging]);
430
 
431
+ const handleMouseDown = useCallback((event: React.MouseEvent<HTMLCanvasElement>) => {
432
+ const mode: ActionMode = event.button === 0 ? 'PRIMARY' : 'SECONDARY';
433
+ handleStart(event.clientX, event.clientY, mode);
434
+ }, [handleStart]);
435
+
436
+ const handleMouseMove = useCallback((event: React.MouseEvent<HTMLCanvasElement>) => {
437
+ const mode: ActionMode = event.buttons === 1 ? 'PRIMARY' : 'SECONDARY';
438
+ handleMove(event.clientX, event.clientY, mode);
439
+ }, [handleMove]);
440
+
441
+ const handleMouseUp = useCallback((event: React.MouseEvent<HTMLCanvasElement>) => {
442
+ const mode: ActionMode = event.buttons === 1 ? 'PRIMARY' : 'SECONDARY';
443
+ handleEnd(event.clientX, event.clientY, mode);
444
+ }, [handleEnd]);
445
+
446
+ const handleTouchStart = useCallback((event: React.TouchEvent<HTMLCanvasElement>) => {
447
+ const mode: ActionMode = event.touches.length === 1 ? 'PRIMARY' : 'SECONDARY';
448
+ const touch = event.touches[0];
449
+ handleStart(touch.clientX, touch.clientY, mode);
450
+ }, [handleStart]);
451
+
452
+ const handleTouchMove = useCallback((event: React.TouchEvent<HTMLCanvasElement>) => {
453
+ const mode: ActionMode = event.touches.length === 1 ? 'PRIMARY' : 'SECONDARY';
454
+ const touch = event.touches[0];
455
+ handleMove(touch.clientX, touch.clientY, mode);
456
+ }, [handleMove]);
457
+
458
+ const handleTouchEnd = useCallback((event: React.TouchEvent<HTMLCanvasElement>) => {
459
+ const mode: ActionMode = event.changedTouches.length === 1 ? 'PRIMARY' : 'SECONDARY';
460
+ const touch = event.changedTouches[0];
461
+ handleEnd(touch.clientX, touch.clientY, mode);
462
+ }, [handleEnd]);
463
 
464
  return {
465
  canvasRef,
466
  canvasRefCallback,
467
  mediaPipeRef,
 
468
  isMediaPipeReady,
469
  isDrawingUtilsReady,
 
 
 
 
 
 
 
 
470
 
471
  handleMouseDown,
472
  handleMouseUp,
473
  handleMouseMove,
474
+ handleTouchStart,
475
+ handleTouchMove,
476
+ handleTouchEnd,
477
 
478
  currentLandmark,
479
  currentOpacity,
client/src/hooks/useFacePokeAPI.ts CHANGED
@@ -1,12 +1,10 @@
1
  import { useEffect, useState } from "react";
2
 
3
  import { facePoke } from "../lib/facePoke";
4
- import { useMainStore } from "./useMainStore";
5
 
6
  export function useFacePokeAPI() {
7
 
8
  // State for FacePoke
9
- const [status, setStatus] = useState('');
10
  const [isDebugMode, setIsDebugMode] = useState(false);
11
  const [interruptMessage, setInterruptMessage] = useState<string | null>(null);
12
 
@@ -33,8 +31,6 @@ export function useFacePokeAPI() {
33
 
34
  return {
35
  facePoke,
36
- status,
37
- setStatus,
38
  isDebugMode,
39
  setIsDebugMode,
40
  interruptMessage,
 
1
  import { useEffect, useState } from "react";
2
 
3
  import { facePoke } from "../lib/facePoke";
 
4
 
5
  export function useFacePokeAPI() {
6
 
7
  // State for FacePoke
 
8
  const [isDebugMode, setIsDebugMode] = useState(false);
9
  const [interruptMessage, setInterruptMessage] = useState<string | null>(null);
10
 
 
31
 
32
  return {
33
  facePoke,
 
 
34
  isDebugMode,
35
  setIsDebugMode,
36
  interruptMessage,
client/src/hooks/useMainStore.ts CHANGED
@@ -1,32 +1,45 @@
1
  import { create } from 'zustand'
2
- import type { ClosestLandmark } from './useFaceLandmarkDetection'
3
- import type { ImageModificationParams } from '@/lib/facePoke'
4
-
5
- interface ImageState {
6
- error: string
7
- imageFile: File | null
8
- originalImage: string
9
- previewImage: string
10
- originalImageHash: string
11
- minLatency: number
12
- averageLatency: number
13
- maxLatency: number
14
- activeLandmark?: ClosestLandmark
15
- params: Partial<ImageModificationParams>
16
  setError: (error?: string) => void
17
- setImageFile: (file: File | null) => void
 
 
 
 
18
  setOriginalImage: (url: string) => void
19
  setOriginalImageHash: (hash: string) => void
20
  setPreviewImage: (url: string) => void
21
  resetImage: () => void
22
  setAverageLatency: (averageLatency: number) => void
23
  setActiveLandmark: (activeLandmark?: ClosestLandmark) => void
 
24
  setParams: (params: Partial<ImageModificationParams>) => void
 
 
 
 
 
 
 
25
  }
26
 
27
- export const useMainStore = create<ImageState>((set, get) => ({
 
28
  error: '',
29
  imageFile: null,
 
 
30
  originalImage: '',
31
  originalImageHash: '',
32
  previewImage: '',
@@ -34,9 +47,56 @@ export const useMainStore = create<ImageState>((set, get) => ({
34
  averageLatency: 190, // this should be the average for most people
35
  maxLatency: 4000, // max time between requests
36
  activeLandmark: undefined,
 
 
 
 
 
 
37
  params: {},
 
 
 
 
 
 
 
38
  setError: (error: string = '') => set({ error }),
39
- setImageFile: (file) => set({ imageFile: file }),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
40
  setOriginalImage: (url) => set({ originalImage: url }),
41
  setOriginalImageHash: (originalImageHash) => set({ originalImageHash }),
42
  setPreviewImage: (url) => set({ previewImage: url }),
@@ -48,6 +108,11 @@ export const useMainStore = create<ImageState>((set, get) => ({
48
  },
49
  setAverageLatency: (averageLatency: number) => set({ averageLatency }),
50
  setActiveLandmark: (activeLandmark?: ClosestLandmark) => set({ activeLandmark }),
 
 
 
 
 
51
  setParams: (params: Partial<ImageModificationParams>) => {
52
  const {params: previousParams } = get()
53
  set({ params: {
@@ -55,4 +120,317 @@ export const useMainStore = create<ImageState>((set, get) => ({
55
  ...params
56
  }})
57
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
58
  }))
 
1
  import { create } from 'zustand'
2
+
3
+ import * as vision from '@mediapipe/tasks-vision'
4
+
5
+ import { truncateFileName } from '@/lib/utils'
6
+ import { convertImageToBase64 } from '@/lib/convertImageToBase64'
7
+ import { type Metadata, type ImageModificationParams, type OnServerResponseParams, type ActionMode, type ImageStateValues, type ClosestLandmark } from '@/types'
8
+ import { mapRange } from '@/lib/mapRange'
9
+ import { facePoke } from '@/lib/facePoke'
10
+
11
+
12
+ export type ImageState = ImageStateValues & {
13
+ setStatus: (status?: string) => void
 
 
14
  setError: (error?: string) => void
15
+ setFaceLandmarks: (faceLandmarks: vision.NormalizedLandmark[][]) => void
16
+ setBlendShapes: (blendShapes: vision.Classifications[]) => void
17
+ setImageFile: (file?: File) => Promise<void>
18
+ setIsFollowingCursor: (isFollowingCursor: boolean) => void
19
+ setIsGazingAtCursor: (isGazingAtCursor: boolean) => void
20
  setOriginalImage: (url: string) => void
21
  setOriginalImageHash: (hash: string) => void
22
  setPreviewImage: (url: string) => void
23
  resetImage: () => void
24
  setAverageLatency: (averageLatency: number) => void
25
  setActiveLandmark: (activeLandmark?: ClosestLandmark) => void
26
+ setMetadata: (metadata?: Metadata) => void
27
  setParams: (params: Partial<ImageModificationParams>) => void
28
+ handleServerResponse: (params: OnServerResponseParams) => Promise<void>
29
+ applyModifiedHeadToCanvas: (headImageBlob: Blob) => Promise<string>
30
+ modifyImage: ({ landmark, vector, mode }: {
31
+ landmark: ClosestLandmark
32
+ vector: { x: number; y: number; z: number }
33
+ mode: ActionMode
34
+ }) => Promise<void>
35
  }
36
 
37
+ export const getDefaultState = (): ImageStateValues => ({
38
+ status: '',
39
  error: '',
40
  imageFile: null,
41
+ isFollowingCursor: false,
42
+ isGazingAtCursor: false,
43
  originalImage: '',
44
  originalImageHash: '',
45
  previewImage: '',
 
47
  averageLatency: 190, // this should be the average for most people
48
  maxLatency: 4000, // max time between requests
49
  activeLandmark: undefined,
50
+ metadata: {
51
+ center: [0, 0],
52
+ size: 0,
53
+ bbox: [[0,0],[0,0],[0,0],[0,0]],
54
+ angle: 0,
55
+ },
56
  params: {},
57
+ faceLandmarks: [],
58
+ blendShapes: [],
59
+ })
60
+
61
+ export const useMainStore = create<ImageState>((set, get) => ({
62
+ ...getDefaultState(),
63
+ setStatus: (status: string = '') => set({ status }),
64
  setError: (error: string = '') => set({ error }),
65
+ setFaceLandmarks: (faceLandmarks: vision.NormalizedLandmark[][]) => {
66
+ set({ faceLandmarks })
67
+ },
68
+ setBlendShapes: (blendShapes: vision.Classifications[]) => {
69
+ set({ blendShapes })
70
+ },
71
+ setImageFile: async (file?: File) => {
72
+ if (!file) {
73
+ set({
74
+ ...getDefaultState(),
75
+ status: 'No file selected',
76
+ })
77
+ return;
78
+ }
79
+
80
+ try {
81
+ const image = await convertImageToBase64(file);
82
+ set({
83
+ ...getDefaultState(),
84
+ imageFile: file,
85
+ status: `File selected: ${truncateFileName(file.name, 16)}`,
86
+ previewImage: image,
87
+ originalImage: image,
88
+ })
89
+ facePoke.loadImage(image);
90
+ } catch (err) {
91
+ console.log(`failed to load the image: `, err);
92
+ set({
93
+ ...getDefaultState(),
94
+ status: 'Failed to load the image',
95
+ })
96
+ }
97
+ },
98
+ setIsFollowingCursor: (isFollowingCursor: boolean) => set({ isFollowingCursor }),
99
+ setIsGazingAtCursor: (isGazingAtCursor: boolean) => set({ isGazingAtCursor }),
100
  setOriginalImage: (url) => set({ originalImage: url }),
101
  setOriginalImageHash: (originalImageHash) => set({ originalImageHash }),
102
  setPreviewImage: (url) => set({ previewImage: url }),
 
108
  },
109
  setAverageLatency: (averageLatency: number) => set({ averageLatency }),
110
  setActiveLandmark: (activeLandmark?: ClosestLandmark) => set({ activeLandmark }),
111
+ setMetadata: (metadata?: Metadata) => set(metadata ? {
112
+ metadata
113
+ } : {
114
+ metadata: getDefaultState().metadata,
115
+ }),
116
  setParams: (params: Partial<ImageModificationParams>) => {
117
  const {params: previousParams } = get()
118
  set({ params: {
 
120
  ...params
121
  }})
122
  },
123
+ handleServerResponse: async (params: OnServerResponseParams) => {
124
+ const { originalImage, setMetadata, setPreviewImage, setOriginalImageHash, applyModifiedHeadToCanvas, modifyImage } = useMainStore.getState();
125
+ if (typeof params.error === "string") {
126
+ console.error(`handleServerResponse: failed to perform the request, resetting the app (${params.error})`)
127
+ setPreviewImage(originalImage)
128
+ setOriginalImageHash('')
129
+ } else if (typeof params.image !== "undefined") {
130
+
131
+ // this is where we decide to paste back the image as a whole,
132
+ // or apply some shenanigans to only past back the head.
133
+ // the part about the head is not done yet, so we do it all for now.
134
+
135
+ // --- old way: use it whole ---
136
+ const image = await convertImageToBase64(params.image);
137
+
138
+ // --- future way: try to only apply the head ---
139
+ // const image = await applyModifiedHeadToCanvas(params.image);
140
+
141
+ setPreviewImage(image);
142
+ } else if (typeof params.loaded !== "undefined") {
143
+ //console.log(`handleServerResponse: received a json`, params)
144
+ setOriginalImageHash(params.loaded.h)
145
+ setMetadata({
146
+ center: params.loaded.c, // center - 2x1
147
+ size: params.loaded.s, // size - scalar
148
+ bbox: params.loaded.b, // bbox - 4x2
149
+ angle: params.loaded.a, //angle - rad, counterclockwise
150
+ })
151
+
152
+ // right after we received the hash, we perform a first blank request
153
+ await modifyImage({
154
+ landmark: {
155
+ group: 'background',
156
+ distance: 0,
157
+ vector: { x: 0.5, y: 0.5, z: 0 }
158
+ },
159
+ vector: {x: 0, y: 0, z: 0 },
160
+ mode: 'PRIMARY'
161
+ })
162
+ } else {
163
+ console.log(`handleServerResponse: received an unknown json`, params)
164
+ }
165
+ },
166
+
167
+ applyModifiedHeadToCanvas: async (headImageBlob: Blob): Promise<string> => {
168
+ return new Promise(async (resolve, reject) => {
169
+ const originalImg = new Image();
170
+
171
+ const { originalImage, metadata } = useMainStore.getState();
172
+ originalImg.onload = async () => {
173
+ const canvas = document.createElement('canvas');
174
+ const ctx = canvas.getContext('2d');
175
+
176
+ if (!ctx) {
177
+ reject(new Error('Failed to get 2D context'));
178
+ return;
179
+ }
180
+
181
+ // Get device pixel ratio
182
+ const pixelRatio = window.devicePixelRatio || 1;
183
+
184
+ canvas.width = originalImg.width;
185
+ canvas.height = originalImg.height;
186
+
187
+ // Draw original image
188
+ ctx.drawImage(originalImg, 0, 0);
189
+
190
+ const headImageBitmap = await createImageBitmap(headImageBlob, {
191
+ resizeQuality: 'high'
192
+ });
193
+
194
+ // Create a temporary canvas for the head image with gradient
195
+ const tempCanvas = document.createElement('canvas');
196
+ const tempCtx = tempCanvas.getContext('2d');
197
+
198
+ if (!tempCtx) {
199
+ reject(new Error('Failed to get 2D context for temporary canvas'));
200
+ return;
201
+ }
202
+
203
+ tempCanvas.width = headImageBitmap.width;
204
+ tempCanvas.height = headImageBitmap.height;
205
+
206
+ // Draw the head image on the temporary canvas
207
+ tempCtx.drawImage(headImageBitmap, 0, 0);
208
+
209
+ // Create gradient mask
210
+ const gradientSize = 20; // Size of the gradient in pixels
211
+ const gradient = tempCtx.createRadialGradient(
212
+ tempCanvas.width / 2, tempCanvas.height / 2, Math.min(tempCanvas.width, tempCanvas.height) / 2 - gradientSize,
213
+ tempCanvas.width / 2, tempCanvas.height / 2, Math.min(tempCanvas.width, tempCanvas.height) / 2
214
+ );
215
+
216
+ gradient.addColorStop(0, 'rgba(0, 0, 0, 1)');
217
+ gradient.addColorStop(1, 'rgba(0, 0, 0, 0)');
218
+
219
+ // Apply gradient mask
220
+ tempCtx.globalCompositeOperation = 'destination-in';
221
+ tempCtx.fillStyle = gradient;
222
+ tempCtx.fillRect(0, 0, tempCanvas.width, tempCanvas.height);
223
+
224
+ console.log("metadata:", metadata);
225
+ ctx.save();
226
+ ctx.rotate(metadata.angle);
227
+
228
+ // TODO: this is where we need to grab the coordinates from the metadata and start drawing our image
229
+ // now the issue if that there is an angle, so we need to see how this impacts the transformation
230
+ ctx.restore();
231
+ /*
232
+ TODO finish the implementation
233
+
234
+ console.log("metadata:", metadata);
235
+ ctx.save();
236
+ ctx.rotate(metadata.angle);
237
+
238
+ // TODO we need the coordinate inside the final image
239
+ ctx.drawImage(
240
+ tempCanvas,
241
+ topLeftCornerX,
242
+ topLeftCornerY,
243
+ width,
244
+ height,
245
+ );
246
+
247
+ ctx.restore();
248
+ */
249
+
250
+
251
+ // Our head image already contains compression artifacts,
252
+ // so let's avoid double-encoding in here and use PNG
253
+ resolve(canvas.toDataURL('image/png'));
254
+ };
255
+
256
+ originalImg.src = originalImage;
257
+ })
258
+ },
259
+ modifyImage: async ({ landmark, vector, mode }: {
260
+ landmark: ClosestLandmark
261
+ vector: { x: number; y: number; z: number }
262
+ mode: ActionMode
263
+ }): Promise<void> => {
264
+
265
+ const {
266
+ originalImage,
267
+ originalImageHash,
268
+ params: previousParams,
269
+ setParams,
270
+ setError,
271
+ isFollowingCursor,
272
+ isGazingAtCursor
273
+ } = get()
274
+
275
+
276
+ if (!originalImage) {
277
+ console.error('Image file or facePoke not available');
278
+ return;
279
+ }
280
+
281
+ const params = {
282
+ ...previousParams
283
+ }
284
+
285
+ const generalControl = {
286
+ minX: -0.30,
287
+ maxX: 0.30,
288
+ minY: -0.30,
289
+ maxY: 0.30
290
+ }
291
+
292
+ const pupilControl = {
293
+ minX: -0.50,
294
+ maxX: 0.50,
295
+ minY: -0.50,
296
+ maxY: 0.50
297
+ }
298
+
299
+ const eyeControl = {
300
+ minX: -0.50,
301
+ maxX: 0.50,
302
+ minY: -0.50,
303
+ maxY: 0.50
304
+ }
305
+
306
+ // for the min and max value, we can look here:
307
+ // https://github.com/fofr/cog-expression-editor/blob/main/predict.py
308
+
309
+ // regardless of the landmark, those rule will apply
310
+ if (isFollowingCursor) {
311
+ // displacing the face horizontally by moving the mouse on the X axis
312
+ // should perform a yaw rotation
313
+ // rotate_yaw (min: -20, max: 20, default: 0)
314
+ const yawMin = -40
315
+ const yawMax = 40
316
+
317
+ // note: we invert the axis here
318
+ params.rotate_yaw = mapRange(-vector.x, generalControl.minX, generalControl.maxX, yawMin, yawMax);
319
+
320
+ // displacing the face vertically by moving the mouse on the Y axis
321
+ // should perform a pitch rotation
322
+ // rotate_pitch (min: -20, max: 20, default: 0)
323
+ const pitchMin = -40
324
+ const pitchMax = 40
325
+ params.rotate_pitch = mapRange(vector.y, generalControl.minY, generalControl.maxY, pitchMin, pitchMax);
326
+ }
327
+
328
+ if (isGazingAtCursor) {
329
+ const pupilsXMin = -15
330
+ const pupilsXMax = 15
331
+ params.pupil_x = mapRange(vector.x, pupilControl.minX, pupilControl.maxX, pupilsXMin, pupilsXMax);
332
+ const pupilsYMin = -2 // -15
333
+ const pupilsYMax = 8 // 15
334
+ params.pupil_y = mapRange(-vector.y, pupilControl.minY, pupilControl.maxY, pupilsYMin, pupilsYMax);
335
+ }
336
+
337
+ // if the user clicked on the primary or secondary button,
338
+ // then it triggers some more things
339
+ if (mode !== 'HOVERING') {
340
+ switch (landmark.group) {
341
+ case 'leftEye':
342
+ case 'rightEye':
343
+ // if (mode === 'PRIMARY') {
344
+ const pupilsXMin = -15
345
+ const pupilsXMax = 15
346
+ params.pupil_x = mapRange(vector.x, pupilControl.minX, pupilControl.maxX, pupilsXMin, pupilsXMax);
347
+
348
+ const eyesMin = -20
349
+ const eyesMax = 5
350
+ params.eyes = mapRange(-vector.y, eyeControl.minX, eyeControl.maxX, eyesMin, eyesMax);
351
+ //}
352
+
353
+ break;
354
+ case 'leftEyebrow':
355
+ case 'rightEyebrow':
356
+ // moving the mouse vertically for the eyebrow
357
+ // should make them up/down
358
+ // eyebrow (min: -10, max: 15, default: 0)
359
+ const eyebrowMin = -10
360
+ const eyebrowMax = 15
361
+ params.eyebrow = mapRange(-vector.y, eyeControl.minY, eyeControl.maxY, eyebrowMin, eyebrowMax);
362
+
363
+ break;
364
+ case 'lips':
365
+ // aaa (min: -30, max: 120, default: 0)
366
+ const aaaMin = -30
367
+ const aaaMax = 120
368
+ params.aaa = mapRange(-vector.y, eyeControl.minY, eyeControl.maxY, aaaMin, aaaMax);
369
+
370
+ // eee (min: -20, max: 15, default: 0)
371
+ const eeeMin = -20
372
+ const eeeMax = 15
373
+ params.eee = mapRange(vector.x, eyeControl.minX, eyeControl.maxX, eeeMin, eeeMax);
374
+
375
+ // woo (min: -20, max: 15, default: 0)
376
+ //const wooMin = -20
377
+ //const wooMax = 15
378
+ //params.woo = mapRange(-vector.x, eyeControl.minX, eyeControl.maxX, wooMin, wooMax);
379
+
380
+ break;
381
+ case 'faceOval':
382
+ // displacing the face horizontally by moving the mouse on the X axis
383
+ // should perform a yaw rotation
384
+ // rotate_roll (min: -20, max: 20, default: 0)
385
+ const rollMin = -40
386
+ const rollMax = 40
387
+
388
+ // note: we invert the axis here
389
+ params.rotate_roll = mapRange(vector.x, eyeControl.minX, eyeControl.maxX, rollMin, rollMax);
390
+ break;
391
+
392
+ case 'background':
393
+ // displacing the face horizontally by moving the mouse on the X axis
394
+ // should perform a yaw rotation
395
+ // rotate_yaw (min: -20, max: 20, default: 0)
396
+ const yawMin = -40
397
+ const yawMax = 40
398
+
399
+ // note: we invert the axis here
400
+ params.rotate_yaw = mapRange(-vector.x, generalControl.minX, generalControl.maxX, yawMin, yawMax);
401
+
402
+ // displacing the face vertically by moving the mouse on the Y axis
403
+ // should perform a pitch rotation
404
+ // rotate_pitch (min: -20, max: 20, default: 0)
405
+ const pitchMin = -40
406
+ const pitchMax = 40
407
+ params.rotate_pitch = mapRange(vector.y, eyeControl.minY, eyeControl.maxY, pitchMin, pitchMax);
408
+ break;
409
+ default:
410
+ return
411
+ }
412
+ }
413
+
414
+ for (const [key, value] of Object.entries(params)) {
415
+ if (isNaN(value as any) || !isFinite(value as any)) {
416
+ console.log(`${key} is NaN, aborting`)
417
+ return
418
+ }
419
+ }
420
+
421
+ //console.log(`PITCH=${params.rotate_pitch || 0}, YAW=${params.rotate_yaw || 0}, ROLL=${params.rotate_roll || 0}`);
422
+
423
+ setParams(params)
424
+
425
+ try {
426
+
427
+ if (originalImageHash) {
428
+ facePoke.transformImage(originalImageHash, params);
429
+ }
430
+
431
+ } catch (error) {
432
+ // console.error('Error modifying image:', error);
433
+ setError('Failed to modify image');
434
+ }
435
+ }
436
  }))
client/src/layout.tsx CHANGED
@@ -5,7 +5,7 @@ export function Layout({ children }: { children: ReactNode }) {
5
  <div className="fixed min-h-screen w-full flex items-center justify-center bg-gradient-to-br from-gray-300 to-stone-300"
6
  style={{ boxShadow: "inset 0 0 10vh 0 rgb(0 0 0 / 30%)" }}>
7
  <div className="min-h-screen w-full py-8 flex flex-col justify-center">
8
- <div className="relative p-4 sm:max-w-5xl sm:mx-auto">
9
  {children}
10
  </div>
11
  </div>
 
5
  <div className="fixed min-h-screen w-full flex items-center justify-center bg-gradient-to-br from-gray-300 to-stone-300"
6
  style={{ boxShadow: "inset 0 0 10vh 0 rgb(0 0 0 / 30%)" }}>
7
  <div className="min-h-screen w-full py-8 flex flex-col justify-center">
8
+ <div className="flex flex-col items-center justify-center p-4 sm:max-w-5xl sm:mx-auto">
9
  {children}
10
  </div>
11
  </div>
client/src/lib/convertImageToBase64.ts CHANGED
@@ -1,4 +1,4 @@
1
- export async function convertImageToBase64(imageFile: File): Promise<string> {
2
  return new Promise((resolve, reject) => {
3
  const reader = new FileReader();
4
 
@@ -11,9 +11,9 @@ export async function convertImageToBase64(imageFile: File): Promise<string> {
11
  };
12
 
13
  reader.onerror = () => {
14
- reject(new Error('Error reading file'));
15
  };
16
 
17
- reader.readAsDataURL(imageFile);
18
  });
19
  }
 
1
+ export async function convertImageToBase64(imageFileOrBlob: File | Blob): Promise<string> {
2
  return new Promise((resolve, reject) => {
3
  const reader = new FileReader();
4
 
 
11
  };
12
 
13
  reader.onerror = () => {
14
+ reject(new Error('Error reading file or blob'));
15
  };
16
 
17
+ reader.readAsDataURL(imageFileOrBlob);
18
  });
19
  }
client/src/lib/facePoke.ts CHANGED
@@ -1,153 +1,35 @@
1
- import { v4 as uuidv4 } from 'uuid';
2
- import { CircularBuffer } from './circularBuffer';
3
- import { useMainStore } from '@/hooks/useMainStore';
4
 
5
- /**
6
- * Represents a tracked request with its UUID and timestamp.
7
- */
8
- export interface TrackedRequest {
9
- uuid: string;
10
- timestamp: number;
11
- }
12
-
13
- /**
14
- * Represents the parameters for image modification.
15
- */
16
- export interface ImageModificationParams {
17
- eyes: number;
18
- eyebrow: number;
19
- wink: number;
20
- pupil_x: number;
21
- pupil_y: number;
22
- aaa: number;
23
- eee: number;
24
- woo: number;
25
- smile: number;
26
- rotate_pitch: number;
27
- rotate_yaw: number;
28
- rotate_roll: number;
29
- }
30
-
31
- /**
32
- * Represents a message to modify an image.
33
- */
34
- export interface ModifyImageMessage {
35
- type: 'modify_image';
36
- image?: string;
37
- image_hash?: string;
38
- params: Partial<ImageModificationParams>;
39
- }
40
-
41
-
42
- /**
43
- * Callback type for handling modified images.
44
- */
45
- type OnModifiedImage = (image: string, image_hash: string) => void;
46
-
47
- /**
48
- * Enum representing the different states of a WebSocket connection.
49
- */
50
- enum WebSocketState {
51
- CONNECTING = 0,
52
- OPEN = 1,
53
- CLOSING = 2,
54
- CLOSED = 3
55
- }
56
 
57
  /**
58
  * FacePoke class manages the WebSocket connection
59
  */
60
  export class FacePoke {
61
  private ws: WebSocket | null = null;
62
- private readonly connectionId: string = uuidv4();
63
  private isUnloading: boolean = false;
64
- private onModifiedImage: OnModifiedImage = () => {};
65
  private reconnectAttempts: number = 0;
66
  private readonly maxReconnectAttempts: number = 5;
67
  private readonly reconnectDelay: number = 5000;
68
  private readonly eventListeners: Map<string, Set<Function>> = new Map();
69
 
70
- private requestTracker: Map<string, TrackedRequest> = new Map();
71
- private responseTimeBuffer: CircularBuffer<number>;
72
- private readonly MAX_TRACKED_TIMES = 5; // Number of recent response times to track
73
-
74
  /**
75
  * Creates an instance of FacePoke.
76
  * Initializes the WebSocket connection.
77
  */
78
  constructor() {
79
- console.log(`[FacePoke] Initializing FacePoke instance with connection ID: ${this.connectionId}`);
80
  this.initializeWebSocket();
81
  this.setupUnloadHandler();
82
-
83
- this.responseTimeBuffer = new CircularBuffer<number>(this.MAX_TRACKED_TIMES);
84
- console.log(`[FacePoke] Initialized response time tracker with capacity: ${this.MAX_TRACKED_TIMES}`);
85
- }
86
-
87
-
88
- /**
89
- * Generates a unique UUID for a request and starts tracking it.
90
- * @returns The generated UUID for the request.
91
- */
92
- private trackRequest(): string {
93
- const uuid = uuidv4();
94
- this.requestTracker.set(uuid, { uuid, timestamp: Date.now() });
95
- // console.log(`[FacePoke] Started tracking request with UUID: ${uuid}`);
96
- return uuid;
97
- }
98
-
99
- /**
100
- * Completes tracking for a request and updates response time statistics.
101
- * @param uuid - The UUID of the completed request.
102
- */
103
- private completeRequest(uuid: string): void {
104
- const request = this.requestTracker.get(uuid);
105
- if (request) {
106
- const responseTime = Date.now() - request.timestamp;
107
- this.responseTimeBuffer.push(responseTime);
108
- this.requestTracker.delete(uuid);
109
- this.updateThrottleTime();
110
- console.log(`[FacePoke] Completed request ${uuid}. Response time: ${responseTime}ms`);
111
- } else {
112
- console.warn(`[FacePoke] Attempted to complete unknown request: ${uuid}`);
113
- }
114
- }
115
-
116
- /**
117
- * Calculates the average response time from recent requests.
118
- * @returns The average response time in milliseconds.
119
- */
120
- private calculateAverageResponseTime(): number {
121
- const times = this.responseTimeBuffer.getAll();
122
-
123
- const averageLatency = useMainStore.getState().averageLatency;
124
-
125
- if (times.length === 0) return averageLatency;
126
- const sum = times.reduce((acc, time) => acc + time, 0);
127
- return sum / times.length;
128
- }
129
-
130
- /**
131
- * Updates the throttle time based on recent response times.
132
- */
133
- private updateThrottleTime(): void {
134
- const { minLatency, maxLatency, averageLatency, setAverageLatency } = useMainStore.getState();
135
- const avgResponseTime = this.calculateAverageResponseTime();
136
- const newLatency = Math.min(minLatency, Math.max(minLatency, avgResponseTime));
137
-
138
- if (newLatency !== averageLatency) {
139
- setAverageLatency(newLatency)
140
- console.log(`[FacePoke] Updated throttle time (latency is ${newLatency}ms)`);
141
- }
142
  }
143
 
144
  /**
145
  * Sets the callback function for handling modified images.
146
  * @param handler - The function to be called when a modified image is received.
147
  */
148
- public setOnModifiedImage(handler: OnModifiedImage): void {
149
- this.onModifiedImage = handler;
150
- console.log(`[FacePoke] onModifiedImage handler set`);
151
  }
152
 
153
  /**
@@ -165,53 +47,36 @@ export class FacePoke {
165
  * Implements exponential backoff for reconnection attempts.
166
  */
167
  private async initializeWebSocket(): Promise<void> {
168
- console.log(`[FacePoke][${this.connectionId}] Initializing WebSocket connection`);
169
 
170
  const connect = () => {
171
  this.ws = new WebSocket(`wss://${window.location.host}/ws`);
172
 
173
  this.ws.onopen = this.handleWebSocketOpen.bind(this);
174
- this.ws.onmessage = this.handleWebSocketMessage.bind(this);
175
  this.ws.onclose = this.handleWebSocketClose.bind(this);
176
  this.ws.onerror = this.handleWebSocketError.bind(this);
 
177
  };
178
 
179
- // const debouncedConnect = debounce(connect, this.reconnectDelay, { leading: true, trailing: false });
180
-
181
  connect(); // Initial connection attempt
182
  }
183
 
 
 
 
 
 
 
 
184
  /**
185
  * Handles the WebSocket open event.
186
  */
187
  private handleWebSocketOpen(): void {
188
- console.log(`[FacePoke][${this.connectionId}] WebSocket connection opened`);
189
  this.reconnectAttempts = 0; // Reset reconnect attempts on successful connection
190
  this.emitEvent('websocketOpen');
191
  }
192
 
193
- // Update handleWebSocketMessage to complete request tracking
194
- private handleWebSocketMessage(event: MessageEvent): void {
195
- try {
196
- const data = JSON.parse(event.data);
197
- // console.log(`[FacePoke][${this.connectionId}] Received JSON data:`, data);
198
-
199
- if (data.uuid) {
200
- this.completeRequest(data.uuid);
201
- }
202
-
203
- if (data.type === 'modified_image') {
204
- if (data?.image) {
205
- this.onModifiedImage(data.image, data.image_hash);
206
- }
207
- }
208
-
209
- this.emitEvent('message', data);
210
- } catch (error) {
211
- console.error(`[FacePoke][${this.connectionId}] Error parsing WebSocket message:`, error);
212
- }
213
- }
214
-
215
  /**
216
  * Handles WebSocket close events.
217
  * Implements reconnection logic with exponential backoff.
@@ -219,9 +84,9 @@ export class FacePoke {
219
  */
220
  private handleWebSocketClose(event: CloseEvent): void {
221
  if (event.wasClean) {
222
- console.log(`[FacePoke][${this.connectionId}] WebSocket connection closed cleanly, code=${event.code}, reason=${event.reason}`);
223
  } else {
224
- console.warn(`[FacePoke][${this.connectionId}] WebSocket connection abruptly closed`);
225
  }
226
 
227
  this.emitEvent('websocketClose', event);
@@ -230,10 +95,10 @@ export class FacePoke {
230
  if (!this.isUnloading && this.reconnectAttempts < this.maxReconnectAttempts) {
231
  this.reconnectAttempts++;
232
  const delay = Math.min(1000 * (2 ** this.reconnectAttempts), 30000); // Exponential backoff, max 30 seconds
233
- console.log(`[FacePoke][${this.connectionId}] Attempting to reconnect in ${delay}ms (Attempt ${this.reconnectAttempts}/${this.maxReconnectAttempts})...`);
234
  setTimeout(() => this.initializeWebSocket(), delay);
235
  } else if (this.reconnectAttempts >= this.maxReconnectAttempts) {
236
- console.error(`[FacePoke][${this.connectionId}] Max reconnect attempts reached. Please refresh the page.`);
237
  this.emitEvent('maxReconnectAttemptsReached');
238
  }
239
  }
@@ -243,7 +108,7 @@ export class FacePoke {
243
  * @param error - The error event.
244
  */
245
  private handleWebSocketError(error: Event): void {
246
- console.error(`[FacePoke][${this.connectionId}] WebSocket error:`, error);
247
  this.emitEvent('websocketError', error);
248
  }
249
 
@@ -261,33 +126,32 @@ export class FacePoke {
261
  this.emitEvent('cleanup');
262
  }
263
 
264
- /**
265
- * Modifies an image based on the provided parameters
266
- * @param image - The data-uri base64 image to modify.
267
- * @param imageHash - The hash of the image to modify.
268
- * @param params - The parameters for image modification.
269
- */
270
- public modifyImage(image: string | null, imageHash: string | null, params: Partial<ImageModificationParams>): void {
271
- try {
272
- const message: ModifyImageMessage = {
273
- type: 'modify_image',
274
- params: params
275
- };
276
 
277
- if (image) {
278
- message.image = image;
279
- } else if (imageHash) {
280
- message.image_hash = imageHash;
281
- } else {
282
- throw new Error('Either image or imageHash must be provided');
283
- }
284
 
285
- this.sendJsonMessage(message);
286
- // console.log(`[FacePoke] Sent modify image request with UUID: ${uuid}`);
287
- } catch (err) {
288
- console.error(`[FacePoke] Failed to modify the image:`, err);
289
- }
 
 
 
 
 
 
 
 
 
 
290
  }
 
291
 
292
  /**
293
  * Sends a JSON message through the WebSocket connection with request tracking.
@@ -301,11 +165,11 @@ export class FacePoke {
301
  this.emitEvent('sendJsonMessageError', error);
302
  throw error;
303
  }
304
-
305
- const uuid = this.trackRequest();
306
- const messageWithUuid = { ...message, uuid };
307
- // console.log(`[FacePoke] Sending JSON message with UUID ${uuid}:`, messageWithUuid);
308
- this.ws.send(JSON.stringify(messageWithUuid));
309
  }
310
 
311
  /**
 
1
+ import { WebSocketState, type ImageModificationParams, type OnServerResponse } from "@/types";
 
 
2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
 
4
  /**
5
  * FacePoke class manages the WebSocket connection
6
  */
7
  export class FacePoke {
8
  private ws: WebSocket | null = null;
 
9
  private isUnloading: boolean = false;
10
+ private onServerResponse: OnServerResponse = async () => {};
11
  private reconnectAttempts: number = 0;
12
  private readonly maxReconnectAttempts: number = 5;
13
  private readonly reconnectDelay: number = 5000;
14
  private readonly eventListeners: Map<string, Set<Function>> = new Map();
15
 
 
 
 
 
16
  /**
17
  * Creates an instance of FacePoke.
18
  * Initializes the WebSocket connection.
19
  */
20
  constructor() {
21
+ console.log(`[FacePoke] Initializing FacePoke instance`);
22
  this.initializeWebSocket();
23
  this.setupUnloadHandler();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
24
  }
25
 
26
  /**
27
  * Sets the callback function for handling modified images.
28
  * @param handler - The function to be called when a modified image is received.
29
  */
30
+ public setOnServerResponse(handler: OnServerResponse): void {
31
+ this.onServerResponse = handler;
32
+ console.log(`[FacePoke] onServerResponse handler set`);
33
  }
34
 
35
  /**
 
47
  * Implements exponential backoff for reconnection attempts.
48
  */
49
  private async initializeWebSocket(): Promise<void> {
50
+ console.log(`[FacePoke] Initializing WebSocket connection`);
51
 
52
  const connect = () => {
53
  this.ws = new WebSocket(`wss://${window.location.host}/ws`);
54
 
55
  this.ws.onopen = this.handleWebSocketOpen.bind(this);
 
56
  this.ws.onclose = this.handleWebSocketClose.bind(this);
57
  this.ws.onerror = this.handleWebSocketError.bind(this);
58
+ this.ws.onmessage = (this.handleWebSocketMessage.bind(this) as any)
59
  };
60
 
 
 
61
  connect(); // Initial connection attempt
62
  }
63
 
64
+ private handleWebSocketMessage(msg: MessageEvent) {
65
+ if (typeof msg.data === "string") {
66
+ this.onServerResponse({ loaded: JSON.parse(msg.data) as any });
67
+ } else if (typeof msg.data !== "undefined" ) {
68
+ this.onServerResponse({ image: msg.data as unknown as Blob });
69
+ }
70
+ }
71
  /**
72
  * Handles the WebSocket open event.
73
  */
74
  private handleWebSocketOpen(): void {
75
+ console.log(`[FacePoke] WebSocket connection opened`);
76
  this.reconnectAttempts = 0; // Reset reconnect attempts on successful connection
77
  this.emitEvent('websocketOpen');
78
  }
79
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
80
  /**
81
  * Handles WebSocket close events.
82
  * Implements reconnection logic with exponential backoff.
 
84
  */
85
  private handleWebSocketClose(event: CloseEvent): void {
86
  if (event.wasClean) {
87
+ console.log(`[FacePoke] WebSocket connection closed cleanly, code=${event.code}, reason=${event.reason}`);
88
  } else {
89
+ console.warn(`[FacePoke] WebSocket connection abruptly closed`);
90
  }
91
 
92
  this.emitEvent('websocketClose', event);
 
95
  if (!this.isUnloading && this.reconnectAttempts < this.maxReconnectAttempts) {
96
  this.reconnectAttempts++;
97
  const delay = Math.min(1000 * (2 ** this.reconnectAttempts), 30000); // Exponential backoff, max 30 seconds
98
+ console.log(`[FacePoke] Attempting to reconnect in ${delay}ms (Attempt ${this.reconnectAttempts}/${this.maxReconnectAttempts})...`);
99
  setTimeout(() => this.initializeWebSocket(), delay);
100
  } else if (this.reconnectAttempts >= this.maxReconnectAttempts) {
101
+ console.error(`[FacePoke] Max reconnect attempts reached. Please refresh the page.`);
102
  this.emitEvent('maxReconnectAttemptsReached');
103
  }
104
  }
 
108
  * @param error - The error event.
109
  */
110
  private handleWebSocketError(error: Event): void {
111
+ console.error(`[FacePoke] WebSocket error:`, error);
112
  this.emitEvent('websocketError', error);
113
  }
114
 
 
126
  this.emitEvent('cleanup');
127
  }
128
 
129
+ public async loadImage(image: string): Promise<void> {
130
+ // Extract the base64 part if it's a data URL
131
+ const base64Data = image.split(',')[1] || image;
 
 
 
 
 
 
 
 
 
132
 
133
+ const buffer = new Uint8Array(atob(base64Data).split('').map(char => char.charCodeAt(0)));
134
+ const blob = new Blob([buffer], { type: 'application/octet-binary' });
135
+ this.sendBlobMessage(await blob.arrayBuffer());
136
+ }
 
 
 
137
 
138
+ public transformImage(hash: string, params: Partial<ImageModificationParams>): void {
139
+ this.sendJsonMessage({ hash, params });
140
+ }
141
+
142
+ private sendBlobMessage(buffer: ArrayBuffer): void {
143
+ if (!this.ws || this.ws.readyState !== WebSocketState.OPEN) {
144
+ const error = new Error('WebSocket connection is not open');
145
+ console.error('[FacePoke] Error sending JSON message:', error);
146
+ this.emitEvent('sendJsonMessageError', error);
147
+ throw error;
148
+ }
149
+ try {
150
+ this.ws.send(buffer);
151
+ } catch (err) {
152
+ console.error(`failed to send the WebSocket message: ${err}`)
153
  }
154
+ }
155
 
156
  /**
157
  * Sends a JSON message through the WebSocket connection with request tracking.
 
165
  this.emitEvent('sendJsonMessageError', error);
166
  throw error;
167
  }
168
+ try {
169
+ this.ws.send(JSON.stringify(message));
170
+ } catch (err) {
171
+ console.error(`failed to send the WebSocket message: ${err}`)
172
+ }
173
  }
174
 
175
  /**
client/src/lib/mapRange.ts ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ // Function to map a value from one range to another
2
+ export const mapRange = (value: number, inMin: number, inMax: number, outMin: number, outMax: number): number => {
3
+ return Math.min(outMax, Math.max(outMin, ((value - inMin) * (outMax - outMin)) / (inMax - inMin) + outMin));
4
+ };
client/src/types.ts ADDED
@@ -0,0 +1,92 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import * as vision from '@mediapipe/tasks-vision';
2
+
3
+ /**
4
+ * Represents the parameters for image modification.
5
+ */
6
+ export interface ImageModificationParams {
7
+ eyes: number;
8
+ eyebrow: number;
9
+ wink: number;
10
+ pupil_x: number;
11
+ pupil_y: number;
12
+ aaa: number;
13
+ eee: number;
14
+ woo: number;
15
+ smile: number;
16
+ rotate_pitch: number;
17
+ rotate_yaw: number;
18
+ rotate_roll: number;
19
+ }
20
+
21
+ export interface Metadata {
22
+ center: number[] //center - 2x1
23
+ size: number // size - scalar
24
+ bbox: number[][] // bbox - 4x2
25
+ angle: number //angle - rad, counterclockwise
26
+ }
27
+
28
+ /**
29
+ * Represents a message to modify an image.
30
+ */
31
+ export interface ModifyImageMessage {
32
+ image?: string;
33
+ hash?: string;
34
+ params: Partial<ImageModificationParams>;
35
+ }
36
+
37
+ export type OnServerResponseParams = {
38
+ image?: Blob
39
+ error?: string
40
+ loaded?: {
41
+ h: string
42
+ } & {
43
+ c: number[] //center - 2x1
44
+ s: number // size - scalar
45
+ b: number[][] // bbox - 4x2
46
+ a: number // angle - rad, counterclockwise
47
+ }
48
+ }
49
+
50
+ /**
51
+ * Callback type for handling modified images.
52
+ */
53
+ export type OnServerResponse = (params: OnServerResponseParams) => Promise<void>;
54
+
55
+ /**
56
+ * Enum representing the different states of a WebSocket connection.
57
+ */
58
+ export enum WebSocketState {
59
+ CONNECTING = 0,
60
+ OPEN = 1,
61
+ CLOSING = 2,
62
+ CLOSED = 3
63
+ }
64
+
65
+ export type ActionMode = 'HOVERING' | 'PRIMARY' | 'SECONDARY'
66
+ export type LandmarkGroup = 'lips' | 'leftEye' | 'leftEyebrow' | 'rightEye' | 'rightEyebrow' | 'faceOval' | 'background';
67
+ export type LandmarkCenter = { x: number; y: number; z: number };
68
+ export type ClosestLandmark = { group: LandmarkGroup; distance: number; vector: { x: number; y: number; z: number } };
69
+
70
+ export type MediaPipeResources = {
71
+ faceLandmarker: vision.FaceLandmarker | null;
72
+ drawingUtils: vision.DrawingUtils | null;
73
+ };
74
+
75
+ export interface ImageStateValues {
76
+ status: string
77
+ error: string
78
+ imageFile: File | null
79
+ isFollowingCursor: boolean
80
+ isGazingAtCursor: boolean
81
+ originalImage: string
82
+ previewImage: string
83
+ originalImageHash: string
84
+ minLatency: number
85
+ averageLatency: number
86
+ maxLatency: number
87
+ activeLandmark?: ClosestLandmark
88
+ metadata: Metadata
89
+ params: Partial<ImageModificationParams>
90
+ faceLandmarks: vision.NormalizedLandmark[][]
91
+ blendShapes: vision.Classifications[]
92
+ }
engine.py CHANGED
@@ -3,6 +3,7 @@ import hashlib
3
  import os
4
  import io
5
  import asyncio
 
6
  import base64
7
  from queue import Queue
8
  from typing import Dict, Any, List, Optional, Union
@@ -15,7 +16,7 @@ from PIL import Image
15
  from liveportrait.config.argument_config import ArgumentConfig
16
  from liveportrait.utils.camera import get_rotation_matrix
17
  from liveportrait.utils.io import resize_to_limit
18
- from liveportrait.utils.crop import prepare_paste_back, paste_back
19
 
20
  # Configure logging
21
  logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
@@ -56,12 +57,11 @@ class Engine:
56
 
57
  self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
58
 
59
- self.image_cache = {} # Stores the original images
60
  self.processed_cache = {} # Stores the processed image data
61
 
62
  logger.info("βœ… FacePoke Engine initialized successfully.")
63
 
64
- def get_image_hash(self, image: Union[Image.Image, str, bytes]) -> str:
65
  """
66
  Compute or retrieve the hash for an image.
67
 
@@ -86,35 +86,23 @@ class Engine:
86
  else:
87
  raise ValueError("Unsupported image type")
88
 
89
- @lru_cache(maxsize=256)
90
- def _process_image(self, image_hash: str) -> Dict[str, Any]:
91
- """
92
- Process the input image and cache the results.
93
-
94
- Args:
95
- image_hash (str): Hash of the input image.
96
-
97
- Returns:
98
- Dict[str, Any]: Processed image data.
99
- """
100
- # let's hide the logs as there are thousands of message slike this
101
- #logger.info(f"Processing image with hash: {image_hash}")
102
 
103
- if image_hash not in self.image_cache:
104
- raise ValueError(f"Image with hash {image_hash} not found in cache")
105
 
106
- image = self.image_cache[image_hash]
107
  img_rgb = np.array(image)
108
 
109
  inference_cfg = self.live_portrait.live_portrait_wrapper.cfg
110
- img_rgb = resize_to_limit(img_rgb, inference_cfg.ref_max_shape, inference_cfg.ref_shape_n)
111
- crop_info = self.live_portrait.cropper.crop_single_image(img_rgb)
112
  img_crop_256x256 = crop_info['img_crop_256x256']
113
 
114
- I_s = self.live_portrait.live_portrait_wrapper.prepare_source(img_crop_256x256)
115
- x_s_info = self.live_portrait.live_portrait_wrapper.get_kp_info(I_s)
116
- f_s = self.live_portrait.live_portrait_wrapper.extract_feature_3d(I_s)
117
- x_s = self.live_portrait.live_portrait_wrapper.transform_keypoint(x_s_info)
118
 
119
  processed_data = {
120
  'img_rgb': img_rgb,
@@ -127,50 +115,78 @@ class Engine:
127
 
128
  self.processed_cache[image_hash] = processed_data
129
 
130
- return processed_data
 
131
 
132
- async def modify_image(self, image_or_hash: Union[Image.Image, str, bytes], params: Dict[str, float]) -> str:
133
- """
134
- Modify the input image based on the provided parameters, using caching for efficiency
135
- and outputting the result as a WebP image.
136
 
137
- Args:
138
- image_or_hash (Union[Image.Image, str, bytes]): Input image as a PIL Image, base64-encoded string,
139
- image bytes, or a hash string.
140
- params (Dict[str, float]): Parameters for face transformation.
 
 
 
141
 
142
- Returns:
143
- str: Modified image as a base64-encoded WebP data URI.
 
 
144
 
145
- Raises:
146
- ValueError: If there's an error modifying the image or WebP is not supported.
147
- """
148
- # let's disable those logs completely as there are thousands of message slike this
149
- #logger.info("Starting image modification")
150
- #logger.debug(f"Modification parameters: {params}")
151
 
152
  try:
153
- image_hash = self.get_image_hash(image_or_hash)
154
-
155
- # If we don't have the image in cache yet, add it
156
- if image_hash not in self.image_cache:
157
- if isinstance(image_or_hash, (Image.Image, bytes)):
158
- self.image_cache[image_hash] = image_or_hash
159
- elif isinstance(image_or_hash, str) and len(image_or_hash) != 32:
160
- # It's a base64 string, not a hash
161
- self.image_cache[image_hash] = base64_data_uri_to_PIL_Image(image_or_hash)
162
- else:
163
- raise ValueError("Image not found in cache and no valid image provided")
164
-
165
- # Process the image (this will use the cache if available)
166
- if image_hash not in self.processed_cache:
167
- processed_data = await asyncio.to_thread(self._process_image, image_hash)
168
- else:
169
- processed_data = self.processed_cache[image_hash]
170
-
171
  # Apply modifications based on params
172
  x_d_new = processed_data['x_s_info']['kp'].clone()
173
- await self._apply_facial_modifications(x_d_new, params)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
174
 
175
  # Apply rotation
176
  R_new = get_rotation_matrix(
@@ -181,90 +197,40 @@ class Engine:
181
  x_d_new = processed_data['x_s_info']['scale'] * (x_d_new @ R_new) + processed_data['x_s_info']['t']
182
 
183
  # Apply stitching
184
- x_d_new = await asyncio.to_thread(self.live_portrait.live_portrait_wrapper.stitching, processed_data['x_s'], x_d_new)
185
 
186
  # Generate the output
187
- out = await asyncio.to_thread(self.live_portrait.live_portrait_wrapper.warp_decode, processed_data['f_s'], processed_data['x_s'], x_d_new)
188
- I_p = self.live_portrait.live_portrait_wrapper.parse_output(out['out'])[0]
 
 
189
 
190
- # Paste back to full size
191
- mask_ori = await asyncio.to_thread(
192
- prepare_paste_back,
 
 
 
 
 
 
 
193
  processed_data['inference_cfg'].mask_crop, processed_data['crop_info']['M_c2o'],
194
  dsize=(processed_data['img_rgb'].shape[1], processed_data['img_rgb'].shape[0])
195
  )
196
- I_p_to_ori_blend = await asyncio.to_thread(
197
- paste_back,
198
- I_p, processed_data['crop_info']['M_c2o'], processed_data['img_rgb'], mask_ori
199
  )
200
-
201
- # Convert the result to a PIL Image
202
  result_image = Image.fromarray(I_p_to_ori_blend)
203
 
204
- # Save as WebP
205
- buffered = io.BytesIO()
206
- result_image.save(buffered, format="WebP", quality=85) # Adjust quality as needed
207
- modified_image_base64 = base64.b64encode(buffered.getvalue()).decode('utf-8')
208
 
209
- #logger.info("Image modification completed successfully")
210
- return f"data:image/webp;base64,{modified_image_base64}"
 
 
211
 
212
  except Exception as e:
213
- #logger.error(f"Error in modify_image: {str(e)}")
214
- #logger.exception("Full traceback:")
215
  raise ValueError(f"Failed to modify image: {str(e)}")
216
-
217
- async def _apply_facial_modifications(self, x_d_new: torch.Tensor, params: Dict[str, float]) -> None:
218
- """
219
- Apply facial modifications to the keypoints based on the provided parameters.
220
-
221
- Args:
222
- x_d_new (torch.Tensor): Tensor of facial keypoints to be modified.
223
- params (Dict[str, float]): Parameters for face transformation.
224
- """
225
- modifications = [
226
- ('smile', [
227
- (0, 20, 1, -0.01), (0, 14, 1, -0.02), (0, 17, 1, 0.0065), (0, 17, 2, 0.003),
228
- (0, 13, 1, -0.00275), (0, 16, 1, -0.00275), (0, 3, 1, -0.0035), (0, 7, 1, -0.0035)
229
- ]),
230
- ('aaa', [
231
- (0, 19, 1, 0.001), (0, 19, 2, 0.0001), (0, 17, 1, -0.0001)
232
- ]),
233
- ('eee', [
234
- (0, 20, 2, -0.001), (0, 20, 1, -0.001), (0, 14, 1, -0.001)
235
- ]),
236
- ('woo', [
237
- (0, 14, 1, 0.001), (0, 3, 1, -0.0005), (0, 7, 1, -0.0005), (0, 17, 2, -0.0005)
238
- ]),
239
- ('wink', [
240
- (0, 11, 1, 0.001), (0, 13, 1, -0.0003), (0, 17, 0, 0.0003),
241
- (0, 17, 1, 0.0003), (0, 3, 1, -0.0003)
242
- ]),
243
- ('pupil_x', [
244
- (0, 11, 0, 0.0007 if params.get('pupil_x', 0) > 0 else 0.001),
245
- (0, 15, 0, 0.001 if params.get('pupil_x', 0) > 0 else 0.0007)
246
- ]),
247
- ('pupil_y', [
248
- (0, 11, 1, -0.001), (0, 15, 1, -0.001)
249
- ]),
250
- ('eyes', [
251
- (0, 11, 1, -0.001), (0, 13, 1, 0.0003), (0, 15, 1, -0.001), (0, 16, 1, 0.0003),
252
- (0, 1, 1, -0.00025), (0, 2, 1, 0.00025)
253
- ]),
254
- ('eyebrow', [
255
- (0, 1, 1, 0.001 if params.get('eyebrow', 0) > 0 else 0.0003),
256
- (0, 2, 1, -0.001 if params.get('eyebrow', 0) > 0 else -0.0003),
257
- (0, 1, 0, -0.001 if params.get('eyebrow', 0) <= 0 else 0),
258
- (0, 2, 0, 0.001 if params.get('eyebrow', 0) <= 0 else 0)
259
- ])
260
- ]
261
-
262
- for param_name, adjustments in modifications:
263
- param_value = params.get(param_name, 0)
264
- for i, j, k, factor in adjustments:
265
- x_d_new[i, j, k] += param_value * factor
266
-
267
- # Special case for pupil_y affecting eyes
268
- x_d_new[0, 11, 1] -= params.get('pupil_y', 0) * 0.001
269
- x_d_new[0, 15, 1] -= params.get('pupil_y', 0) * 0.001
270
- params['eyes'] = params.get('eyes', 0) - params.get('pupil_y', 0) / 2.
 
3
  import os
4
  import io
5
  import asyncio
6
+ from async_lru import alru_cache
7
  import base64
8
  from queue import Queue
9
  from typing import Dict, Any, List, Optional, Union
 
16
  from liveportrait.config.argument_config import ArgumentConfig
17
  from liveportrait.utils.camera import get_rotation_matrix
18
  from liveportrait.utils.io import resize_to_limit
19
+ from liveportrait.utils.crop import prepare_paste_back, paste_back, parse_bbox_from_landmark
20
 
21
  # Configure logging
22
  logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 
57
 
58
  self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
59
 
 
60
  self.processed_cache = {} # Stores the processed image data
61
 
62
  logger.info("βœ… FacePoke Engine initialized successfully.")
63
 
64
+ async def get_image_hash(self, image: Union[Image.Image, str, bytes]) -> str:
65
  """
66
  Compute or retrieve the hash for an image.
67
 
 
86
  else:
87
  raise ValueError("Unsupported image type")
88
 
89
+ @alru_cache(maxsize=512)
90
+ async def load_image(self, data):
91
+ image = Image.open(io.BytesIO(data))
 
 
 
 
 
 
 
 
 
 
92
 
93
+ image_hash = await self.get_image_hash(image)
 
94
 
 
95
  img_rgb = np.array(image)
96
 
97
  inference_cfg = self.live_portrait.live_portrait_wrapper.cfg
98
+ img_rgb = await asyncio.to_thread(resize_to_limit, img_rgb, inference_cfg.ref_max_shape, inference_cfg.ref_shape_n)
99
+ crop_info = await asyncio.to_thread(self.live_portrait.cropper.crop_single_image, img_rgb)
100
  img_crop_256x256 = crop_info['img_crop_256x256']
101
 
102
+ I_s = await asyncio.to_thread(self.live_portrait.live_portrait_wrapper.prepare_source, img_crop_256x256)
103
+ x_s_info = await asyncio.to_thread(self.live_portrait.live_portrait_wrapper.get_kp_info, I_s)
104
+ f_s = await asyncio.to_thread(self.live_portrait.live_portrait_wrapper.extract_feature_3d, I_s)
105
+ x_s = await asyncio.to_thread(self.live_portrait.live_portrait_wrapper.transform_keypoint, x_s_info)
106
 
107
  processed_data = {
108
  'img_rgb': img_rgb,
 
115
 
116
  self.processed_cache[image_hash] = processed_data
117
 
118
+ # Calculate the bounding box
119
+ bbox_info = parse_bbox_from_landmark(processed_data['crop_info']['lmk_crop'], scale=1.0)
120
 
121
+ return {
122
+ 'h': image_hash,
 
 
123
 
124
+ # those aren't easy to serialize
125
+ 'c': bbox_info['center'], # 2x1
126
+ 's': bbox_info['size'], # scalar
127
+ 'b': bbox_info['bbox'], # 4x2
128
+ 'a': bbox_info['angle'], # rad, counterclockwise
129
+ # 'bbox_rot': bbox_info['bbox_rot'].toList(), # 4x2
130
+ }
131
 
132
+ def transform_image(self, image_hash: str, params: Dict[str, float]) -> bytes:
133
+ # If we don't have the image in cache yet, add it
134
+ if image_hash not in self.processed_cache:
135
+ raise ValueError("cache miss")
136
 
137
+ processed_data = self.processed_cache[image_hash]
 
 
 
 
 
138
 
139
  try:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
140
  # Apply modifications based on params
141
  x_d_new = processed_data['x_s_info']['kp'].clone()
142
+
143
+ modifications = [
144
+ ('smile', [
145
+ (0, 20, 1, -0.01), (0, 14, 1, -0.02), (0, 17, 1, 0.0065), (0, 17, 2, 0.003),
146
+ (0, 13, 1, -0.00275), (0, 16, 1, -0.00275), (0, 3, 1, -0.0035), (0, 7, 1, -0.0035)
147
+ ]),
148
+ ('aaa', [
149
+ (0, 19, 1, 0.001), (0, 19, 2, 0.0001), (0, 17, 1, -0.0001)
150
+ ]),
151
+ ('eee', [
152
+ (0, 20, 2, -0.001), (0, 20, 1, -0.001), (0, 14, 1, -0.001)
153
+ ]),
154
+ ('woo', [
155
+ (0, 14, 1, 0.001), (0, 3, 1, -0.0005), (0, 7, 1, -0.0005), (0, 17, 2, -0.0005)
156
+ ]),
157
+ ('wink', [
158
+ (0, 11, 1, 0.001), (0, 13, 1, -0.0003), (0, 17, 0, 0.0003),
159
+ (0, 17, 1, 0.0003), (0, 3, 1, -0.0003)
160
+ ]),
161
+ ('pupil_x', [
162
+ (0, 11, 0, 0.0007 if params.get('pupil_x', 0) > 0 else 0.001),
163
+ (0, 15, 0, 0.001 if params.get('pupil_x', 0) > 0 else 0.0007)
164
+ ]),
165
+ ('pupil_y', [
166
+ (0, 11, 1, -0.001), (0, 15, 1, -0.001)
167
+ ]),
168
+ ('eyes', [
169
+ (0, 11, 1, -0.001), (0, 13, 1, 0.0003), (0, 15, 1, -0.001), (0, 16, 1, 0.0003),
170
+ (0, 1, 1, -0.00025), (0, 2, 1, 0.00025)
171
+ ]),
172
+ ('eyebrow', [
173
+ (0, 1, 1, 0.001 if params.get('eyebrow', 0) > 0 else 0.0003),
174
+ (0, 2, 1, -0.001 if params.get('eyebrow', 0) > 0 else -0.0003),
175
+ (0, 1, 0, -0.001 if params.get('eyebrow', 0) <= 0 else 0),
176
+ (0, 2, 0, 0.001 if params.get('eyebrow', 0) <= 0 else 0)
177
+ ])
178
+ ]
179
+
180
+ for param_name, adjustments in modifications:
181
+ param_value = params.get(param_name, 0)
182
+ for i, j, k, factor in adjustments:
183
+ x_d_new[i, j, k] += param_value * factor
184
+
185
+ # Special case for pupil_y affecting eyes
186
+ x_d_new[0, 11, 1] -= params.get('pupil_y', 0) * 0.001
187
+ x_d_new[0, 15, 1] -= params.get('pupil_y', 0) * 0.001
188
+ params['eyes'] = params.get('eyes', 0) - params.get('pupil_y', 0) / 2.
189
+
190
 
191
  # Apply rotation
192
  R_new = get_rotation_matrix(
 
197
  x_d_new = processed_data['x_s_info']['scale'] * (x_d_new @ R_new) + processed_data['x_s_info']['t']
198
 
199
  # Apply stitching
200
+ x_d_new = self.live_portrait.live_portrait_wrapper.stitching(processed_data['x_s'], x_d_new)
201
 
202
  # Generate the output
203
+ out = self.live_portrait.live_portrait_wrapper.warp_decode(processed_data['f_s'], processed_data['x_s'], x_d_new)
204
+ I_p = self.live_portrait.live_portrait_wrapper.parse_output(out['out'])
205
+
206
+ buffered = io.BytesIO()
207
 
208
+ ####################################################
209
+ # this part is about stitching the image back into the original.
210
+ #
211
+ # this is an expensive operation, not just because of the compute
212
+ # but because the payload will also be bigger (we send back the whole pic)
213
+ #
214
+ # I'm currently running some experiments to do it in the frontend
215
+ #
216
+ # --- old way: we do it in the server-side: ---
217
+ mask_ori = prepare_paste_back(
218
  processed_data['inference_cfg'].mask_crop, processed_data['crop_info']['M_c2o'],
219
  dsize=(processed_data['img_rgb'].shape[1], processed_data['img_rgb'].shape[0])
220
  )
221
+ I_p_to_ori_blend = paste_back(
222
+ I_p[0], processed_data['crop_info']['M_c2o'], processed_data['img_rgb'], mask_ori
 
223
  )
 
 
224
  result_image = Image.fromarray(I_p_to_ori_blend)
225
 
226
+ # --- maybe future way: do it in the frontend: ---
227
+ #result_image = Image.fromarray(I_p[0])
228
+ ####################################################
 
229
 
230
+ # write it into a webp
231
+ result_image.save(buffered, format="WebP", quality=82, lossless=False, method=6)
232
+
233
+ return buffered.getvalue()
234
 
235
  except Exception as e:
 
 
236
  raise ValueError(f"Failed to modify image: {str(e)}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
liveportrait/utils/crop.py CHANGED
@@ -409,4 +409,4 @@ def paste_back(image_to_processed, crop_M_c2o, rgb_ori, mask_ori):
409
  dsize = (rgb_ori.shape[1], rgb_ori.shape[0])
410
  result = _transform_img(image_to_processed, crop_M_c2o, dsize=dsize)
411
  result = np.clip(mask_ori * result + (1 - mask_ori) * rgb_ori, 0, 255).astype(np.uint8)
412
- return result
 
409
  dsize = (rgb_ori.shape[1], rgb_ori.shape[0])
410
  result = _transform_img(image_to_processed, crop_M_c2o, dsize=dsize)
411
  result = np.clip(mask_ori * result + (1 - mask_ori) * rgb_ori, 0, 255).astype(np.uint8)
412
+ return result
loader.py CHANGED
@@ -5,8 +5,6 @@ import asyncio
5
  import aiohttp
6
  import requests
7
  from huggingface_hub import hf_hub_download
8
- import sentencepiece
9
-
10
 
11
  # Configure logging
12
  logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
@@ -24,6 +22,7 @@ HF_REPO_ID = "jbilcke-hf/model-cocktail"
24
  MODEL_FILES = [
25
  "dwpose/dw-ll_ucoco_384.pth",
26
  "face-detector/s3fd-619a316812.pth",
 
27
  "liveportrait/spade_generator.pth",
28
  "liveportrait/warping_module.pth",
29
  "liveportrait/motion_extractor.pth",
@@ -31,6 +30,16 @@ MODEL_FILES = [
31
  "liveportrait/appearance_feature_extractor.pth",
32
  "liveportrait/landmark.onnx",
33
 
 
 
 
 
 
 
 
 
 
 
34
  # this is a hack, instead we should probably try to
35
  # fix liveportrait/utils/dependencies/insightface/utils/storage.py
36
  "insightface/models/buffalo_l.zip",
 
5
  import aiohttp
6
  import requests
7
  from huggingface_hub import hf_hub_download
 
 
8
 
9
  # Configure logging
10
  logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
 
22
  MODEL_FILES = [
23
  "dwpose/dw-ll_ucoco_384.pth",
24
  "face-detector/s3fd-619a316812.pth",
25
+
26
  "liveportrait/spade_generator.pth",
27
  "liveportrait/warping_module.pth",
28
  "liveportrait/motion_extractor.pth",
 
30
  "liveportrait/appearance_feature_extractor.pth",
31
  "liveportrait/landmark.onnx",
32
 
33
+ # For animal mode 🐢🐱
34
+ # however they say animal mode doesn't support stitching yet?
35
+ # https://github.com/KwaiVGI/LivePortrait/blob/main/assets/docs/changelog/2024-08-02.md#updates-on-animals-mode
36
+ #"liveportrait-animals/warping_module.pth",
37
+ #"liveportrait-animals/spade_generator.pth",
38
+ #"liveportrait-animals/motion_extractor.pth",
39
+ #"liveportrait-animals/appearance_feature_extractor.pth",
40
+ #"liveportrait-animals/stitching_retargeting_module.pth",
41
+ #"liveportrait-animals/xpose.pth",
42
+
43
  # this is a hack, instead we should probably try to
44
  # fix liveportrait/utils/dependencies/insightface/utils/storage.py
45
  "insightface/models/buffalo_l.zip",
public/index.js CHANGED
@@ -29660,148 +29660,7 @@ Yh.prototype.detectForVideo = Yh.prototype.F, Yh.prototype.detect = Yh.prototype
29660
  return Ka(Yh, t2, e2);
29661
  }, Yh.POSE_CONNECTIONS = Eh;
29662
 
29663
- // node_modules/uuid/dist/esm-browser/stringify.js
29664
- function unsafeStringify(arr, offset = 0) {
29665
- return (byteToHex[arr[offset + 0]] + byteToHex[arr[offset + 1]] + byteToHex[arr[offset + 2]] + byteToHex[arr[offset + 3]] + "-" + byteToHex[arr[offset + 4]] + byteToHex[arr[offset + 5]] + "-" + byteToHex[arr[offset + 6]] + byteToHex[arr[offset + 7]] + "-" + byteToHex[arr[offset + 8]] + byteToHex[arr[offset + 9]] + "-" + byteToHex[arr[offset + 10]] + byteToHex[arr[offset + 11]] + byteToHex[arr[offset + 12]] + byteToHex[arr[offset + 13]] + byteToHex[arr[offset + 14]] + byteToHex[arr[offset + 15]]).toLowerCase();
29666
- }
29667
- var byteToHex = [];
29668
- for (i2 = 0;i2 < 256; ++i2) {
29669
- byteToHex.push((i2 + 256).toString(16).slice(1));
29670
- }
29671
- var i2;
29672
-
29673
- // node_modules/uuid/dist/esm-browser/rng.js
29674
- var getRandomValues;
29675
- var rnds8 = new Uint8Array(16);
29676
- function rng() {
29677
- if (!getRandomValues) {
29678
- getRandomValues = typeof crypto !== "undefined" && crypto.getRandomValues && crypto.getRandomValues.bind(crypto);
29679
- if (!getRandomValues) {
29680
- throw new Error("crypto.getRandomValues() not supported. See https://github.com/uuidjs/uuid#getrandomvalues-not-supported");
29681
- }
29682
- }
29683
- return getRandomValues(rnds8);
29684
- }
29685
-
29686
- // node_modules/uuid/dist/esm-browser/native.js
29687
- var randomUUID = typeof crypto !== "undefined" && crypto.randomUUID && crypto.randomUUID.bind(crypto);
29688
- var native_default = {
29689
- randomUUID
29690
- };
29691
-
29692
- // node_modules/uuid/dist/esm-browser/v4.js
29693
- var v4 = function(options, buf, offset) {
29694
- if (native_default.randomUUID && !buf && !options) {
29695
- return native_default.randomUUID();
29696
- }
29697
- options = options || {};
29698
- var rnds = options.random || (options.rng || rng)();
29699
- rnds[6] = rnds[6] & 15 | 64;
29700
- rnds[8] = rnds[8] & 63 | 128;
29701
- if (buf) {
29702
- offset = offset || 0;
29703
- for (var i2 = 0;i2 < 16; ++i2) {
29704
- buf[offset + i2] = rnds[i2];
29705
- }
29706
- return buf;
29707
- }
29708
- return unsafeStringify(rnds);
29709
- };
29710
- var v4_default = v4;
29711
- // src/lib/circularBuffer.ts
29712
- class CircularBuffer {
29713
- capacity;
29714
- buffer;
29715
- pointer;
29716
- constructor(capacity) {
29717
- this.capacity = capacity;
29718
- this.buffer = new Array(capacity);
29719
- this.pointer = 0;
29720
- }
29721
- push(item) {
29722
- this.buffer[this.pointer] = item;
29723
- this.pointer = (this.pointer + 1) % this.capacity;
29724
- }
29725
- getAll() {
29726
- return this.buffer.filter((item) => item !== undefined);
29727
- }
29728
- }
29729
-
29730
- // node_modules/zustand/esm/vanilla.mjs
29731
- var createStoreImpl = (createState) => {
29732
- let state;
29733
- const listeners = new Set;
29734
- const setState = (partial, replace) => {
29735
- const nextState = typeof partial === "function" ? partial(state) : partial;
29736
- if (!Object.is(nextState, state)) {
29737
- const previousState = state;
29738
- state = (replace != null ? replace : typeof nextState !== "object" || nextState === null) ? nextState : Object.assign({}, state, nextState);
29739
- listeners.forEach((listener) => listener(state, previousState));
29740
- }
29741
- };
29742
- const getState = () => state;
29743
- const getInitialState = () => initialState;
29744
- const subscribe = (listener) => {
29745
- listeners.add(listener);
29746
- return () => listeners.delete(listener);
29747
- };
29748
- const api = { setState, getState, getInitialState, subscribe };
29749
- const initialState = state = createState(setState, getState, api);
29750
- return api;
29751
- };
29752
- var createStore = (createState) => createState ? createStoreImpl(createState) : createStoreImpl;
29753
-
29754
- // node_modules/zustand/esm/react.mjs
29755
- var import_react3 = __toESM(require_react(), 1);
29756
- var useStore = function(api, selector = identity) {
29757
- const slice = import_react3.default.useSyncExternalStore(api.subscribe, () => selector(api.getState()), () => selector(api.getInitialState()));
29758
- import_react3.default.useDebugValue(slice);
29759
- return slice;
29760
- };
29761
- var identity = (arg) => arg;
29762
- var createImpl = (createState) => {
29763
- const api = createStore(createState);
29764
- const useBoundStore = (selector) => useStore(api, selector);
29765
- Object.assign(useBoundStore, api);
29766
- return useBoundStore;
29767
- };
29768
- var create = (createState) => createState ? createImpl(createState) : createImpl;
29769
-
29770
- // src/hooks/useMainStore.ts
29771
- var useMainStore = create((set, get) => ({
29772
- error: "",
29773
- imageFile: null,
29774
- originalImage: "",
29775
- originalImageHash: "",
29776
- previewImage: "",
29777
- minLatency: 20,
29778
- averageLatency: 190,
29779
- maxLatency: 4000,
29780
- activeLandmark: undefined,
29781
- params: {},
29782
- setError: (error = "") => set({ error }),
29783
- setImageFile: (file) => set({ imageFile: file }),
29784
- setOriginalImage: (url) => set({ originalImage: url }),
29785
- setOriginalImageHash: (originalImageHash) => set({ originalImageHash }),
29786
- setPreviewImage: (url) => set({ previewImage: url }),
29787
- resetImage: () => {
29788
- const { originalImage } = get();
29789
- if (originalImage) {
29790
- set({ previewImage: originalImage });
29791
- }
29792
- },
29793
- setAverageLatency: (averageLatency) => set({ averageLatency }),
29794
- setActiveLandmark: (activeLandmark) => set({ activeLandmark }),
29795
- setParams: (params) => {
29796
- const { params: previousParams } = get();
29797
- set({ params: {
29798
- ...previousParams,
29799
- ...params
29800
- } });
29801
- }
29802
- }));
29803
-
29804
- // src/lib/facePoke.ts
29805
  var WebSocketState;
29806
  (function(WebSocketState2) {
29807
  WebSocketState2[WebSocketState2["CONNECTING"] = 0] = "CONNECTING";
@@ -29810,63 +29669,24 @@ var WebSocketState;
29810
  WebSocketState2[WebSocketState2["CLOSED"] = 3] = "CLOSED";
29811
  })(WebSocketState || (WebSocketState = {}));
29812
 
 
29813
  class FacePoke {
29814
  ws = null;
29815
- connectionId = v4_default();
29816
  isUnloading = false;
29817
- onModifiedImage = () => {
29818
  };
29819
  reconnectAttempts = 0;
29820
  maxReconnectAttempts = 5;
29821
  reconnectDelay = 5000;
29822
  eventListeners = new Map;
29823
- requestTracker = new Map;
29824
- responseTimeBuffer;
29825
- MAX_TRACKED_TIMES = 5;
29826
  constructor() {
29827
- console.log(`[FacePoke] Initializing FacePoke instance with connection ID: ${this.connectionId}`);
29828
  this.initializeWebSocket();
29829
  this.setupUnloadHandler();
29830
- this.responseTimeBuffer = new CircularBuffer(this.MAX_TRACKED_TIMES);
29831
- console.log(`[FacePoke] Initialized response time tracker with capacity: ${this.MAX_TRACKED_TIMES}`);
29832
- }
29833
- trackRequest() {
29834
- const uuid = v4_default();
29835
- this.requestTracker.set(uuid, { uuid, timestamp: Date.now() });
29836
- return uuid;
29837
- }
29838
- completeRequest(uuid) {
29839
- const request = this.requestTracker.get(uuid);
29840
- if (request) {
29841
- const responseTime = Date.now() - request.timestamp;
29842
- this.responseTimeBuffer.push(responseTime);
29843
- this.requestTracker.delete(uuid);
29844
- this.updateThrottleTime();
29845
- console.log(`[FacePoke] Completed request ${uuid}. Response time: ${responseTime}ms`);
29846
- } else {
29847
- console.warn(`[FacePoke] Attempted to complete unknown request: ${uuid}`);
29848
- }
29849
- }
29850
- calculateAverageResponseTime() {
29851
- const times = this.responseTimeBuffer.getAll();
29852
- const averageLatency = useMainStore.getState().averageLatency;
29853
- if (times.length === 0)
29854
- return averageLatency;
29855
- const sum = times.reduce((acc, time) => acc + time, 0);
29856
- return sum / times.length;
29857
- }
29858
- updateThrottleTime() {
29859
- const { minLatency, maxLatency, averageLatency, setAverageLatency } = useMainStore.getState();
29860
- const avgResponseTime = this.calculateAverageResponseTime();
29861
- const newLatency = Math.min(minLatency, Math.max(minLatency, avgResponseTime));
29862
- if (newLatency !== averageLatency) {
29863
- setAverageLatency(newLatency);
29864
- console.log(`[FacePoke] Updated throttle time (latency is ${newLatency}ms)`);
29865
- }
29866
  }
29867
- setOnModifiedImage(handler) {
29868
- this.onModifiedImage = handler;
29869
- console.log(`[FacePoke] onModifiedImage handler set`);
29870
  }
29871
  async startWebSocket() {
29872
  console.log(`[FacePoke] Starting WebSocket connection.`);
@@ -29875,56 +29695,47 @@ class FacePoke {
29875
  }
29876
  }
29877
  async initializeWebSocket() {
29878
- console.log(`[FacePoke][${this.connectionId}] Initializing WebSocket connection`);
29879
  const connect = () => {
29880
  this.ws = new WebSocket(`wss://${window.location.host}/ws`);
29881
  this.ws.onopen = this.handleWebSocketOpen.bind(this);
29882
- this.ws.onmessage = this.handleWebSocketMessage.bind(this);
29883
  this.ws.onclose = this.handleWebSocketClose.bind(this);
29884
  this.ws.onerror = this.handleWebSocketError.bind(this);
 
29885
  };
29886
  connect();
29887
  }
 
 
 
 
 
 
 
29888
  handleWebSocketOpen() {
29889
- console.log(`[FacePoke][${this.connectionId}] WebSocket connection opened`);
29890
  this.reconnectAttempts = 0;
29891
  this.emitEvent("websocketOpen");
29892
  }
29893
- handleWebSocketMessage(event) {
29894
- try {
29895
- const data = JSON.parse(event.data);
29896
- if (data.uuid) {
29897
- this.completeRequest(data.uuid);
29898
- }
29899
- if (data.type === "modified_image") {
29900
- if (data?.image) {
29901
- this.onModifiedImage(data.image, data.image_hash);
29902
- }
29903
- }
29904
- this.emitEvent("message", data);
29905
- } catch (error) {
29906
- console.error(`[FacePoke][${this.connectionId}] Error parsing WebSocket message:`, error);
29907
- }
29908
- }
29909
  handleWebSocketClose(event) {
29910
  if (event.wasClean) {
29911
- console.log(`[FacePoke][${this.connectionId}] WebSocket connection closed cleanly, code=${event.code}, reason=${event.reason}`);
29912
  } else {
29913
- console.warn(`[FacePoke][${this.connectionId}] WebSocket connection abruptly closed`);
29914
  }
29915
  this.emitEvent("websocketClose", event);
29916
  if (!this.isUnloading && this.reconnectAttempts < this.maxReconnectAttempts) {
29917
  this.reconnectAttempts++;
29918
  const delay = Math.min(1000 * 2 ** this.reconnectAttempts, 30000);
29919
- console.log(`[FacePoke][${this.connectionId}] Attempting to reconnect in ${delay}ms (Attempt ${this.reconnectAttempts}/${this.maxReconnectAttempts})...`);
29920
  setTimeout(() => this.initializeWebSocket(), delay);
29921
  } else if (this.reconnectAttempts >= this.maxReconnectAttempts) {
29922
- console.error(`[FacePoke][${this.connectionId}] Max reconnect attempts reached. Please refresh the page.`);
29923
  this.emitEvent("maxReconnectAttemptsReached");
29924
  }
29925
  }
29926
  handleWebSocketError(error) {
29927
- console.error(`[FacePoke][${this.connectionId}] WebSocket error:`, error);
29928
  this.emitEvent("websocketError", error);
29929
  }
29930
  cleanup() {
@@ -29937,22 +29748,26 @@ class FacePoke {
29937
  console.log("[FacePoke] Cleanup completed");
29938
  this.emitEvent("cleanup");
29939
  }
29940
- modifyImage(image, imageHash, params) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
29941
  try {
29942
- const message = {
29943
- type: "modify_image",
29944
- params
29945
- };
29946
- if (image) {
29947
- message.image = image;
29948
- } else if (imageHash) {
29949
- message.image_hash = imageHash;
29950
- } else {
29951
- throw new Error("Either image or imageHash must be provided");
29952
- }
29953
- this.sendJsonMessage(message);
29954
  } catch (err) {
29955
- console.error(`[FacePoke] Failed to modify the image:`, err);
29956
  }
29957
  }
29958
  sendJsonMessage(message) {
@@ -29962,9 +29777,11 @@ class FacePoke {
29962
  this.emitEvent("sendJsonMessageError", error);
29963
  throw error;
29964
  }
29965
- const uuid = this.trackRequest();
29966
- const messageWithUuid = { ...message, uuid };
29967
- this.ws.send(JSON.stringify(messageWithUuid));
 
 
29968
  }
29969
  setupUnloadHandler() {
29970
  window.addEventListener("beforeunload", () => {
@@ -30000,6 +29817,337 @@ class FacePoke {
30000
  }
30001
  var facePoke = new FacePoke;
30002
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
30003
  // node_modules/beautiful-react-hooks/esm/useThrottledCallback.js
30004
  var import_react6 = __toESM(require_react(), 1);
30005
  var import_lodash = __toESM(require_lodash(), 1);
@@ -32818,34 +32966,20 @@ var FACEMESH_TESSELATION = Object.freeze(new Set([
32818
 
32819
  // src/hooks/useFaceLandmarkDetection.tsx
32820
  function useFaceLandmarkDetection() {
32821
- const error = useMainStore((s2) => s2.error);
32822
  const setError = useMainStore((s2) => s2.setError);
32823
- const imageFile = useMainStore((s2) => s2.imageFile);
32824
- const setImageFile = useMainStore((s2) => s2.setImageFile);
32825
- const originalImage = useMainStore((s2) => s2.originalImage);
32826
- const originalImageHash = useMainStore((s2) => s2.originalImageHash);
32827
- const setOriginalImageHash = useMainStore((s2) => s2.setOriginalImageHash);
32828
  const previewImage = useMainStore((s2) => s2.previewImage);
32829
- const setPreviewImage = useMainStore((s2) => s2.setPreviewImage);
32830
- const resetImage = useMainStore((s2) => s2.resetImage);
32831
- window.debugJuju = useMainStore;
32832
- const averageLatency = 220;
32833
- const [faceLandmarks, setFaceLandmarks] = import_react7.useState([]);
32834
  const [isMediaPipeReady, setIsMediaPipeReady] = import_react7.useState(false);
32835
  const [isDrawingUtilsReady, setIsDrawingUtilsReady] = import_react7.useState(false);
32836
- const [blendShapes, setBlendShapes] = import_react7.useState([]);
32837
  const [dragStart, setDragStart] = import_react7.useState(null);
32838
- const [dragEnd, setDragEnd] = import_react7.useState(null);
32839
  const [isDragging, setIsDragging] = import_react7.useState(false);
32840
- const [isWaitingForResponse, setIsWaitingForResponse] = import_react7.useState(false);
32841
  const dragStartRef = import_react7.useRef(null);
32842
- const currentMousePosRef = import_react7.useRef(null);
32843
- const lastModifiedImageHashRef = import_react7.useRef(null);
32844
  const [currentLandmark, setCurrentLandmark] = import_react7.useState(null);
32845
  const [previousLandmark, setPreviousLandmark] = import_react7.useState(null);
32846
  const [currentOpacity, setCurrentOpacity] = import_react7.useState(0);
32847
  const [previousOpacity, setPreviousOpacity] = import_react7.useState(0);
32848
- const [isHovering, setIsHovering] = import_react7.useState(false);
32849
  const canvasRef = import_react7.useRef(null);
32850
  const mediaPipeRef = import_react7.useRef({
32851
  faceLandmarker: null,
@@ -32882,8 +33016,8 @@ function useFaceLandmarkDetection() {
32882
  } else {
32883
  faceLandmarker.close();
32884
  }
32885
- } catch (error2) {
32886
- console.error("Error during MediaPipe initialization:", error2);
32887
  setError("Failed to initialize face detection. Please try refreshing the page.");
32888
  }
32889
  };
@@ -32969,6 +33103,7 @@ function useFaceLandmarkDetection() {
32969
  }
32970
  }, [landmarkCenters]);
32971
  const detectFaceLandmarks = import_react7.useCallback(async (imageDataUrl) => {
 
32972
  if (!isMediaPipeReady) {
32973
  console.log("MediaPipe not ready. Skipping detection.");
32974
  return;
@@ -33068,200 +33203,137 @@ function useFaceLandmarkDetection() {
33068
  }
33069
  detectFaceLandmarks(previewImage);
33070
  }, [isMediaPipeReady, isDrawingUtilsReady, previewImage]);
33071
- const modifyImage = import_react7.useCallback(({ landmark, vector }) => {
33072
- const {
33073
- originalImage: originalImage2,
33074
- originalImageHash: originalImageHash2,
33075
- params: previousParams,
33076
- setParams,
33077
- setError: setError2
33078
- } = useMainStore.getState();
33079
- if (!originalImage2) {
33080
- console.error("Image file or facePoke not available");
33081
- return;
33082
- }
33083
- const params = {
33084
- ...previousParams
33085
- };
33086
- const minX = -0.5;
33087
- const maxX = 0.5;
33088
- const minY = -0.5;
33089
- const maxY = 0.5;
33090
- const mapRange = (value, inMin, inMax, outMin, outMax) => {
33091
- return Math.min(outMax, Math.max(outMin, (value - inMin) * (outMax - outMin) / (inMax - inMin) + outMin));
33092
- };
33093
- console.log("modifyImage:", {
33094
- originalImage: originalImage2,
33095
- originalImageHash: originalImageHash2,
33096
- landmark,
33097
- vector,
33098
- minX,
33099
- maxX,
33100
- minY,
33101
- maxY
33102
- });
33103
- switch (landmark.group) {
33104
- case "leftEye":
33105
- case "rightEye":
33106
- const eyesMin = -20;
33107
- const eyesMax = 5;
33108
- params.eyes = mapRange(-vector.y, minX, maxX, eyesMin, eyesMax);
33109
- break;
33110
- case "leftEyebrow":
33111
- case "rightEyebrow":
33112
- const eyebrowMin = -10;
33113
- const eyebrowMax = 15;
33114
- params.eyebrow = mapRange(-vector.y, minY, maxY, eyebrowMin, eyebrowMax);
33115
- break;
33116
- case "lips":
33117
- const eeeMin = -20;
33118
- const eeeMax = 15;
33119
- params.eee = mapRange(-vector.y, minY, maxY, eeeMin, eeeMax);
33120
- const wooMin = -20;
33121
- const wooMax = 15;
33122
- params.woo = mapRange(-vector.x, minX, maxX, wooMin, wooMax);
33123
- break;
33124
- case "faceOval":
33125
- const rollMin = -40;
33126
- const rollMax = 40;
33127
- params.rotate_roll = mapRange(vector.x, minX, maxX, rollMin, rollMax);
33128
- break;
33129
- case "background":
33130
- const yawMin = -40;
33131
- const yawMax = 40;
33132
- params.rotate_yaw = mapRange(-vector.x, minX, maxX, yawMin, yawMax);
33133
- const pitchMin = -40;
33134
- const pitchMax = 40;
33135
- params.rotate_pitch = mapRange(vector.y, minY, maxY, pitchMin, pitchMax);
33136
- break;
33137
- default:
33138
- return;
33139
- }
33140
- for (const [key, value] of Object.entries(params)) {
33141
- if (isNaN(value) || !isFinite(value)) {
33142
- console.log(`${key} is NaN, aborting`);
33143
- return;
33144
- }
33145
- }
33146
- console.log(`PITCH=${params.rotate_pitch || 0}, YAW=${params.rotate_yaw || 0}, ROLL=${params.rotate_roll || 0}`);
33147
- setParams(params);
33148
- try {
33149
- if (!lastModifiedImageHashRef.current || lastModifiedImageHashRef.current !== originalImageHash2) {
33150
- lastModifiedImageHashRef.current = originalImageHash2;
33151
- facePoke.modifyImage(originalImage2, null, params);
33152
- } else {
33153
- facePoke.modifyImage(null, lastModifiedImageHashRef.current, params);
33154
- }
33155
- } catch (error2) {
33156
- setError2("Failed to modify image");
33157
- }
33158
- }, []);
33159
  const modifyImageWithRateLimit = useThrottledCallback_default((params) => {
33160
- modifyImage(params);
33161
- }, [modifyImage], averageLatency);
33162
- const handleMouseEnter = import_react7.useCallback(() => {
33163
- setIsHovering(true);
33164
- }, []);
33165
- const handleMouseLeave = import_react7.useCallback(() => {
33166
- setIsHovering(false);
33167
- }, []);
33168
- const handleMouseDown = import_react7.useCallback((event) => {
33169
  if (!canvasRef.current)
33170
  return;
33171
  const rect = canvasRef.current.getBoundingClientRect();
33172
- const x2 = (event.clientX - rect.left) / rect.width;
33173
- const y2 = (event.clientY - rect.top) / rect.height;
33174
- const landmark = findClosestLandmark(x2, y2);
33175
- console.log(`Mouse down on ${landmark.group}`);
33176
  setActiveLandmark(landmark);
33177
- setDragStart({ x: x2, y: y2 });
33178
- dragStartRef.current = { x: x2, y: y2 };
33179
  }, [findClosestLandmark, setActiveLandmark, setDragStart]);
33180
- const handleMouseMove = import_react7.useCallback((event) => {
33181
  if (!canvasRef.current)
33182
  return;
33183
  const rect = canvasRef.current.getBoundingClientRect();
33184
- const x2 = (event.clientX - rect.left) / rect.width;
33185
- const y2 = (event.clientY - rect.top) / rect.height;
 
 
 
 
 
 
 
 
 
 
 
33186
  if (dragStart && dragStartRef.current) {
33187
- const landmark = findClosestLandmark(x2, y2, currentLandmark?.group);
33188
- console.log(`Dragging mouse (was over ${currentLandmark?.group || "nothing"}, now over ${landmark.group})`);
33189
  modifyImageWithRateLimit({
33190
  landmark: currentLandmark || landmark,
33191
- vector: {
33192
- x: x2 - landmarkCenters[landmark.group].x,
33193
- y: y2 - landmarkCenters[landmark.group].y,
33194
- z: 0
33195
- }
33196
  });
33197
- setIsDragging(true);
33198
  } else {
33199
- const landmark = findClosestLandmark(x2, y2);
33200
  if (!currentLandmark || currentLandmark?.group !== landmark?.group) {
33201
  setActiveLandmark(landmark);
33202
  }
33203
- setIsHovering(true);
 
 
 
 
33204
  }
33205
- }, [currentLandmark, dragStart, setIsHovering, setActiveLandmark, setIsDragging, modifyImageWithRateLimit, landmarkCenters]);
33206
- const handleMouseUp = import_react7.useCallback((event) => {
33207
  if (!canvasRef.current)
33208
  return;
33209
  const rect = canvasRef.current.getBoundingClientRect();
33210
- const x2 = (event.clientX - rect.left) / rect.width;
33211
- const y2 = (event.clientY - rect.top) / rect.height;
33212
  if (dragStart && dragStartRef.current) {
33213
- const landmark = findClosestLandmark(x2, y2, currentLandmark?.group);
33214
- console.log(`Mouse up (was over ${currentLandmark?.group || "nothing"}, now over ${landmark.group})`);
33215
  modifyImageWithRateLimit({
33216
  landmark: currentLandmark || landmark,
33217
  vector: {
33218
- x: x2 - landmarkCenters[landmark.group].x,
33219
- y: y2 - landmarkCenters[landmark.group].y,
33220
  z: 0
33221
- }
 
33222
  });
33223
  }
33224
  setIsDragging(false);
33225
  dragStartRef.current = null;
33226
  setActiveLandmark(undefined);
33227
- }, [currentLandmark, isDragging, modifyImageWithRateLimit, findClosestLandmark, setActiveLandmark, landmarkCenters, modifyImageWithRateLimit, setIsDragging]);
33228
- import_react7.useEffect(() => {
33229
- facePoke.setOnModifiedImage((image, image_hash) => {
33230
- if (image) {
33231
- setPreviewImage(image);
33232
- }
33233
- setOriginalImageHash(image_hash);
33234
- lastModifiedImageHashRef.current = image_hash;
33235
- });
33236
- }, [setPreviewImage, setOriginalImageHash]);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
33237
  return {
33238
  canvasRef,
33239
  canvasRefCallback,
33240
  mediaPipeRef,
33241
- faceLandmarks,
33242
  isMediaPipeReady,
33243
  isDrawingUtilsReady,
33244
- blendShapes,
33245
- setFaceLandmarks,
33246
- setBlendShapes,
33247
  handleMouseDown,
33248
  handleMouseUp,
33249
  handleMouseMove,
33250
- handleMouseEnter,
33251
- handleMouseLeave,
 
33252
  currentLandmark,
33253
  currentOpacity
33254
  };
33255
  }
33256
 
33257
- // src/components/PoweredBy.tsx
33258
  var jsx_dev_runtime2 = __toESM(require_jsx_dev_runtime(), 1);
33259
- function PoweredBy() {
33260
  return jsx_dev_runtime2.jsxDEV("div", {
33261
  className: "flex flex-row items-center justify-center font-sans mt-4 w-full",
33262
  children: [
33263
  jsx_dev_runtime2.jsxDEV("span", {
33264
- className: "mr-1",
 
 
 
 
 
33265
  children: jsx_dev_runtime2.jsxDEV("img", {
33266
  src: "/hf-logo.svg",
33267
  alt: "Hugging Face",
@@ -33297,7 +33369,6 @@ function Spinner() {
33297
  // src/hooks/useFacePokeAPI.ts
33298
  var import_react8 = __toESM(require_react(), 1);
33299
  function useFacePokeAPI() {
33300
- const [status, setStatus] = import_react8.useState("");
33301
  const [isDebugMode, setIsDebugMode] = import_react8.useState(false);
33302
  const [interruptMessage, setInterruptMessage] = import_react8.useState(null);
33303
  const [isLoading, setIsLoading] = import_react8.useState(false);
@@ -33316,8 +33387,6 @@ function useFacePokeAPI() {
33316
  }, []);
33317
  return {
33318
  facePoke,
33319
- status,
33320
- setStatus,
33321
  isDebugMode,
33322
  setIsDebugMode,
33323
  interruptMessage,
@@ -33335,31 +33404,13 @@ function Layout({ children }) {
33335
  children: jsx_dev_runtime4.jsxDEV("div", {
33336
  className: "min-h-screen w-full py-8 flex flex-col justify-center",
33337
  children: jsx_dev_runtime4.jsxDEV("div", {
33338
- className: "relative p-4 sm:max-w-5xl sm:mx-auto",
33339
  children
33340
  }, undefined, false, undefined, this)
33341
  }, undefined, false, undefined, this)
33342
  }, undefined, false, undefined, this);
33343
  }
33344
 
33345
- // src/lib/convertImageToBase64.ts
33346
- async function convertImageToBase64(imageFile) {
33347
- return new Promise((resolve, reject) => {
33348
- const reader = new FileReader;
33349
- reader.onload = () => {
33350
- if (typeof reader.result === "string") {
33351
- resolve(reader.result);
33352
- } else {
33353
- reject(new Error("Failed to convert image to base64"));
33354
- }
33355
- };
33356
- reader.onerror = () => {
33357
- reject(new Error("Error reading file"));
33358
- };
33359
- reader.readAsDataURL(imageFile);
33360
- });
33361
- }
33362
-
33363
  // src/app.tsx
33364
  var jsx_dev_runtime5 = __toESM(require_jsx_dev_runtime(), 1);
33365
  function App() {
@@ -33367,66 +33418,34 @@ function App() {
33367
  const setError = useMainStore((s2) => s2.setError);
33368
  const imageFile = useMainStore((s2) => s2.imageFile);
33369
  const setImageFile = useMainStore((s2) => s2.setImageFile);
33370
- const originalImage = useMainStore((s2) => s2.originalImage);
33371
- const setOriginalImage = useMainStore((s2) => s2.setOriginalImage);
 
 
33372
  const previewImage = useMainStore((s2) => s2.previewImage);
33373
- const setPreviewImage = useMainStore((s2) => s2.setPreviewImage);
33374
- const resetImage = useMainStore((s2) => s2.resetImage);
33375
- const setOriginalImageHash = useMainStore((s2) => s2.setOriginalImageHash);
33376
  const {
33377
- status,
33378
- setStatus,
33379
  isDebugMode,
33380
  setIsDebugMode,
33381
  interruptMessage
33382
  } = useFacePokeAPI();
33383
  const {
33384
- canvasRef,
33385
  canvasRefCallback,
33386
- mediaPipeRef,
33387
- faceLandmarks,
33388
  isMediaPipeReady,
33389
- blendShapes,
33390
- setFaceLandmarks,
33391
- setBlendShapes,
33392
  handleMouseDown,
33393
  handleMouseUp,
33394
  handleMouseMove,
33395
- handleMouseEnter,
33396
- handleMouseLeave,
 
33397
  currentOpacity
33398
  } = useFaceLandmarkDetection();
33399
  const videoRef = import_react9.useRef(null);
33400
- const handleFileChange = import_react9.useCallback(async (event) => {
33401
  const files = event.target.files;
33402
- if (files && files[0]) {
33403
- setImageFile(files[0]);
33404
- setStatus(`File selected: ${truncateFileName(files[0].name, 16)}`);
33405
- try {
33406
- const image = await convertImageToBase64(files[0]);
33407
- setPreviewImage(image);
33408
- setOriginalImage(image);
33409
- setOriginalImageHash("");
33410
- } catch (err) {
33411
- console.log(`failed to convert the image: `, err);
33412
- setImageFile(null);
33413
- setStatus("");
33414
- setPreviewImage("");
33415
- setOriginalImage("");
33416
- setOriginalImageHash("");
33417
- setFaceLandmarks([]);
33418
- setBlendShapes([]);
33419
- }
33420
- } else {
33421
- setImageFile(null);
33422
- setStatus("");
33423
- setPreviewImage("");
33424
- setOriginalImage("");
33425
- setOriginalImageHash("");
33426
- setFaceLandmarks([]);
33427
- setBlendShapes([]);
33428
- }
33429
- }, [isMediaPipeReady, setImageFile, setPreviewImage, setOriginalImage, setOriginalImageHash, setFaceLandmarks, setBlendShapes, setStatus]);
33430
  const handleDownload = import_react9.useCallback(() => {
33431
  if (previewImage) {
33432
  const link = document.createElement("a");
@@ -33503,7 +33522,7 @@ function App() {
33503
  className: "flex items-center space-x-2",
33504
  children: [
33505
  jsx_dev_runtime5.jsxDEV("div", {
33506
- className: "relative",
33507
  children: [
33508
  jsx_dev_runtime5.jsxDEV("input", {
33509
  id: "imageInput",
@@ -33518,7 +33537,7 @@ function App() {
33518
  className: `cursor-pointer inline-flex items-center px-3 h-10 border border-transparent text-sm font-medium rounded-md text-white ${isMediaPipeReady ? "bg-slate-600 hover:bg-slate-500" : "bg-slate-500 cursor-not-allowed"} focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-slate-500 shadow-xl`,
33519
  children: [
33520
  jsx_dev_runtime5.jsxDEV(Spinner, {}, undefined, false, undefined, this),
33521
- imageFile ? truncateFileName(imageFile.name, 32) : isMediaPipeReady ? "Choose a portrait photo (.jpg, .png, .webp)" : "Initializing..."
33522
  ]
33523
  }, undefined, true, undefined, this)
33524
  ]
@@ -33535,18 +33554,21 @@ function App() {
33535
  }, undefined, true, undefined, this)
33536
  ]
33537
  }, undefined, true, undefined, this),
33538
- previewImage && jsx_dev_runtime5.jsxDEV("label", {
33539
- className: "mt-4 flex items-center",
33540
- children: [
33541
- jsx_dev_runtime5.jsxDEV("input", {
33542
- type: "checkbox",
33543
- checked: isDebugMode,
33544
- onChange: (e2) => setIsDebugMode(e2.target.checked),
33545
- className: "mr-2"
33546
- }, undefined, false, undefined, this),
33547
- "Show face landmarks on hover"
33548
- ]
33549
- }, undefined, true, undefined, this)
 
 
 
33550
  ]
33551
  }, undefined, true, undefined, this),
33552
  previewImage && jsx_dev_runtime5.jsxDEV("div", {
@@ -33560,11 +33582,12 @@ function App() {
33560
  jsx_dev_runtime5.jsxDEV("canvas", {
33561
  ref: canvasRefCallback,
33562
  className: "absolute top-0 left-0 w-full h-full select-none",
33563
- onMouseEnter: handleMouseEnter,
33564
- onMouseLeave: handleMouseLeave,
33565
  onMouseDown: handleMouseDown,
33566
  onMouseUp: handleMouseUp,
33567
  onMouseMove: handleMouseMove,
 
 
 
33568
  style: {
33569
  position: "absolute",
33570
  top: 0,
@@ -33580,7 +33603,7 @@ function App() {
33580
  canDisplayBlendShapes && displayBlendShapes
33581
  ]
33582
  }, undefined, true, undefined, this),
33583
- jsx_dev_runtime5.jsxDEV(PoweredBy, {}, undefined, false, undefined, this)
33584
  ]
33585
  }, undefined, true, undefined, this);
33586
  }
 
29660
  return Ka(Yh, t2, e2);
29661
  }, Yh.POSE_CONNECTIONS = Eh;
29662
 
29663
+ // src/types.ts
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
29664
  var WebSocketState;
29665
  (function(WebSocketState2) {
29666
  WebSocketState2[WebSocketState2["CONNECTING"] = 0] = "CONNECTING";
 
29669
  WebSocketState2[WebSocketState2["CLOSED"] = 3] = "CLOSED";
29670
  })(WebSocketState || (WebSocketState = {}));
29671
 
29672
+ // src/lib/facePoke.ts
29673
  class FacePoke {
29674
  ws = null;
 
29675
  isUnloading = false;
29676
+ onServerResponse = async () => {
29677
  };
29678
  reconnectAttempts = 0;
29679
  maxReconnectAttempts = 5;
29680
  reconnectDelay = 5000;
29681
  eventListeners = new Map;
 
 
 
29682
  constructor() {
29683
+ console.log(`[FacePoke] Initializing FacePoke instance`);
29684
  this.initializeWebSocket();
29685
  this.setupUnloadHandler();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
29686
  }
29687
+ setOnServerResponse(handler) {
29688
+ this.onServerResponse = handler;
29689
+ console.log(`[FacePoke] onServerResponse handler set`);
29690
  }
29691
  async startWebSocket() {
29692
  console.log(`[FacePoke] Starting WebSocket connection.`);
 
29695
  }
29696
  }
29697
  async initializeWebSocket() {
29698
+ console.log(`[FacePoke] Initializing WebSocket connection`);
29699
  const connect = () => {
29700
  this.ws = new WebSocket(`wss://${window.location.host}/ws`);
29701
  this.ws.onopen = this.handleWebSocketOpen.bind(this);
 
29702
  this.ws.onclose = this.handleWebSocketClose.bind(this);
29703
  this.ws.onerror = this.handleWebSocketError.bind(this);
29704
+ this.ws.onmessage = this.handleWebSocketMessage.bind(this);
29705
  };
29706
  connect();
29707
  }
29708
+ handleWebSocketMessage(msg) {
29709
+ if (typeof msg.data === "string") {
29710
+ this.onServerResponse({ loaded: JSON.parse(msg.data) });
29711
+ } else if (typeof msg.data !== "undefined") {
29712
+ this.onServerResponse({ image: msg.data });
29713
+ }
29714
+ }
29715
  handleWebSocketOpen() {
29716
+ console.log(`[FacePoke] WebSocket connection opened`);
29717
  this.reconnectAttempts = 0;
29718
  this.emitEvent("websocketOpen");
29719
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
29720
  handleWebSocketClose(event) {
29721
  if (event.wasClean) {
29722
+ console.log(`[FacePoke] WebSocket connection closed cleanly, code=${event.code}, reason=${event.reason}`);
29723
  } else {
29724
+ console.warn(`[FacePoke] WebSocket connection abruptly closed`);
29725
  }
29726
  this.emitEvent("websocketClose", event);
29727
  if (!this.isUnloading && this.reconnectAttempts < this.maxReconnectAttempts) {
29728
  this.reconnectAttempts++;
29729
  const delay = Math.min(1000 * 2 ** this.reconnectAttempts, 30000);
29730
+ console.log(`[FacePoke] Attempting to reconnect in ${delay}ms (Attempt ${this.reconnectAttempts}/${this.maxReconnectAttempts})...`);
29731
  setTimeout(() => this.initializeWebSocket(), delay);
29732
  } else if (this.reconnectAttempts >= this.maxReconnectAttempts) {
29733
+ console.error(`[FacePoke] Max reconnect attempts reached. Please refresh the page.`);
29734
  this.emitEvent("maxReconnectAttemptsReached");
29735
  }
29736
  }
29737
  handleWebSocketError(error) {
29738
+ console.error(`[FacePoke] WebSocket error:`, error);
29739
  this.emitEvent("websocketError", error);
29740
  }
29741
  cleanup() {
 
29748
  console.log("[FacePoke] Cleanup completed");
29749
  this.emitEvent("cleanup");
29750
  }
29751
+ async loadImage(image) {
29752
+ const base64Data = image.split(",")[1] || image;
29753
+ const buffer = new Uint8Array(atob(base64Data).split("").map((char) => char.charCodeAt(0)));
29754
+ const blob = new Blob([buffer], { type: "application/octet-binary" });
29755
+ this.sendBlobMessage(await blob.arrayBuffer());
29756
+ }
29757
+ transformImage(hash, params) {
29758
+ this.sendJsonMessage({ hash, params });
29759
+ }
29760
+ sendBlobMessage(buffer) {
29761
+ if (!this.ws || this.ws.readyState !== WebSocketState.OPEN) {
29762
+ const error = new Error("WebSocket connection is not open");
29763
+ console.error("[FacePoke] Error sending JSON message:", error);
29764
+ this.emitEvent("sendJsonMessageError", error);
29765
+ throw error;
29766
+ }
29767
  try {
29768
+ this.ws.send(buffer);
 
 
 
 
 
 
 
 
 
 
 
29769
  } catch (err) {
29770
+ console.error(`failed to send the WebSocket message: ${err}`);
29771
  }
29772
  }
29773
  sendJsonMessage(message) {
 
29777
  this.emitEvent("sendJsonMessageError", error);
29778
  throw error;
29779
  }
29780
+ try {
29781
+ this.ws.send(JSON.stringify(message));
29782
+ } catch (err) {
29783
+ console.error(`failed to send the WebSocket message: ${err}`);
29784
+ }
29785
  }
29786
  setupUnloadHandler() {
29787
  window.addEventListener("beforeunload", () => {
 
29817
  }
29818
  var facePoke = new FacePoke;
29819
 
29820
+ // node_modules/zustand/esm/vanilla.mjs
29821
+ var createStoreImpl = (createState) => {
29822
+ let state;
29823
+ const listeners = new Set;
29824
+ const setState = (partial, replace) => {
29825
+ const nextState = typeof partial === "function" ? partial(state) : partial;
29826
+ if (!Object.is(nextState, state)) {
29827
+ const previousState = state;
29828
+ state = (replace != null ? replace : typeof nextState !== "object" || nextState === null) ? nextState : Object.assign({}, state, nextState);
29829
+ listeners.forEach((listener) => listener(state, previousState));
29830
+ }
29831
+ };
29832
+ const getState = () => state;
29833
+ const getInitialState = () => initialState;
29834
+ const subscribe = (listener) => {
29835
+ listeners.add(listener);
29836
+ return () => listeners.delete(listener);
29837
+ };
29838
+ const api = { setState, getState, getInitialState, subscribe };
29839
+ const initialState = state = createState(setState, getState, api);
29840
+ return api;
29841
+ };
29842
+ var createStore = (createState) => createState ? createStoreImpl(createState) : createStoreImpl;
29843
+
29844
+ // node_modules/zustand/esm/react.mjs
29845
+ var import_react3 = __toESM(require_react(), 1);
29846
+ var useStore = function(api, selector = identity) {
29847
+ const slice = import_react3.default.useSyncExternalStore(api.subscribe, () => selector(api.getState()), () => selector(api.getInitialState()));
29848
+ import_react3.default.useDebugValue(slice);
29849
+ return slice;
29850
+ };
29851
+ var identity = (arg) => arg;
29852
+ var createImpl = (createState) => {
29853
+ const api = createStore(createState);
29854
+ const useBoundStore = (selector) => useStore(api, selector);
29855
+ Object.assign(useBoundStore, api);
29856
+ return useBoundStore;
29857
+ };
29858
+ var create = (createState) => createState ? createImpl(createState) : createImpl;
29859
+
29860
+ // src/lib/convertImageToBase64.ts
29861
+ async function convertImageToBase64(imageFileOrBlob) {
29862
+ return new Promise((resolve, reject) => {
29863
+ const reader = new FileReader;
29864
+ reader.onload = () => {
29865
+ if (typeof reader.result === "string") {
29866
+ resolve(reader.result);
29867
+ } else {
29868
+ reject(new Error("Failed to convert image to base64"));
29869
+ }
29870
+ };
29871
+ reader.onerror = () => {
29872
+ reject(new Error("Error reading file or blob"));
29873
+ };
29874
+ reader.readAsDataURL(imageFileOrBlob);
29875
+ });
29876
+ }
29877
+
29878
+ // src/lib/mapRange.ts
29879
+ var mapRange = (value, inMin, inMax, outMin, outMax) => {
29880
+ return Math.min(outMax, Math.max(outMin, (value - inMin) * (outMax - outMin) / (inMax - inMin) + outMin));
29881
+ };
29882
+
29883
+ // src/hooks/useMainStore.ts
29884
+ var getDefaultState = () => ({
29885
+ status: "",
29886
+ error: "",
29887
+ imageFile: null,
29888
+ isFollowingCursor: false,
29889
+ isGazingAtCursor: false,
29890
+ originalImage: "",
29891
+ originalImageHash: "",
29892
+ previewImage: "",
29893
+ minLatency: 20,
29894
+ averageLatency: 190,
29895
+ maxLatency: 4000,
29896
+ activeLandmark: undefined,
29897
+ metadata: {
29898
+ center: [0, 0],
29899
+ size: 0,
29900
+ bbox: [[0, 0], [0, 0], [0, 0], [0, 0]],
29901
+ angle: 0
29902
+ },
29903
+ params: {},
29904
+ faceLandmarks: [],
29905
+ blendShapes: []
29906
+ });
29907
+ var useMainStore = create((set, get) => ({
29908
+ ...getDefaultState(),
29909
+ setStatus: (status = "") => set({ status }),
29910
+ setError: (error = "") => set({ error }),
29911
+ setFaceLandmarks: (faceLandmarks) => {
29912
+ set({ faceLandmarks });
29913
+ },
29914
+ setBlendShapes: (blendShapes) => {
29915
+ set({ blendShapes });
29916
+ },
29917
+ setImageFile: async (file) => {
29918
+ if (!file) {
29919
+ set({
29920
+ ...getDefaultState(),
29921
+ status: "No file selected"
29922
+ });
29923
+ return;
29924
+ }
29925
+ try {
29926
+ const image = await convertImageToBase64(file);
29927
+ set({
29928
+ ...getDefaultState(),
29929
+ imageFile: file,
29930
+ status: `File selected: ${truncateFileName(file.name, 16)}`,
29931
+ previewImage: image,
29932
+ originalImage: image
29933
+ });
29934
+ facePoke.loadImage(image);
29935
+ } catch (err) {
29936
+ console.log(`failed to load the image: `, err);
29937
+ set({
29938
+ ...getDefaultState(),
29939
+ status: "Failed to load the image"
29940
+ });
29941
+ }
29942
+ },
29943
+ setIsFollowingCursor: (isFollowingCursor) => set({ isFollowingCursor }),
29944
+ setIsGazingAtCursor: (isGazingAtCursor) => set({ isGazingAtCursor }),
29945
+ setOriginalImage: (url) => set({ originalImage: url }),
29946
+ setOriginalImageHash: (originalImageHash) => set({ originalImageHash }),
29947
+ setPreviewImage: (url) => set({ previewImage: url }),
29948
+ resetImage: () => {
29949
+ const { originalImage } = get();
29950
+ if (originalImage) {
29951
+ set({ previewImage: originalImage });
29952
+ }
29953
+ },
29954
+ setAverageLatency: (averageLatency) => set({ averageLatency }),
29955
+ setActiveLandmark: (activeLandmark) => set({ activeLandmark }),
29956
+ setMetadata: (metadata) => set(metadata ? {
29957
+ metadata
29958
+ } : {
29959
+ metadata: getDefaultState().metadata
29960
+ }),
29961
+ setParams: (params) => {
29962
+ const { params: previousParams } = get();
29963
+ set({ params: {
29964
+ ...previousParams,
29965
+ ...params
29966
+ } });
29967
+ },
29968
+ handleServerResponse: async (params) => {
29969
+ const { originalImage, setMetadata, setPreviewImage, setOriginalImageHash, applyModifiedHeadToCanvas, modifyImage } = useMainStore.getState();
29970
+ if (typeof params.error === "string") {
29971
+ console.error(`handleServerResponse: failed to perform the request, resetting the app (${params.error})`);
29972
+ setPreviewImage(originalImage);
29973
+ setOriginalImageHash("");
29974
+ } else if (typeof params.image !== "undefined") {
29975
+ const image = await convertImageToBase64(params.image);
29976
+ setPreviewImage(image);
29977
+ } else if (typeof params.loaded !== "undefined") {
29978
+ setOriginalImageHash(params.loaded.h);
29979
+ setMetadata({
29980
+ center: params.loaded.c,
29981
+ size: params.loaded.s,
29982
+ bbox: params.loaded.b,
29983
+ angle: params.loaded.a
29984
+ });
29985
+ await modifyImage({
29986
+ landmark: {
29987
+ group: "background",
29988
+ distance: 0,
29989
+ vector: { x: 0.5, y: 0.5, z: 0 }
29990
+ },
29991
+ vector: { x: 0, y: 0, z: 0 },
29992
+ mode: "PRIMARY"
29993
+ });
29994
+ } else {
29995
+ console.log(`handleServerResponse: received an unknown json`, params);
29996
+ }
29997
+ },
29998
+ applyModifiedHeadToCanvas: async (headImageBlob) => {
29999
+ return new Promise(async (resolve, reject) => {
30000
+ const originalImg = new Image;
30001
+ const { originalImage, metadata } = useMainStore.getState();
30002
+ originalImg.onload = async () => {
30003
+ const canvas = document.createElement("canvas");
30004
+ const ctx = canvas.getContext("2d");
30005
+ if (!ctx) {
30006
+ reject(new Error("Failed to get 2D context"));
30007
+ return;
30008
+ }
30009
+ const pixelRatio = window.devicePixelRatio || 1;
30010
+ canvas.width = originalImg.width;
30011
+ canvas.height = originalImg.height;
30012
+ ctx.drawImage(originalImg, 0, 0);
30013
+ const headImageBitmap = await createImageBitmap(headImageBlob, {
30014
+ resizeQuality: "high"
30015
+ });
30016
+ const tempCanvas = document.createElement("canvas");
30017
+ const tempCtx = tempCanvas.getContext("2d");
30018
+ if (!tempCtx) {
30019
+ reject(new Error("Failed to get 2D context for temporary canvas"));
30020
+ return;
30021
+ }
30022
+ tempCanvas.width = headImageBitmap.width;
30023
+ tempCanvas.height = headImageBitmap.height;
30024
+ tempCtx.drawImage(headImageBitmap, 0, 0);
30025
+ const gradientSize = 20;
30026
+ const gradient = tempCtx.createRadialGradient(tempCanvas.width / 2, tempCanvas.height / 2, Math.min(tempCanvas.width, tempCanvas.height) / 2 - gradientSize, tempCanvas.width / 2, tempCanvas.height / 2, Math.min(tempCanvas.width, tempCanvas.height) / 2);
30027
+ gradient.addColorStop(0, "rgba(0, 0, 0, 1)");
30028
+ gradient.addColorStop(1, "rgba(0, 0, 0, 0)");
30029
+ tempCtx.globalCompositeOperation = "destination-in";
30030
+ tempCtx.fillStyle = gradient;
30031
+ tempCtx.fillRect(0, 0, tempCanvas.width, tempCanvas.height);
30032
+ console.log("metadata:", metadata);
30033
+ ctx.save();
30034
+ ctx.rotate(metadata.angle);
30035
+ ctx.restore();
30036
+ resolve(canvas.toDataURL("image/png"));
30037
+ };
30038
+ originalImg.src = originalImage;
30039
+ });
30040
+ },
30041
+ modifyImage: async ({ landmark, vector, mode }) => {
30042
+ const {
30043
+ originalImage,
30044
+ originalImageHash,
30045
+ params: previousParams,
30046
+ setParams,
30047
+ setError,
30048
+ isFollowingCursor,
30049
+ isGazingAtCursor
30050
+ } = get();
30051
+ if (!originalImage) {
30052
+ console.error("Image file or facePoke not available");
30053
+ return;
30054
+ }
30055
+ const params = {
30056
+ ...previousParams
30057
+ };
30058
+ const generalControl = {
30059
+ minX: -0.3,
30060
+ maxX: 0.3,
30061
+ minY: -0.3,
30062
+ maxY: 0.3
30063
+ };
30064
+ const pupilControl = {
30065
+ minX: -0.5,
30066
+ maxX: 0.5,
30067
+ minY: -0.5,
30068
+ maxY: 0.5
30069
+ };
30070
+ const eyeControl = {
30071
+ minX: -0.5,
30072
+ maxX: 0.5,
30073
+ minY: -0.5,
30074
+ maxY: 0.5
30075
+ };
30076
+ if (isFollowingCursor) {
30077
+ const yawMin = -40;
30078
+ const yawMax = 40;
30079
+ params.rotate_yaw = mapRange(-vector.x, generalControl.minX, generalControl.maxX, yawMin, yawMax);
30080
+ const pitchMin = -40;
30081
+ const pitchMax = 40;
30082
+ params.rotate_pitch = mapRange(vector.y, generalControl.minY, generalControl.maxY, pitchMin, pitchMax);
30083
+ }
30084
+ if (isGazingAtCursor) {
30085
+ const pupilsXMin = -15;
30086
+ const pupilsXMax = 15;
30087
+ params.pupil_x = mapRange(vector.x, pupilControl.minX, pupilControl.maxX, pupilsXMin, pupilsXMax);
30088
+ const pupilsYMin = -2;
30089
+ const pupilsYMax = 8;
30090
+ params.pupil_y = mapRange(-vector.y, pupilControl.minY, pupilControl.maxY, pupilsYMin, pupilsYMax);
30091
+ }
30092
+ if (mode !== "HOVERING") {
30093
+ switch (landmark.group) {
30094
+ case "leftEye":
30095
+ case "rightEye":
30096
+ const pupilsXMin = -15;
30097
+ const pupilsXMax = 15;
30098
+ params.pupil_x = mapRange(vector.x, pupilControl.minX, pupilControl.maxX, pupilsXMin, pupilsXMax);
30099
+ const eyesMin = -20;
30100
+ const eyesMax = 5;
30101
+ params.eyes = mapRange(-vector.y, eyeControl.minX, eyeControl.maxX, eyesMin, eyesMax);
30102
+ break;
30103
+ case "leftEyebrow":
30104
+ case "rightEyebrow":
30105
+ const eyebrowMin = -10;
30106
+ const eyebrowMax = 15;
30107
+ params.eyebrow = mapRange(-vector.y, eyeControl.minY, eyeControl.maxY, eyebrowMin, eyebrowMax);
30108
+ break;
30109
+ case "lips":
30110
+ const aaaMin = -30;
30111
+ const aaaMax = 120;
30112
+ params.aaa = mapRange(-vector.y, eyeControl.minY, eyeControl.maxY, aaaMin, aaaMax);
30113
+ const eeeMin = -20;
30114
+ const eeeMax = 15;
30115
+ params.eee = mapRange(vector.x, eyeControl.minX, eyeControl.maxX, eeeMin, eeeMax);
30116
+ break;
30117
+ case "faceOval":
30118
+ const rollMin = -40;
30119
+ const rollMax = 40;
30120
+ params.rotate_roll = mapRange(vector.x, eyeControl.minX, eyeControl.maxX, rollMin, rollMax);
30121
+ break;
30122
+ case "background":
30123
+ const yawMin = -40;
30124
+ const yawMax = 40;
30125
+ params.rotate_yaw = mapRange(-vector.x, generalControl.minX, generalControl.maxX, yawMin, yawMax);
30126
+ const pitchMin = -40;
30127
+ const pitchMax = 40;
30128
+ params.rotate_pitch = mapRange(vector.y, eyeControl.minY, eyeControl.maxY, pitchMin, pitchMax);
30129
+ break;
30130
+ default:
30131
+ return;
30132
+ }
30133
+ }
30134
+ for (const [key, value] of Object.entries(params)) {
30135
+ if (isNaN(value) || !isFinite(value)) {
30136
+ console.log(`${key} is NaN, aborting`);
30137
+ return;
30138
+ }
30139
+ }
30140
+ setParams(params);
30141
+ try {
30142
+ if (originalImageHash) {
30143
+ facePoke.transformImage(originalImageHash, params);
30144
+ }
30145
+ } catch (error) {
30146
+ setError("Failed to modify image");
30147
+ }
30148
+ }
30149
+ }));
30150
+
30151
  // node_modules/beautiful-react-hooks/esm/useThrottledCallback.js
30152
  var import_react6 = __toESM(require_react(), 1);
30153
  var import_lodash = __toESM(require_lodash(), 1);
 
32966
 
32967
  // src/hooks/useFaceLandmarkDetection.tsx
32968
  function useFaceLandmarkDetection() {
 
32969
  const setError = useMainStore((s2) => s2.setError);
 
 
 
 
 
32970
  const previewImage = useMainStore((s2) => s2.previewImage);
32971
+ const handleServerResponse = useMainStore((s2) => s2.handleServerResponse);
32972
+ const faceLandmarks = useMainStore((s2) => s2.faceLandmarks);
32973
+ const throttleInMs = 180;
 
 
32974
  const [isMediaPipeReady, setIsMediaPipeReady] = import_react7.useState(false);
32975
  const [isDrawingUtilsReady, setIsDrawingUtilsReady] = import_react7.useState(false);
 
32976
  const [dragStart, setDragStart] = import_react7.useState(null);
 
32977
  const [isDragging, setIsDragging] = import_react7.useState(false);
 
32978
  const dragStartRef = import_react7.useRef(null);
 
 
32979
  const [currentLandmark, setCurrentLandmark] = import_react7.useState(null);
32980
  const [previousLandmark, setPreviousLandmark] = import_react7.useState(null);
32981
  const [currentOpacity, setCurrentOpacity] = import_react7.useState(0);
32982
  const [previousOpacity, setPreviousOpacity] = import_react7.useState(0);
 
32983
  const canvasRef = import_react7.useRef(null);
32984
  const mediaPipeRef = import_react7.useRef({
32985
  faceLandmarker: null,
 
33016
  } else {
33017
  faceLandmarker.close();
33018
  }
33019
+ } catch (error) {
33020
+ console.error("Error during MediaPipe initialization:", error);
33021
  setError("Failed to initialize face detection. Please try refreshing the page.");
33022
  }
33023
  };
 
33103
  }
33104
  }, [landmarkCenters]);
33105
  const detectFaceLandmarks = import_react7.useCallback(async (imageDataUrl) => {
33106
+ const { setFaceLandmarks, setBlendShapes } = useMainStore.getState();
33107
  if (!isMediaPipeReady) {
33108
  console.log("MediaPipe not ready. Skipping detection.");
33109
  return;
 
33203
  }
33204
  detectFaceLandmarks(previewImage);
33205
  }, [isMediaPipeReady, isDrawingUtilsReady, previewImage]);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
33206
  const modifyImageWithRateLimit = useThrottledCallback_default((params) => {
33207
+ useMainStore.getState().modifyImage(params);
33208
+ }, [], throttleInMs);
33209
+ import_react7.useEffect(() => {
33210
+ facePoke.setOnServerResponse(handleServerResponse);
33211
+ }, [handleServerResponse]);
33212
+ const handleStart = import_react7.useCallback((x2, y2, mode) => {
 
 
 
33213
  if (!canvasRef.current)
33214
  return;
33215
  const rect = canvasRef.current.getBoundingClientRect();
33216
+ const normalizedX = (x2 - rect.left) / rect.width;
33217
+ const normalizedY = (y2 - rect.top) / rect.height;
33218
+ const landmark = findClosestLandmark(normalizedX, normalizedY);
 
33219
  setActiveLandmark(landmark);
33220
+ setDragStart({ x: normalizedX, y: normalizedY });
33221
+ dragStartRef.current = { x: normalizedX, y: normalizedY };
33222
  }, [findClosestLandmark, setActiveLandmark, setDragStart]);
33223
+ const handleMove = import_react7.useCallback((x2, y2, mode) => {
33224
  if (!canvasRef.current)
33225
  return;
33226
  const rect = canvasRef.current.getBoundingClientRect();
33227
+ const normalizedX = (x2 - rect.left) / rect.width;
33228
+ const normalizedY = (y2 - rect.top) / rect.height;
33229
+ const landmark = findClosestLandmark(normalizedX, normalizedY, dragStart && dragStartRef.current ? currentLandmark?.group : undefined);
33230
+ const landmarkData = landmarkCenters[landmark?.group];
33231
+ const vector = landmarkData ? {
33232
+ x: normalizedX - landmarkData.x,
33233
+ y: normalizedY - landmarkData.y,
33234
+ z: 0
33235
+ } : {
33236
+ x: 0.5,
33237
+ y: 0.5,
33238
+ z: 0
33239
+ };
33240
  if (dragStart && dragStartRef.current) {
33241
+ setIsDragging(true);
 
33242
  modifyImageWithRateLimit({
33243
  landmark: currentLandmark || landmark,
33244
+ vector,
33245
+ mode
 
 
 
33246
  });
 
33247
  } else {
 
33248
  if (!currentLandmark || currentLandmark?.group !== landmark?.group) {
33249
  setActiveLandmark(landmark);
33250
  }
33251
+ modifyImageWithRateLimit({
33252
+ landmark,
33253
+ vector,
33254
+ mode: "HOVERING"
33255
+ });
33256
  }
33257
+ }, [currentLandmark, dragStart, setActiveLandmark, setIsDragging, modifyImageWithRateLimit, landmarkCenters]);
33258
+ const handleEnd = import_react7.useCallback((x2, y2, mode) => {
33259
  if (!canvasRef.current)
33260
  return;
33261
  const rect = canvasRef.current.getBoundingClientRect();
33262
+ const normalizedX = (x2 - rect.left) / rect.width;
33263
+ const normalizedY = (y2 - rect.top) / rect.height;
33264
  if (dragStart && dragStartRef.current) {
33265
+ const landmark = findClosestLandmark(normalizedX, normalizedY, currentLandmark?.group);
 
33266
  modifyImageWithRateLimit({
33267
  landmark: currentLandmark || landmark,
33268
  vector: {
33269
+ x: normalizedX - landmarkCenters[landmark.group].x,
33270
+ y: normalizedY - landmarkCenters[landmark.group].y,
33271
  z: 0
33272
+ },
33273
+ mode
33274
  });
33275
  }
33276
  setIsDragging(false);
33277
  dragStartRef.current = null;
33278
  setActiveLandmark(undefined);
33279
+ }, [currentLandmark, isDragging, modifyImageWithRateLimit, findClosestLandmark, setActiveLandmark, landmarkCenters, setIsDragging]);
33280
+ const handleMouseDown = import_react7.useCallback((event) => {
33281
+ const mode = event.button === 0 ? "PRIMARY" : "SECONDARY";
33282
+ handleStart(event.clientX, event.clientY, mode);
33283
+ }, [handleStart]);
33284
+ const handleMouseMove = import_react7.useCallback((event) => {
33285
+ const mode = event.buttons === 1 ? "PRIMARY" : "SECONDARY";
33286
+ handleMove(event.clientX, event.clientY, mode);
33287
+ }, [handleMove]);
33288
+ const handleMouseUp = import_react7.useCallback((event) => {
33289
+ const mode = event.buttons === 1 ? "PRIMARY" : "SECONDARY";
33290
+ handleEnd(event.clientX, event.clientY, mode);
33291
+ }, [handleEnd]);
33292
+ const handleTouchStart = import_react7.useCallback((event) => {
33293
+ const mode = event.touches.length === 1 ? "PRIMARY" : "SECONDARY";
33294
+ const touch = event.touches[0];
33295
+ handleStart(touch.clientX, touch.clientY, mode);
33296
+ }, [handleStart]);
33297
+ const handleTouchMove = import_react7.useCallback((event) => {
33298
+ const mode = event.touches.length === 1 ? "PRIMARY" : "SECONDARY";
33299
+ const touch = event.touches[0];
33300
+ handleMove(touch.clientX, touch.clientY, mode);
33301
+ }, [handleMove]);
33302
+ const handleTouchEnd = import_react7.useCallback((event) => {
33303
+ const mode = event.changedTouches.length === 1 ? "PRIMARY" : "SECONDARY";
33304
+ const touch = event.changedTouches[0];
33305
+ handleEnd(touch.clientX, touch.clientY, mode);
33306
+ }, [handleEnd]);
33307
  return {
33308
  canvasRef,
33309
  canvasRefCallback,
33310
  mediaPipeRef,
 
33311
  isMediaPipeReady,
33312
  isDrawingUtilsReady,
 
 
 
33313
  handleMouseDown,
33314
  handleMouseUp,
33315
  handleMouseMove,
33316
+ handleTouchStart,
33317
+ handleTouchMove,
33318
+ handleTouchEnd,
33319
  currentLandmark,
33320
  currentOpacity
33321
  };
33322
  }
33323
 
33324
+ // src/components/About.tsx
33325
  var jsx_dev_runtime2 = __toESM(require_jsx_dev_runtime(), 1);
33326
+ function About() {
33327
  return jsx_dev_runtime2.jsxDEV("div", {
33328
  className: "flex flex-row items-center justify-center font-sans mt-4 w-full",
33329
  children: [
33330
  jsx_dev_runtime2.jsxDEV("span", {
33331
+ className: "text-neutral-900 text-sm",
33332
+ style: { textShadow: "rgb(255 255 255 / 80%) 0px 0px 2px" },
33333
+ children: "Click and drag on the image."
33334
+ }, undefined, false, undefined, this),
33335
+ jsx_dev_runtime2.jsxDEV("span", {
33336
+ className: "ml-2 mr-1",
33337
  children: jsx_dev_runtime2.jsxDEV("img", {
33338
  src: "/hf-logo.svg",
33339
  alt: "Hugging Face",
 
33369
  // src/hooks/useFacePokeAPI.ts
33370
  var import_react8 = __toESM(require_react(), 1);
33371
  function useFacePokeAPI() {
 
33372
  const [isDebugMode, setIsDebugMode] = import_react8.useState(false);
33373
  const [interruptMessage, setInterruptMessage] = import_react8.useState(null);
33374
  const [isLoading, setIsLoading] = import_react8.useState(false);
 
33387
  }, []);
33388
  return {
33389
  facePoke,
 
 
33390
  isDebugMode,
33391
  setIsDebugMode,
33392
  interruptMessage,
 
33404
  children: jsx_dev_runtime4.jsxDEV("div", {
33405
  className: "min-h-screen w-full py-8 flex flex-col justify-center",
33406
  children: jsx_dev_runtime4.jsxDEV("div", {
33407
+ className: "flex flex-col items-center justify-center p-4 sm:max-w-5xl sm:mx-auto",
33408
  children
33409
  }, undefined, false, undefined, this)
33410
  }, undefined, false, undefined, this)
33411
  }, undefined, false, undefined, this);
33412
  }
33413
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
33414
  // src/app.tsx
33415
  var jsx_dev_runtime5 = __toESM(require_jsx_dev_runtime(), 1);
33416
  function App() {
 
33418
  const setError = useMainStore((s2) => s2.setError);
33419
  const imageFile = useMainStore((s2) => s2.imageFile);
33420
  const setImageFile = useMainStore((s2) => s2.setImageFile);
33421
+ const isGazingAtCursor = useMainStore((s2) => s2.isGazingAtCursor);
33422
+ const setIsGazingAtCursor = useMainStore((s2) => s2.setIsGazingAtCursor);
33423
+ const isFollowingCursor = useMainStore((s2) => s2.isFollowingCursor);
33424
+ const setIsFollowingCursor = useMainStore((s2) => s2.setIsFollowingCursor);
33425
  const previewImage = useMainStore((s2) => s2.previewImage);
33426
+ const status = useMainStore((s2) => s2.status);
33427
+ const blendShapes = useMainStore((s2) => s2.blendShapes);
 
33428
  const {
 
 
33429
  isDebugMode,
33430
  setIsDebugMode,
33431
  interruptMessage
33432
  } = useFacePokeAPI();
33433
  const {
 
33434
  canvasRefCallback,
 
 
33435
  isMediaPipeReady,
 
 
 
33436
  handleMouseDown,
33437
  handleMouseUp,
33438
  handleMouseMove,
33439
+ handleTouchStart,
33440
+ handleTouchMove,
33441
+ handleTouchEnd,
33442
  currentOpacity
33443
  } = useFaceLandmarkDetection();
33444
  const videoRef = import_react9.useRef(null);
33445
+ const handleFileChange = import_react9.useCallback((event) => {
33446
  const files = event.target.files;
33447
+ setImageFile(files?.[0] || undefined);
33448
+ }, [setImageFile]);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
33449
  const handleDownload = import_react9.useCallback(() => {
33450
  if (previewImage) {
33451
  const link = document.createElement("a");
 
33522
  className: "flex items-center space-x-2",
33523
  children: [
33524
  jsx_dev_runtime5.jsxDEV("div", {
33525
+ className: "flex items-center justify-center",
33526
  children: [
33527
  jsx_dev_runtime5.jsxDEV("input", {
33528
  id: "imageInput",
 
33537
  className: `cursor-pointer inline-flex items-center px-3 h-10 border border-transparent text-sm font-medium rounded-md text-white ${isMediaPipeReady ? "bg-slate-600 hover:bg-slate-500" : "bg-slate-500 cursor-not-allowed"} focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-slate-500 shadow-xl`,
33538
  children: [
33539
  jsx_dev_runtime5.jsxDEV(Spinner, {}, undefined, false, undefined, this),
33540
+ imageFile ? truncateFileName(imageFile.name, 32) : isMediaPipeReady ? "Choose a portrait photo" : "Initializing..."
33541
  ]
33542
  }, undefined, true, undefined, this)
33543
  ]
 
33554
  }, undefined, true, undefined, this)
33555
  ]
33556
  }, undefined, true, undefined, this),
33557
+ previewImage && jsx_dev_runtime5.jsxDEV("div", {
33558
+ className: "flex items-center space-x-2",
33559
+ children: jsx_dev_runtime5.jsxDEV("label", {
33560
+ className: "mt-4 flex items-center",
33561
+ children: [
33562
+ jsx_dev_runtime5.jsxDEV("input", {
33563
+ type: "checkbox",
33564
+ checked: isDebugMode,
33565
+ onChange: (e2) => setIsDebugMode(e2.target.checked),
33566
+ className: "mr-2"
33567
+ }, undefined, false, undefined, this),
33568
+ "Show face markers"
33569
+ ]
33570
+ }, undefined, true, undefined, this)
33571
+ }, undefined, false, undefined, this)
33572
  ]
33573
  }, undefined, true, undefined, this),
33574
  previewImage && jsx_dev_runtime5.jsxDEV("div", {
 
33582
  jsx_dev_runtime5.jsxDEV("canvas", {
33583
  ref: canvasRefCallback,
33584
  className: "absolute top-0 left-0 w-full h-full select-none",
 
 
33585
  onMouseDown: handleMouseDown,
33586
  onMouseUp: handleMouseUp,
33587
  onMouseMove: handleMouseMove,
33588
+ onTouchStart: handleTouchStart,
33589
+ onTouchMove: handleTouchMove,
33590
+ onTouchEnd: handleTouchEnd,
33591
  style: {
33592
  position: "absolute",
33593
  top: 0,
 
33603
  canDisplayBlendShapes && displayBlendShapes
33604
  ]
33605
  }, undefined, true, undefined, this),
33606
+ jsx_dev_runtime5.jsxDEV(About, {}, undefined, false, undefined, this)
33607
  ]
33608
  }, undefined, true, undefined, this);
33609
  }
requirements.txt CHANGED
@@ -7,12 +7,15 @@
7
  torch
8
  torchvision
9
  torchaudio
10
-
11
 
12
  # --------------------------------------------------------------------
13
  # Common libraries for LivePortrait and all
14
  # --------------------------------------------------------------------
15
 
 
 
 
16
  # note: gradio is only used for the cropping utility
17
  gradio==5.0.0b4
18
 
@@ -35,8 +38,8 @@ ffmpeg-python==0.2.0
35
  onnxruntime-gpu==1.19.2
36
  onnx==1.16.2
37
  scikit-image==0.20.0
38
- albumentations==1.4.10
39
- matplotlib==3.9.0
40
  tyro==0.8.5
41
  chumpy==0.70
42
 
@@ -45,16 +48,24 @@ accelerate==0.34.2
45
  tensorflow==2.12.0
46
  tensorboard==2.12.0
47
  transformers==4.39.2
48
- huggingface-hub==0.25.1
49
- safetensors==0.4.5
50
 
51
  gdown==5.2.0
52
  requests==2.32.3
53
  omegaconf==2.3.0
 
54
  pydantic==2.9.2
55
- dnspython>=2.0.0
56
- ifaddr>=0.2.0
 
 
 
 
57
  aiohttp==3.10.5
 
 
 
 
 
58
 
59
  # --------------------------------------------------------------------
60
  # Used for advanced LivePortrait features
 
7
  torch
8
  torchvision
9
  torchaudio
10
+ torchgeometry
11
 
12
  # --------------------------------------------------------------------
13
  # Common libraries for LivePortrait and all
14
  # --------------------------------------------------------------------
15
 
16
+ # LRU cache compatible with asyncio
17
+ async-lru==2.0.4
18
+
19
  # note: gradio is only used for the cropping utility
20
  gradio==5.0.0b4
21
 
 
38
  onnxruntime-gpu==1.19.2
39
  onnx==1.16.2
40
  scikit-image==0.20.0
41
+ albumentations==1.3.1
42
+ matplotlib==3.7.2
43
  tyro==0.8.5
44
  chumpy==0.70
45
 
 
48
  tensorflow==2.12.0
49
  tensorboard==2.12.0
50
  transformers==4.39.2
 
 
51
 
52
  gdown==5.2.0
53
  requests==2.32.3
54
  omegaconf==2.3.0
55
+
56
  pydantic==2.9.2
57
+
58
+ # --------------------------------------------------------------------
59
+ # RESERVED FOR FUTURE USAGE
60
+ #
61
+ # (it adds bloat, so you can remove them if you want)
62
+ # --------------------------------------------------------------------
63
  aiohttp==3.10.5
64
+ av==12.3.0
65
+ einops==0.7.0
66
+ safetensors==0.4.5
67
+ huggingface-hub==0.25.1
68
+ optimum-quanto==0.2.4
69
 
70
  # --------------------------------------------------------------------
71
  # Used for advanced LivePortrait features