open_mouth.py

#I'm truly sorry, but I must admit the code is very confusing. comment still written in Japanese


"""
# Currently on hold,Verifying whether to use other technologies.
# this is temporaly fixed for work huggingface space

スクリプト名
batch_open_mouth.py

概要
静止画から口を開ける画像を作成

説明

引数
argparseを参照

不具合
横方向は思ったより機能していない。
Issueが山盛り
https://github.com/akjava/lip_recognition_tools/issues

著者: Akihito Miyazaki
作成日: 2024-04-23
更新履歴:
  - 2024-04-23: 最初のリリース
  - 2024-09-15:hole_offsetを追加
# 口の中の位置は、create_hole_image.pyの画像を変更すること

"""

import cv2
import numpy as np
from PIL import Image
import lip_utils
import create_top_lip
import create_bottom_lip
import create_chin_image
import create_no_mouth
import create_hole_image
import os
import argparse
import landmarks68_utils
import math
import sys

from glibvision.common_utils import check_exists_files

#arg version not tested
def parse_arguments():
   parser = argparse.ArgumentParser(description='Open Mouth')
   #parser.add_argument('--scale',"-sc",help='スケール精度が上がる',default=4,type=int)
   parser.add_argument('--no_close_lip',"-ccl",help='Close Lip画像を作らない',action="store_false")
   parser.add_argument('--landmark',"-l",help='landmarkdata')
   parser.add_argument('--input',"-i",help='変換する画像の元(必須) 口を閉じていること',required=True)
   parser.add_argument('--output',"-o",help='画像の保存先(別途一時的なレイヤーファイルも作られる)')
   parser.add_argument('--open_size_x',"-x",help='横方向へ広がるサイズ(いまいち機能していない)',default = 0,type=int)
   parser.add_argument('--open_size_y',"-y",help='縦方向への口の広がり(最大20ぐらい)',default=9,type=int)
   parser.add_argument('--hole_offset',"-hole",help='口内画像の上下',type=int,default=0)
   parser.add_argument('--hole_image_name',"-hname",help='口内画像・ただし、hole_images内にあること',default="dark01.jpg")
   parser.add_argument('--side_edge_expand',"-see",help='横の端をどれだけ動かすか',type=float,default=0.02)
   parser.add_argument('--inside_layer_low_depth',"-illd",action="store_true",help="basically not good small size but works for large and img2img")

   lip_utils.DEBUG=True
   #parser.add_argument('--hole_image_key',"-hi",help='口内画像',default="hole_01")
   return parser.parse_args()

if __name__ == "__main__":
   args=parse_arguments()
   # 画像ファイルのパス
   img_path = args.input

   output = args.output

   if output==None:
      base,ext = os.path.splitext(img_path)
      output = f"{base}_{args.open_size_y:02d}.jpg"
   

   #landmark = landmark_utils.create_landmarks_path(img_path,args.landmark)
   landmark = None
   if check_exists_files([landmark,img_path],[],False):
      print("File Error happend and exit app")
      exit(1)
   img = cv2.imread(img_path) # force load 3 channel
   #landmarks_list = landmark_utils.load_landmarks_json(landmark)
   #side_edge_expand = args.side_edge_expand
   use_close_lip = args.no_close_lip
   process_open_mouth(img,None,use_close_lip)#TODO test
# LOAD Image and Landmarkdata

def process_open_mouth(cv_image,landmarks_list,open_size_x=0,open_size_y=8,use_close_lip=True,inside_layer_low_depth=False,hole_offset=0,hole_image_name="dark01.jpg",side_edge_expand=0.02):
   img = cv_image 
   img_h, img_w = lip_utils.get_image_size(img)
   

   ## MODIFY POINTS
   top_points=lip_utils.get_landmark_points(landmarks_list,lip_utils.TOP_LIP)
   print(top_points)



   
   '''
   right_outer = top_points[0]
   left_outer = top_points[6]
   lip_width = lip_utils.distance_2d(left_outer,right_outer)
   #print(lip_width)
   lip_diff = [left_outer[0]-right_outer[0],left_outer[1]-right_outer[1]]

   side_edge_expand_point =[lip_diff[0]*side_edge_expand,lip_diff[1]*side_edge_expand]
   #print(side_edge_expand_point)
   print(f"side-edge expanded {side_edge_expand_point}")

   top_points[0][0]-=int(side_edge_expand_point[0])
   top_points[6][0]+=int(side_edge_expand_point[0])
   top_points[0][1]-=int(side_edge_expand_point[1])
   top_points[6][1]+=int(side_edge_expand_point[1])
   #img = cv2.imread(img_path,cv2.IMREAD_UNCHANGED) #4channel got problem use green back
   #img = cv2.cvtColor(img, cv2.COLOR_BGRA2BGR)
   '''
   # 常に作成
   if use_close_lip: # store falseというわかりにくいやつ
      import close_lip
      img,mask = close_lip.process_close_lip_image(img,landmarks_list)
      close_lip_image = img
      #return img


  

   margin = 12



   hole_points = lip_utils.get_lip_hole_points(landmarks_list)

   #print(hole_points)


   ## LIP MOVE UP
   (bottom_width,bottom_height)=lip_utils.get_bottom_lip_width_height(landmarks_list)
   left_thick,mid_thick,right_thick = lip_utils.get_top_lip_thicks(landmarks_list)
   bottom_base =  bottom_height/1.5

   diff_left = max(0,int(left_thick - bottom_base))
   diff_right = max(0,int(right_thick - bottom_base))
   diff_mid = max(0,int((diff_right+diff_left)*0.4))
   diff_avg = int((diff_right+diff_left)*0.5)


   # たいてい歯がご認識、そのぶん戻す。4は下唇との比較で
   fix_top_thick_hole_points = []
   top_point = [1,2,3]
   for idx,point in enumerate(hole_points):
      if idx in top_point:
         new_point = np.copy(point)
         if idx == 2:
            new_point[1] -= int(diff_avg*0.5) # TODO calcurate
         else:
            new_point[1] -= int(diff_avg*1) # TODO calcurate
         fix_top_thick_hole_points.append(new_point)
      else:
         fix_top_thick_hole_points.append(point)


   mask = lip_utils.create_mask_from_points(img,fix_top_thick_hole_points,2,2)
   inverse_mask = cv2.bitwise_not(mask)
   if lip_utils.DEBUG:
      cv2.imwrite("holeed_mask.jpg",mask)
      cv2.imwrite("holeed_inverse_mask.jpg",inverse_mask)

   img_transparent = lip_utils.apply_mask_alpha(img,mask)
   img_inpainted = cv2.inpaint(img,mask,3,cv2.INPAINT_TELEA)
   if lip_utils.DEBUG:
      cv2.imwrite("holeed_transparent.png",img_transparent)
      cv2.imwrite("holeed_inpainted.jpg",img_inpainted)

   #img_holed = np.copy(img)


   ## APPLY MASK OTHER WAY TODO check later
   #I'm not sure this logic
   #mask_2d = mask[:, :, 0] 
   #img_holed[:, :, 3] = mask_2d
   #cv2.imwrite("holed_image_mask.png",mask)


   # create gaussin image
   gaussian_size = 10
   gaused = cv2.GaussianBlur(img_inpainted, (0,0 ), sigmaX=gaussian_size, sigmaY=gaussian_size)
   #img_holed = lip_utils.apply_mask_alpha(img_holed,mask)
   #lip_utils.fill_points(hole_points,img,2,(255,0,0),(255,0,0))
   #lip_utils.fill_points(hole_points,img,0,(255,0,0,0),(255,0,0,0))
   if lip_utils.DEBUG:
      cv2.imwrite("holed_gaused.jpg",gaused)
   mask_1d = np.squeeze(mask)
   img_inpainted[mask_1d==255] = gaused[mask_1d==255]


   # image bitwise faild
   thresh, binary_mask = cv2.threshold(mask, 127, 255, cv2.THRESH_BINARY)
#result = cv2.bitwise_and(img_inpainted, gaused, mask=inverse_mask) # transform mask area
#result = cv2.bitwise_and(gaused, result, mask=binary_mask) # transform mask area
#result = cv2.bitwise_or(result, img_inpainted) # transform remains?
#cv2.imwrite("holeed_bitwise.jpg",result)

#exit(0)

# 重みを計算する関数を定義します。
# この例では、単純な重み付け方法を使用していますが、
# 実際のアプリケーションでは、より複雑なロジックを使用することができます。
# examle
   def calculate_weights(image):
      # ここで重みを計算します。この例では、単純な例を示しています。
      # 実際のアプリケーションでは、画像の特性に基づいて重みを計算することができます。
      weights = np.ones_like(image) * 0.5 # 例: すべてのピクセルに対して0.5の重みを割り当てます。
      return weights

# 画像ごとに重みを計算します。

#weights1 = 1.0 - img_holed[:, :, 3] / 255.0  
#weights1 = 1.0 - mask / 255.0 
#weights2 = 1.0 - weights1
#weights1 = calculate_weights(img)
#weights2 = calculate_weights(img_holed)

# 重み付きの加算を行います。
#result = (img_holed * weights1 + img * weights2) / (weights1 + weights2)
# 重み付きの加算を行います。アルファチャンネルを除いたRGBチャンネルに対して加算を行います。
#result_rgb = (img_holed[:, :, :3] * weights1[:, :, np.newaxis] + img[:, :, :3] * weights2[:, :, np.newaxis]) / (weights1[:, :, np.newaxis] + weights2[:, :, np.newaxis])

# アルファチャンネルを計算します。
# TODO support alpha
#result_alpha = (weights1 + weights2)
# 結果をアルファチャンネルと結合します。#somehow error toto check
#result = cv2.merge([result_rgb, result_alpha.astype(np.uint8)])


#result_rgb = cv2.cvtColor(img_holed, cv2.COLOR_BGRA2BGR)
#result = result.astype(np.uint8)
#cv2.imwrite("holed_image_mixed.jpg",result_rgb)
#exit(0)



   top_lip_layer,lip_mask = create_top_lip.process_lip_image(img_transparent,landmarks_list, margin, open_size_y, open_size_x)# Y is first
   hole_image = create_hole_image.process_create_hole_image(img,landmarks_list,open_size_y,open_size_x,hole_offset,hole_image_name)
   hole_image_apply_mask = lip_utils.apply_mask(hole_image,lip_mask)
   if lip_utils.DEBUG:
      cv2.imwrite("hole_image.jpg",hole_image)
      cv2.imwrite("hole_image_apply_mask.png",hole_image_apply_mask)

   bottom_lip_layer = create_bottom_lip.process_lip_image(img,landmarks_list, margin, open_size_y*2, open_size_x)
   chin_layer = create_chin_image.process_chin_image(img,landmarks_list, margin, open_size_y, open_size_x)

   no_mouth_face = create_no_mouth.process_create_no_mouth_image(img,landmarks_list)


   chin_points = lip_utils.get_landmark_points(landmarks_list,lip_utils.POINTS_CHIN)
   points =[]
   lip_points = lip_utils.get_lip_mask_points(landmarks_list)
   center_lips = lip_points[2:5]
   print("center")
   print(center_lips)
   center_lips = center_lips[::-1]
   print(center_lips)
   points.extend(center_lips+chin_points[4:13])
   print(points)
   for i in range(4,len(points)-1):
      points[i][1] += open_size_y

   jaw_mask_line = lip_utils.create_mask(no_mouth_face,(0,0,0))
   cv2.polylines(jaw_mask_line, [np.array(points)], isClosed=True, color=(0,255,0), thickness=1)
   if lip_utils.DEBUG:
      cv2.imwrite("open_mouth_jaw_mask_line.jpg",jaw_mask_line)

   dilation_size=3
   jaw_mask = lip_utils.create_mask_from_points(img,points,dilation_size,3)
#cv2.imwrite("open_mouth_jaw_mask.jpg",jaw_mask)


   
   from PIL import Image

   def convert_cv2_to_pil(cv2_img,is_bgra=True):
      """
      OpenCV (cv2) 画像を PIL 画像に変換する関数
      """
      # BGR から RGB に変換
      if is_bgra:
         rgb_img = cv2.cvtColor(cv2_img, cv2.COLOR_BGRA2RGBA)
      # PIL Image オブジェクトを作成
      pil_img = Image.fromarray(rgb_img)
      return pil_img

   # 画像のリストを作成
   pil_images =[]
#pil_images.append(Image.open("face_no_lip.jpg").convert("RGBA"))

#below are wrong too weak hole_image
#layers = [no_mouth_face,hole_image,top_lip_layer,bottom_lip_layer,chin_layer]

   #this order is right
   # second chind is wrong,when animation ghosted
   if inside_layer_low_depth:
      layers = [no_mouth_face,hole_image_apply_mask,top_lip_layer,chin_layer,bottom_lip_layer]
   else:
      layers = [no_mouth_face,top_lip_layer,chin_layer,bottom_lip_layer,hole_image_apply_mask]
   for layer in layers:
      pil_images.append(convert_cv2_to_pil(layer))


   #images = [convert_cv2_to_pil(mask),convert_cv2_to_pil(face_size_image)]
   layers = layers[::-1]
   output_image = None
   for i in range(len(pil_images)):
      if output_image == None:
         #cv2_image_rgb = cv2.cvtColor(layers[i], cv2.COLOR_BGRA2RGBA)
         #pil_image1 = Image.fromarray(cv2_image_rgb)
         output_image = pil_images[i]
         continue
      else:
         pil_image1 = output_image

   #cv2_image_rgb = cv2.cvtColor(layers[i], cv2.COLOR_BGRA2RGBA)
   #pil_image2 = Image.fromarray(cv2_image_rgb)

      output_image = Image.alpha_composite(pil_image1, pil_images[i])

   #output_image = lip_utils.alpha_blend_with_image2_alpha(output_image,layers[i+1])


   output_image = output_image.convert("RGB")
   
   #import webp
   #webp.save_images(pil_images, 'anim.webp', fps=10, lossless=True)

   return output_image

   name,ext = os.path.splitext(output)





   if ext == "":
      output += ".jpg"
      
   output_image_path = output.replace(".png",".jpg")
   output_image.save(output_image_path)#force save jpeg
   cv2.imwrite(output_image_path.replace(".jpg","_mask.jpg"),jaw_mask)

   if close_lip_image is not None:
      pass # no save 連番でつくるはめになる 保留
      #close_lip_path = f"{name}_close-lip.jpg"
      cv2.imwrite("close-lip.jpg",close_lip_image)

   print(f"open-mouth created {output}")
   #cv2.imwrite(output,output_image)

   # アニメーションとして保存 for later checking TODO add option