create_top_lip.py

import cv2
import numpy as np
from PIL import Image
import lip_utils

def process_lip_image(img,landmarks_list, margin, open_size_y, open_size_x):
    print(open_size_x)
    """
    唇画像を処理する関数
    """
    img_h, img_w = lip_utils.get_image_size(img)


    open_size_x = 0 #stop support this
    print("currently stop support open-sizex")

    #for 40 # TODO recheck later issues/91
    side_tips = 0 # TODO depent on size or point
    #side_tips = 0
    side_tips = open_size_x
    edge_x = int(open_size_x*1.4) #som magic number

    mid_lip_move_artio = open_size_y/80.0 if open_size_y>0 else 0
    mid_lip_shrink_artio = open_size_x/4 if open_size_x>0 else 0

    # 上唇の抽出と処理
    top_lip_rgba, cropped_box = lip_utils.get_alpha_image(img, landmarks_list, lip_utils.POINTS_TOP_LIP, margin, margin, 4)
    if lip_utils.DEBUG:
        cv2.imwrite("top_lip_rgba.png",top_lip_rgba)
    new_h,new_w = lip_utils.get_image_size(top_lip_rgba)
    w = new_w
    h = new_h
    print(f"top-lip-alpha-margined-size:w = {new_w} h = {new_h} margin = {margin}")
    align_points = lip_utils.get_top_lip_align_points(landmarks_list)

    box = cropped_box
    top_points=lip_utils.get_landmark_points(landmarks_list,lip_utils.POINTS_TOP_LIP)
    cropped_lip_points = [(point[0] - box[0][0], point[1] - box[0][1]) for point in top_points]
    lip_points = [(point[0] - box[0][0], point[1] - box[0][1]) for point in align_points]
    middle_lip = ((lip_points[0][0] + lip_points[1][0]) / 2, (lip_points[0][1] + lip_points[1][1]) / 2)
    print(f"middle:{middle_lip}")


    #DEV
    print(f"box {cropped_box[0][0]},{cropped_box[0][1]}")
    face_size_image=lip_utils.create_rgba(img_w,img_h)
    lip_utils.copy_image(face_size_image,top_lip_rgba,cropped_box[0][0]-margin,cropped_box[0][1]-margin)
    if lip_utils.DEBUG:
        cv2.imwrite("top_lip_layer.png",face_size_image)




   

    #intではないとエラー
    middle_y = max(1,int(middle_lip[1]-5)) # force move up


    # ３分割
    mid_x1=int(w/3) # LEFT
    mid_x2 = w -mid_x1 # RIGHT
    print(f"image width = {new_w} mid_left = {mid_x1} mid_right ={mid_x2}")

    cx, cy, cx2, cy2 = 0, middle_y,mid_x1, h

    print("###",w,",",middle_y)
    crop_top = lip_utils.crop_image(top_lip_rgba,0,0,w,middle_y)#full top

    #if use mid only left right change control of up
    #crop_top = lip_utils.crop_image(top_lip_rgba,mid_x1,0,mid_x2,middle_y)
    if lip_utils.DEBUG:
        cv2.imwrite("crop_top.png",crop_top)
    #cv2.imwrite("top_lip_mid.png",crop_mid)

    below_top_lip_image = lip_utils.crop_image(top_lip_rgba,0,middle_y,w,h)
    below_top_lip_image_h,below_top_lip_image_w = lip_utils.get_image_size(below_top_lip_image)
    if lip_utils.DEBUG:
        cv2.imwrite("below_top_lip_image.png",below_top_lip_image)
    print(f"below_top_lip_image w = {below_top_lip_image_w}, h= {below_top_lip_image_h}")

    #中央部分を切り抜く

    mid_x1_x2_half = int((mid_x2+mid_x1)/2)
    print(mid_x1_x2_half)
    crop_mid_left = lip_utils.crop_image(below_top_lip_image,mid_x1,0,mid_x1_x2_half,below_top_lip_image_h)
    lip_utils.print_width_height(crop_mid_left,"crop_mid_left")
    crop_mid_h,crop_mid_w = lip_utils.get_image_size(crop_mid_left)
    max_w = crop_mid_w
    max_h = crop_mid_h
    moveup_lip_mid = lip_utils.create_moved_image(crop_mid_left, [(0,0),(max_w,0),
                    (0,max_h),(max_w,max_h)],

                    [(0,0),(crop_mid_w,0),
                    (0,int(max_h)),(max_w,int(crop_mid_h*(1.0 - mid_lip_move_artio)))]# TODO ratio
                    )
    lip_utils.print_width_height(moveup_lip_mid,"crop_mid_left-moved")
   
    if lip_utils.DEBUG:
        cv2.imwrite("moveup_lip_mid_left.png",moveup_lip_mid)

    crop_mid_right = lip_utils.crop_image(below_top_lip_image,mid_x1_x2_half,0,mid_x2,below_top_lip_image_h)
    crop_mid_h,crop_mid_w = lip_utils.get_image_size(crop_mid_right)
    max_w = crop_mid_w
    max_h = crop_mid_h
    lip_utils.print_width_height(crop_mid_right,"crop_mid_right")
    moveup_lip_mid_right = lip_utils.create_moved_image(crop_mid_right, [(0,0),(max_w,0),
                    (0,max_h),(max_w,max_h)],

                    [(0,0),(max_w,0),
                    (0,int(max_h*(1.0-mid_lip_move_artio))),(max_w,int(max_h))]
                    )
    lip_utils.print_width_height(moveup_lip_mid_right,"crop_mid_right-moved")
    if lip_utils.DEBUG:
        cv2.imwrite("moveup_lip_mid_right.png",moveup_lip_mid_right)




    # 最終画像　サイズは、最初の 切り抜きに 口を開く + open_size_y
    top_lip_final_image=lip_utils.create_rgba(w,h+open_size_y+1)#some how transform image expand
    final_image_h,final_image_w = lip_utils.get_image_size(top_lip_final_image)
    print(f"final-image-size:w = {final_image_w} h = {final_image_h}")


    # left block
    left_lip_image = lip_utils.crop_image(below_top_lip_image,side_tips,0,mid_x1,below_top_lip_image_h)
    left_lip_image_h,left_lip_image_w = lip_utils.get_image_size(left_lip_image)
    print(f"left-image-cropped:w = {left_lip_image_w} h = {left_lip_image_h}")
    # this left transofom is very important change result (dont feel strange +open_size_x)
    max_w = left_lip_image_w
    max_h = left_lip_image_h

    opend_lip_left = lip_utils.create_moved_image(left_lip_image, 
                    [(0,0),(max_w,0),
                    (0,max_h),(max_w,max_h)],

                    [(0,0+open_size_y),(max_w+open_size_x,0),
                    (0,max_h+open_size_y),(max_w+open_size_x,max_h)]
                    )
  

    new_h,new_w = lip_utils.get_image_size(opend_lip_left)
    print(f"left-image-moved:w = {new_w} h = {new_h}")
    if lip_utils.DEBUG:
        cv2.imwrite("top_lip_opend_left.png",opend_lip_left)

    right_lip_image = lip_utils.crop_image(below_top_lip_image,mid_x2,0,below_top_lip_image_w-side_tips,below_top_lip_image_h)
    right_lip_image_h,right_lip_image_w = lip_utils.get_image_size(right_lip_image)
    print(f"right-image-cropped:w = {right_lip_image_w} h = {right_lip_image_h}")
    max_w = right_lip_image_w
    #cv2.imwrite("top_lip_opend_left.png",opend_lip_left)
    # right block
    #right-image-cropped:w = 39 h = 32
    opend_lip_right = lip_utils.create_moved_image(right_lip_image, 
                    [(0,0),(right_lip_image_w-1,0),
                    (0,right_lip_image_h-1),(right_lip_image_w-1,right_lip_image_h-1)],

                    [(-0,0),(right_lip_image_w-1+open_size_x,open_size_y), # remove corner shrink it broke image
                    (0,int(crop_mid_h-1)),(right_lip_image_w+open_size_x-1,right_lip_image_h-1+open_size_y)]
                    #,(39+open_size_x,right_lip_image_h+open_size_y) #TOD
                    )


    new_h,new_w = lip_utils.get_image_size(opend_lip_right)
    right_image_w_changed = new_w-right_lip_image_w

    print(f"right-image-moved:w = {new_w} h = {new_h}")
    if lip_utils.DEBUG:
        cv2.imwrite("top_lip_opend_right.png",opend_lip_right)


    move_x = open_size_x +(open_size_x-right_image_w_changed)
    print(f"right_image_w_changed ={right_image_w_changed} open_size_x ={open_size_x} move_x ={move_x}")

    lip_utils.copy_image(top_lip_final_image,crop_top,0,0) # full version
    #lip_utils.copy_image(top_lip_final_image,crop_top,mid_x1,0)
    print(f"open size x = {open_size_x}")
    lip_utils.copy_image(top_lip_final_image,opend_lip_left,side_tips,middle_y)# open_size_x must slided and minus value

    #mid 
    lip_utils.copy_image(top_lip_final_image,moveup_lip_mid,mid_x1,middle_y)
    lip_utils.copy_image(top_lip_final_image,moveup_lip_mid_right,mid_x1_x2_half,middle_y)
    
    lip_utils.copy_image(top_lip_final_image,opend_lip_right,mid_x2,middle_y)


    if lip_utils.DEBUG:
        cv2.imwrite("top_lip_opend.png",top_lip_final_image)
    face_size_image=lip_utils.create_rgba(img_w,img_h)
    lip_utils.copy_image(face_size_image,top_lip_final_image,box[0][0],box[0][1])
    if lip_utils.DEBUG:
        cv2.imwrite("top_lip_layer.png",face_size_image)
    

    # possible bug inverted
    points = lip_utils.get_lip_hole_points(landmarks_list)
    #points = lip_utils.get_lip_hole_top_points(landmarks_list)

    statics=[1,2,3]
    half =[0,4,5,9]

    #not effect now  open-sizex set 0 at begging
    m = 1
   

    ## TOP Lip Move Up basically upper thick is 1.5 x lower lip toher are teeth
    (bottom_width,bottom_height)=lip_utils.get_bottom_lip_width_height(landmarks_list)
    left_thick,mid_thick,right_thick = lip_utils.get_top_lip_thicks(landmarks_list)
    bottom_base =  bottom_height/1.5
    
    diff_left = max(0,int(left_thick - bottom_base))
    diff_right = max(0,int(right_thick - bottom_base))
    diff_mid = max(0,int((diff_right+diff_left)*0.4))
    print(f"bottom base = {bottom_base} left thick ={left_thick} diff ={diff_left}")
    print(f"bottom base = {bottom_base} left thick ={left_thick} mid ={diff_mid}")


    (bottom_width,bottom_height)=lip_utils.get_bottom_lip_width_height(landmarks_list)

    mid_lip_drop_size = lip_utils.get_bottom_mid_drop_size(open_size_y,bottom_height)
    print(f"mid_lip_drop_size = {mid_lip_drop_size}")
    moved_points = []
    for idx,point in enumerate(points):
        if idx not in statics:
            if idx in half:
                plus_x = 0
                if idx  == 5 :#or idx ==0
                    plus_x =  open_size_x*m
                elif  idx == 9:#idx == 4 or
                    plus_x = -open_size_x*m
                elif idx == 0:
                    plus_x = open_size_x*m
                elif idx == 4:
                    plus_x =-open_size_x*m
                print(f"idx ={idx} plus {plus_x}")
                
                moved_points.append((point[0]+plus_x,point[1]+open_size_y/2))
            else:
                #bottom
                moved_points.append((point[0],point[1]+int(open_size_y*2)+mid_lip_drop_size))
        else:
            print(f"static ? {idx}")
            #static top
            #moved_points.append((point[0],point[1]-crop_mid_h/4)) #for open 40
            #moved_points.append((point[0],point[1]))
            if idx == 3:
                 moved_points.append((point[0],point[1]-diff_left))
                 print(f"left top lip move up {diff_left}")
            elif idx == 2:
                moved_points.append((point[0],point[1]-diff_mid))
            elif idx == 1:
                moved_points.append((point[0],point[1]-diff_right))
                print(f"right top lip move up {diff_right}")
                

    


    # force moved
    #moved_points[1][1] = moved_points[1][1] -4

    tmp = lip_utils.create_mask_from_points(img,points,int(open_size_y/2),0)
    if lip_utils.DEBUG:
        cv2.imwrite("lip_hole_mask_base.jpg",tmp)

    gaus = 2 # ADD OPTION
    mask = lip_utils.create_mask_from_points(img,moved_points,int(open_size_y/2),gaus)
    if lip_utils.DEBUG:
        cv2.imwrite("lip_hole_mask.jpg",mask)


    return face_size_image,mask

if __name__ == "__main__":
  # 画像ファイルのパス
  img_path = "straight.jpg"
  # パラメータ
  margin = 4
  open_size_y = 20
  open_size_x = 0

  # 関数の呼び出し
  process_lip_image(img_path, margin, open_size_y, open_size_x)