from PIL import Image
import torch
from point_e.diffusion.configs import DIFFUSION_CONFIGS, diffusion_from_config
from point_e.diffusion.sampler import PointCloudSampler
from point_e.models.download import load_checkpoint
from point_e.models.configs import MODEL_CONFIGS, model_from_config
from point_e.util.plotting import plot_point_cloud
from point_e.util.pc_to_mesh import marching_cubes_mesh
import skimage.measure
from pyntcloud import PyntCloud
import matplotlib.colors
import plotly.graph_objs as go
import trimesh
import gradio as gr
state = ""
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
def set_state(s):
print(s)
global state
state = s
def get_state():
return state
set_state('Creating txt2mesh model...')
t2m_name = 'base40M-textvec' # 'base40M'
t2m_model = model_from_config(MODEL_CONFIGS[t2m_name], device)
t2m_model.eval()
base_diffusion_t2m = diffusion_from_config(DIFFUSION_CONFIGS[t2m_name])
set_state('Downloading txt2mesh checkpoint...')
t2m_model.load_state_dict(load_checkpoint(t2m_name, device))
def load_img2mesh_model(model_name):
set_state(f'Creating img2mesh model {model_name}...')
i2m_name = model_name
i2m_model = model_from_config(MODEL_CONFIGS[i2m_name], device)
i2m_model.eval()
base_diffusion_i2m = diffusion_from_config(DIFFUSION_CONFIGS[i2m_name])
set_state(f'Downloading img2mesh checkpoint {model_name}...')
i2m_model.load_state_dict(load_checkpoint(i2m_name, device))
return i2m_model, base_diffusion_i2m
img2mesh_model_name = 'base40M' #'base300M' #'base1B'
i2m_model, base_diffusion_i2m = load_img2mesh_model(img2mesh_model_name)
set_state('Creating upsample model...')
upsampler_model = model_from_config(MODEL_CONFIGS['upsample'], device)
upsampler_model.eval()
upsampler_diffusion = diffusion_from_config(DIFFUSION_CONFIGS['upsample'])
set_state('Downloading upsampler checkpoint...')
upsampler_model.load_state_dict(load_checkpoint('upsample', device))
set_state('Creating SDF model...')
sdf_name = 'sdf'
sdf_model = model_from_config(MODEL_CONFIGS[sdf_name], device)
sdf_model.eval()
set_state('Loading SDF model...')
sdf_model.load_state_dict(load_checkpoint(sdf_name, device))
set_state('')
def get_sampler(model_name, txt2obj, guidance_scale):
global img2mesh_model_name
global base_diffusion_i2m
global i2m_model
if model_name != img2mesh_model_name:
img2mesh_model_name = model_name
i2m_model, base_diffusion_i2m = load_img2mesh_model(model_name)
return PointCloudSampler(
device=device,
models=[t2m_model if txt2obj else i2m_model, upsampler_model],
diffusions=[base_diffusion_t2m if txt2obj else base_diffusion_i2m, upsampler_diffusion],
num_points=[1024, 4096 - 1024],
aux_channels=['R', 'G', 'B'],
guidance_scale=[guidance_scale, 0.0 if txt2obj else guidance_scale],
model_kwargs_key_filter=('texts', '') if txt2obj else ("*",)
)
def generate(model_name, input, guidance_scale, grid_size):
set_state('Entered generate function...')
if isinstance(input, Image.Image):
input = prepare_img(input)
# if input is a string, it's a text prompt
sampler = get_sampler(model_name, txt2obj=True if isinstance(input, str) else False, guidance_scale=guidance_scale)
# Produce a sample from the model.
set_state('Sampling...')
samples = None
kw_args = dict(texts=[input]) if isinstance(input, str) else dict(images=[input])
for x in sampler.sample_batch_progressive(batch_size=1, model_kwargs=kw_args):
samples = x
set_state('Converting to point cloud...')
pc = sampler.output_to_point_clouds(samples)[0]
set_state('Converting to mesh...')
save_ply(pc, 'output.ply', grid_size)
set_state('')
return pc_to_plot(pc), ply_to_obj('output.ply', 'output.obj'), gr.update(value='output.obj', visible=True)
def prepare_img(img):
w, h = img.size
if w > h:
img = img.crop((w - h) / 2, 0, w - (w - h) / 2, h)
else:
img = img.crop((0, (h - w) / 2, w, h - (h - w) / 2))
# resize to 256x256
img = img.resize((256, 256))
return img
def pc_to_plot(pc):
return go.Figure(
data=[
go.Scatter3d(
x=pc.coords[:,0], y=pc.coords[:,1], z=pc.coords[:,2],
mode='markers',
marker=dict(
size=2,
color=['rgb({},{},{})'.format(r,g,b) for r,g,b in zip(pc.channels["R"], pc.channels["G"], pc.channels["B"])],
)
)
],
layout=dict(
scene=dict(xaxis=dict(visible=False), yaxis=dict(visible=False), zaxis=dict(visible=False))
),
)
def ply_to_obj(ply_file, obj_file):
mesh = trimesh.load(ply_file)
mesh.export(obj_file)
return obj_file
def save_ply(pc, file_name, grid_size):
# Produce a mesh (with vertex colors)
mesh = marching_cubes_mesh(
pc=pc,
model=sdf_model,
batch_size=4096,
grid_size=grid_size, # increase to 128 for resolution used in evals
progress=True,
)
# Write the mesh to a PLY file to import into some other program.
with open(file_name, 'wb') as f:
mesh.write_ply(f)
with gr.Blocks() as app:
gr.Markdown("## Point-E text-to-3D Demo")
gr.Markdown("This is a demo for [Point-E: A System for Generating 3D Point Clouds from Complex Prompts](https://arxiv.org/abs/2212.08751) by OpenAI. Check out the [GitHub repo](https://github.com/openai/point-e) for more information.")
with gr.Row():
with gr.Column():
with gr.Tab("Text to 3D"):
prompt = gr.Textbox(label="Prompt", placeholder="A cactus in a pot")
btn_generate_txt2obj = gr.Button(value="Generate")
with gr.Tab("Image to 3D"):
img = gr.Image(label="Image")
btn_generate_img2obj = gr.Button(value="Generate")
with gr.Accordion("Advanced settings", open=False):
dropdown_models = gr.Dropdown(label="Model", value="base40M", choices=["base40M"]) #, "base300M", "base1B"])
guidance_scale = gr.Slider(label="Guidance scale", value=3.0, minimum=3.0, maximum=10.0, step=1.0)
grid_size = gr.Slider(label="Grid size", value=32, minimum=16, maximum=128, step=16)
with gr.Column():
plot = gr.Plot(label="Point cloud")
# btn_pc_to_obj = gr.Button(value="Convert to OBJ", visible=False)
model_3d = gr.Model3D(value=None)
file_out = gr.File(label="Obj file", visible=False)
state_info = state_info = gr.Textbox(label="State", show_label=False).style(container=False)
# inputs = [dropdown_models, prompt, img, guidance_scale, grid_size]
outputs = [plot, model_3d, file_out]
prompt.submit(generate, inputs=[dropdown_models, prompt, guidance_scale, grid_size], outputs=outputs)
btn_generate_txt2obj.click(generate, inputs=[dropdown_models, prompt, guidance_scale, grid_size], outputs=outputs)
btn_generate_img2obj.click(generate, inputs=[dropdown_models, img, guidance_scale, grid_size], outputs=outputs)
# btn_pc_to_obj.click(ply_to_obj, inputs=plot, outputs=[model_3d, file_out])
gr.HTML("""
""")
# app.load(get_state, inputs=[], outputs=state_info, every=0.5, show_progress=False)
app.queue()
# app.launch(debug=True, share=True, height=768)
app.launch(debug=True)
