# GPU Installation on a CUDA 11.6 Machine
conda create -n learning3d python=3.10
pip install torch --index-url https://download.pytorch.org/whl/cu116 (modify according to your cuda version)
pip install fvcore iopath
pip install "git+https://github.com/facebookresearch/pytorch3d.git@stable" (this will take some time to compile)
pip install -r requirements.txt
# CPU Installation
conda create -n learning3d python=3.10
pip install torch --index-url https://download.pytorch.org/whl/cpu
pip install fvcore iopath
pip install "git+https://github.com/facebookresearch/pytorch3d.git@stable"
pip install -r requirements.txt

def render_setup(filepath="data/cow.obj", image_size=256, color1=None,color2=None,
Camera_R=None,Camera_T=None,device=None,savepath='01setup.jpg',record=True):
# The device tells us whether we are rendering with GPU or CPU. The rendering will
# be *much* faster if you have a CUDA-enabled NVIDIA GPU. However, your code will
# still run fine on a CPU.
# The default is to run on CPU, so if you do not have a GPU, you do not need to
# worry about specifying the device in all of these functions.
if device is None:
device = get_device()
# Get the renderer.
renderer = get_mesh_renderer(image_size=image_size)
# Get the vertices, faces, and textures.
vertices, faces = load_cow_mesh(filepath)
vertices = vertices.unsqueeze(0)  # (N_v, 3) -> (1, N_v, 3)
faces = faces.unsqueeze(0)  # (N_f, 3) -> (1, N_f, 3)
assert(color1 is not None)
if color1 and color2:
color1=varying_color(vertices,color1,color2)
textures = torch.ones_like(vertices)  # (1, N_v, 3)
textures = textures * torch.tensor(color1)  # (1, N_v, 3)
mesh = pytorch3d.structures.Meshes(
verts=vertices,
faces=faces,
textures=pytorch3d.renderer.TexturesVertex(textures),
)
mesh = mesh.to(device)
print(torch.eye(3).unsqueeze(0))
print(Camera_R)
print(Camera_T)
# Prepare the camera:
cameras = pytorch3d.renderer.FoVPerspectiveCameras(
R=torch.eye(3).unsqueeze(0) if Camera_R is None else Camera_R, T=torch.tensor([[0, 0, 3]] if Camera_T is None else Camera_T),
fov=60, device=device)
# Place a point light in front of the cow.
lights = pytorch3d.renderer.PointLights(location=[[0, 0, -3]], device=device)
rend = renderer(mesh, cameras=cameras, lights=lights)
rend = rend.cpu().numpy()[0, ..., :3]  # (B, H, W, 4) -> (H, W, 3)
# The .cpu moves the tensor to GPU (if needed).
if record:
if '/' in savepath:
dir = ''.join(savepath.split('/')[:-1])
os.makedirs(dir, exist_ok=True)
os.chdir(dir)
savepath = savepath.split('/')[-1]
if Camera_R is not None and Camera_T is not None:
plt.imsave(f'{Camera_R[0].flatten().numpy(),Camera_T[0].numpy()}'+savepath,numpy.uint8(rend*255))
else:
plt.imsave(savepath,numpy.uint8(rend*255))
return rend

def varying_color(vertices,color1,color2):
assert(vertices.shape[0]==1)
z = vertices[0, :, 2]
z_min = torch.min(vertices[0, :, 2])
z_max = torch.max(vertices[0, :, 2])
color1 = torch.tensor(color1).view(1, 3)
color2 = torch.tensor(color2).view(1, 3)
alpha = (z - z_min) / (z_max - z_min)
var_color = torch.matmul(alpha.reshape(-1,1), color2) + torch.matmul(1 - alpha.reshape(-1, 1), color1)
var_color=var_color.unsqueeze(0)
assert(var_color.shape==vertices.shape)
return var_color

def render_setup_from_pointcloud(filepath, image_size=256,
Camera_R=None,Camera_T=None,device=None,savepath='01setup.jpg',record=True):
if device is None:
device = get_device()
# Get the renderer.
renderer = get_points_renderer(image_size=image_size,radius=0.01)
# Get the vertices, faces, and textures.
data = load_rgbd_data(filepath)
print(data.keys())
# Prepare the camera:
camera_fixed = pytorch3d.renderer.FoVPerspectiveCameras(
R=torch.eye(3).unsqueeze(0), T=torch.tensor([[0, 0, 3]]),
fov=60, device=device)
points,rgbs=unproject_depth_image(torch.tensor(data["rgb1"]),torch.tensor(data["mask1"]),
torch.tensor(data["depth1"]),camera_fixed)
points2,rgbs2=unproject_depth_image(torch.tensor(data["rgb2"]),torch.tensor(data["mask2"]),
torch.tensor(data["depth2"]),camera_fixed)
pcloud = pytorch3d.structures.Pointclouds(
points=points.unsqueeze(0),
features=rgbs.unsqueeze(0)
)
pcloud = pcloud.to(device)
pcloud2 = pytorch3d.structures.Pointclouds(
points=points2.unsqueeze(0),
features=rgbs2.unsqueeze(0)
)
pcloud2 = pcloud2.to(device)
pcloud_cb = pytorch3d.structures.Pointclouds(
points=torch.cat([points,points2],dim=0).unsqueeze(0),
features=torch.cat([rgbs,rgbs2],dim=0).unsqueeze(0)
)
pcloud_cb = pcloud_cb.to(device)
pclouds=[pcloud,pcloud2,pcloud_cb]
rends=[]
# Place a point light in front of the cow.
lights = pytorch3d.renderer.PointLights(location=[[0, 0, -3]], device=device)
camera_dynamic=pytorch3d.renderer.FoVPerspectiveCameras(
R=torch.eye(3).unsqueeze(0) if Camera_R is None else Camera_R,
T=torch.tensor([[0, 0, 3]] if Camera_T is None else Camera_T),
fov=60, device=device)
for i,pcloud in enumerate(pclouds):
rend = renderer(pcloud, cameras=camera_dynamic, lights=lights)
rend = rend.cpu().numpy()[0, ..., :3]  # (B, H, W, 4) -> (H, W, 3)
# The .cpu moves the tensor to GPU (if needed).
if record:
if '/' in savepath:
dir=''.join(savepath.split('/')[:-1])
os.makedirs(dir,exist_ok=True)
os.chdir(dir)
savepath=savepath.split('/')[-1]
if Camera_R is not None and Camera_T is not None:
plt.imsave(f'{i+1}'+f'{float(Camera_R[0].numpy().sum()), float(Camera_T[0].numpy().sum())}'+savepath , numpy.uint8(rend * 255))
else:
plt.imsave(f'{i+1}'+savepath, numpy.uint8(rend * 255))
rends.append(rend)
return rends

def gif_360(n_render,color1,color2,savepath):
current_dir=os.getcwd()
my_images = []
for i in range(0, n_render):
R, T = pytorch3d.renderer.cameras.look_at_view_transform(dist=3, azim=180 + 360 * i / n_render)
image = render_setup(filepath=args.cow_path, image_size=args.image_size,color1=color1,color2=color2,
Camera_R=R, Camera_T=T,record=False)
my_images.append(numpy.uint8(image[:, :, :] * 255))
print(i, "/", n_render)
if '/' in savepath:
os.makedirs(''.join(savepath.split('/')[:-1]),exist_ok=True)
os.chdir(current_dir)
imageio.mimwrite(savepath, my_images, fps=24)

def gif_360_pcloud(n_render,savepath):
if savepath.split('.')[-1]!='gif':
raise ValueError("Savepath should be only in the format of gif.")
current_dir=os.getcwd()
my_images = []
for i in range(0, n_render):
R, T = pytorch3d.renderer.cameras.look_at_view_transform(dist=4, azim=180 + 360 * i / n_render)
images = render_setup_from_pointcloud(filepath=args.bridge_path, image_size=args.image_size,
Camera_R=R, Camera_T=T,savepath=savepath.replace("gif","jpg"),record=False)
for img in images:
my_images.append(numpy.uint8(img[:, :, :] * 255))
print(i, "/", n_render)
os.chdir(current_dir)
imageio.mimwrite(savepath, my_images, fps=24)
return my_images