Interactive visualization

This tutorial introduces user interaction features of the visualizer window.

 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
# examples/python/visualization/interactive_visualization.py

import numpy as np
import copy
import open3d as o3d


def demo_crop_geometry():
    print("Demo for manual geometry cropping")
    print(
        "1) Press 'Y' twice to align geometry with negative direction of y-axis"
    )
    print("2) Press 'K' to lock screen and to switch to selection mode")
    print("3) Drag for rectangle selection,")
    print("   or use ctrl + left click for polygon selection")
    print("4) Press 'C' to get a selected geometry")
    print("5) Press 'S' to save the selected geometry")
    print("6) Press 'F' to switch to freeview mode")
    pcd_data = o3d.data.DemoICPPointClouds()
    pcd = o3d.io.read_point_cloud(pcd_data.paths[0])
    o3d.visualization.draw_geometries_with_editing([pcd])


def draw_registration_result(source, target, transformation):
    source_temp = copy.deepcopy(source)
    target_temp = copy.deepcopy(target)
    source_temp.paint_uniform_color([1, 0.706, 0])
    target_temp.paint_uniform_color([0, 0.651, 0.929])
    source_temp.transform(transformation)
    o3d.visualization.draw_geometries([source_temp, target_temp])


def pick_points(pcd):
    print("")
    print(
        "1) Please pick at least three correspondences using [shift + left click]"
    )
    print("   Press [shift + right click] to undo point picking")
    print("2) After picking points, press 'Q' to close the window")
    vis = o3d.visualization.VisualizerWithEditing()
    vis.create_window()
    vis.add_geometry(pcd)
    vis.run()  # user picks points
    vis.destroy_window()
    print("")
    return vis.get_picked_points()


def demo_manual_registration():
    print("Demo for manual ICP")
    pcd_data = o3d.data.DemoICPPointClouds()
    source = o3d.io.read_point_cloud(pcd_data.paths[0])
    target = o3d.io.read_point_cloud(pcd_data.paths[2])
    print("Visualization of two point clouds before manual alignment")
    draw_registration_result(source, target, np.identity(4))

    # pick points from two point clouds and builds correspondences
    picked_id_source = pick_points(source)
    picked_id_target = pick_points(target)
    assert (len(picked_id_source) >= 3 and len(picked_id_target) >= 3)
    assert (len(picked_id_source) == len(picked_id_target))
    corr = np.zeros((len(picked_id_source), 2))
    corr[:, 0] = picked_id_source
    corr[:, 1] = picked_id_target

    # estimate rough transformation using correspondences
    print("Compute a rough transform using the correspondences given by user")
    p2p = o3d.pipelines.registration.TransformationEstimationPointToPoint()
    trans_init = p2p.compute_transformation(source, target,
                                            o3d.utility.Vector2iVector(corr))

    # point-to-point ICP for refinement
    print("Perform point-to-point ICP refinement")
    threshold = 0.03  # 3cm distance threshold
    reg_p2p = o3d.pipelines.registration.registration_icp(
        source, target, threshold, trans_init,
        o3d.pipelines.registration.TransformationEstimationPointToPoint())
    draw_registration_result(source, target, reg_p2p.transformation)
    print("")


if __name__ == "__main__":
    demo_crop_geometry()
    demo_manual_registration()

This script executes two applications of user interaction: demo_crop_geometry and demo_manual_registration.

Crop geometry

37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
        "1) Press 'Y' twice to align geometry with negative direction of y-axis"
    )
    print("2) Press 'K' to lock screen and to switch to selection mode")
    print("3) Drag for rectangle selection,")
    print("   or use ctrl + left click for polygon selection")
    print("4) Press 'C' to get a selected geometry")
    print("5) Press 'S' to save the selected geometry")
    print("6) Press 'F' to switch to freeview mode")
    pcd_data = o3d.data.DemoICPPointClouds()
    pcd = o3d.io.read_point_cloud(pcd_data.paths[0])
    o3d.visualization.draw_geometries_with_editing([pcd])


def draw_registration_result(source, target, transformation):
    source_temp = copy.deepcopy(source)

This function simply reads a point cloud and calls draw_geometries_with_editing. This function provides vertex selection and cropping.

Note

Open3D has a VisualizerWithEditing class that inherits Visualizer class. It adds graphic user interaction features. Likewise examples in Customized visualization, VisualizerWithEditing() can be explicitly used instead of draw_geometries_with_editing([pcd]).

Once a geometry is displayed, press Y twice to align geometry with negative direction of y-axis. After adjusting viewing orientation, press K to lock screen and to switch to the selection mode.

../../_images/crop_lock.png

Tip

The practical step for selecting area is to align the geometry with arbitrary axis using orthographic projection model. This trick makes selection easier, because it avoids self-occlusion hassle due to perspective projection.

To select a region, use either mouse drag (rectangle selection) or ctrl + left mouse click (polygon selection). The below example shows a selected area using a polygon.

../../_images/crop_selection.png

Note that the selected area is dark shaded. To keep the selected area and discard the rest, press C. A dialog box appears, which can be used to save the cropped geometry. The cropping result is shown after saving.

../../_images/crop_save.png ../../_images/crop_result.png

To finish selection mode, press F to switch to freeview mode.

../../_images/crop_freeview.png

Manual registration

Select correspondences

The following script registers two point clouds using point-to-point ICP. It gets initial alignment via user interaction.

61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
    print(
        "1) Please pick at least three correspondences using [shift + left click]"
    )
    print("   Press [shift + right click] to undo point picking")
    print("2) After picking points, press 'Q' to close the window")
    vis = o3d.visualization.VisualizerWithEditing()
    vis.create_window()
    vis.add_geometry(pcd)
    vis.run()  # user picks points
    vis.destroy_window()
    print("")
    return vis.get_picked_points()


def demo_manual_registration():
    print("Demo for manual ICP")

The script reads two point clouds, and visualizes the point clouds before alignment.

../../_images/manual_icp_initial.png 
52
53
54
55
56
57
58
59
60
    target_temp = copy.deepcopy(target)
    source_temp.paint_uniform_color([1, 0.706, 0])
    target_temp.paint_uniform_color([0, 0.651, 0.929])
    source_temp.transform(transformation)
    o3d.visualization.draw_geometries([source_temp, target_temp])


def pick_points(pcd):
    print("")

The function pick_points(pcd) makes an instance of VisualizerWithEditing. To mimic draw_geometries, it creates windows, adds the geometry, visualizes the geometry, and then terminates. A novel interface function from VisualizerWithEditing is get_picked_points() that returns the indices of user-picked vertices.

To pick a vertex, press shift + left click on a window. If a vertex is selected, the visualizer window overlays a sphere on a selected vertex. For example, after picking three vertices in the source point cloud, it shows:

../../_images/manual_icp_source.png

This will print:

Picked point #58481 (2.14, 1.56, 1.53) to add in queue.
Picked point #77321 (2.86, 1.92, 1.09) to add in queue.
Picked point #42639 (3.28, 1.53, 1.45) to add in queue.

Press Q to close the window. The next step is to pick the same correspondences in the target point cloud. The color of the sphere helps to identify the same correspondence.

../../_images/manual_icp_target.png

This will print:

Picked point #54028 (1.62, 1.81, 1.23) to add in queue.
Picked point #97115 (2.45, 2.19, 1.11) to add in queue.
Picked point #47467 (2.75, 1.71, 1.45) to add in queue.

Tip

To get a good registration result, try to pick more than three points that are well-distributed in the scene. Using a vertex in the corner region is a good way to easily pick the right correspondence.

Registration using user correspondences

 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
    pcd_data = o3d.data.DemoICPPointClouds()
    source = o3d.io.read_point_cloud(pcd_data.paths[0])
    target = o3d.io.read_point_cloud(pcd_data.paths[2])
    print("Visualization of two point clouds before manual alignment")
    draw_registration_result(source, target, np.identity(4))

    # pick points from two point clouds and builds correspondences
    picked_id_source = pick_points(source)
    picked_id_target = pick_points(target)
    assert (len(picked_id_source) >= 3 and len(picked_id_target) >= 3)
    assert (len(picked_id_source) == len(picked_id_target))
    corr = np.zeros((len(picked_id_source), 2))
    corr[:, 0] = picked_id_source
    corr[:, 1] = picked_id_target

    # estimate rough transformation using correspondences
    print("Compute a rough transform using the correspondences given by user")
    p2p = o3d.pipelines.registration.TransformationEstimationPointToPoint()
    trans_init = p2p.compute_transformation(source, target,
                                            o3d.utility.Vector2iVector(corr))

    # point-to-point ICP for refinement
    print("Perform point-to-point ICP refinement")
    threshold = 0.03  # 3cm distance threshold
    reg_p2p = o3d.pipelines.registration.registration_icp(
        source, target, threshold, trans_init,
        o3d.pipelines.registration.TransformationEstimationPointToPoint())
    draw_registration_result(source, target, reg_p2p.transformation)
    print("")


if __name__ == "__main__":
    demo_crop_geometry()
    demo_manual_registration()

The later part of the demo computes an initial transformation based on the user-provided correspondences. This script builds pairs of correspondences using Vector2iVector(corr). It utilizes TransformationEstimationPointToPoint.compute_transformation to compute the initial transformation from the correspondences. The initial transformation is refined using registration_icp.

The registration result is as follows:

../../_images/manual_icp_alignment.png