Open3D (C++ API)  0.18.0+252c867
FilamentScene.h
Go to the documentation of this file.
1 // ----------------------------------------------------------------------------
2 // - Open3D: www.open3d.org -
3 // ----------------------------------------------------------------------------
4 // Copyright (c) 2018-2023 www.open3d.org
5 // SPDX-License-Identifier: MIT
6 // ----------------------------------------------------------------------------
7 
8 #pragma once
9 
10 // 4068: Filament has some clang-specific vectorizing pragma's that MSVC flags
11 // 4146: Filament's utils/algorithm.h utils::details::ctz() tries to negate
12 // an unsigned int.
13 // 4293: Filament's utils/algorithm.h utils::details::clz() does strange
14 // things with MSVC. Somehow sizeof(unsigned int) > 4, but its size is
15 // 32 so that x >> 32 gives a warning. (Or maybe the compiler can't
16 // determine the if statement does not run.)
17 // 4305: LightManager.h needs to specify some constants as floats
18 #ifdef _MSC_VER
19 #pragma warning(push)
20 #pragma warning(disable : 4068 4146 4293 4305)
21 #endif // _MSC_VER
22 
23 #include <filament/LightManager.h>
24 #include <filament/RenderableManager.h>
25 #include <utils/Entity.h>
26 
27 #ifdef _MSC_VER
28 #pragma warning(pop)
29 #endif // _MSC_VER
30 
31 #include <Eigen/Geometry>
32 #include <unordered_map>
33 #include <vector>
34 
41 
43 namespace filament {
44 class Box;
45 class Engine;
46 class IndirectLight;
47 class Renderer;
48 class Scene;
49 class Skybox;
50 class TransformManager;
51 class VertexBuffer;
52 } // namespace filament
54 
55 namespace open3d {
56 namespace visualization {
57 namespace rendering {
58 
59 class FilamentView;
60 class GeometryBuffersBuilder;
61 class Renderer;
62 class View;
63 
64 // Contains renderable objects like geometry and lights
65 // Can have multiple views
66 class FilamentScene : public Scene {
67 public:
68  using Transform = Eigen::Transform<float, 3, Eigen::Affine>;
69 
70  FilamentScene(filament::Engine& engine,
71  FilamentResourceManager& resource_mgr,
72  Renderer& renderer);
74 
75  Scene* Copy() override;
76 
77  // NOTE: Temporarily needed to support old View interface for ImGUI
79  std::int32_t y,
80  std::uint32_t w,
81  std::uint32_t h) override;
82 
83  View* GetView(const ViewHandle& view_id) const override;
84  void SetViewActive(const ViewHandle& view_id, bool is_active) override;
85  void SetRenderOnce(const ViewHandle& view_id) override;
86  void RemoveView(const ViewHandle& view_id) override;
87 
88  // Camera
89  void AddCamera(const std::string& camera_name,
90  std::shared_ptr<Camera> cam) override;
91  void RemoveCamera(const std::string& camera_name) override;
92  void SetActiveCamera(const std::string& camera_name) override;
93 
94  // Scene geometry
95  bool AddGeometry(const std::string& object_name,
96  const geometry::Geometry3D& geometry,
97  const MaterialRecord& material,
98  const std::string& downsampled_name = "",
99  size_t downsample_threshold = SIZE_MAX) override;
100  bool AddGeometry(const std::string& object_name,
101  const t::geometry::Geometry& geometry,
102  const MaterialRecord& material,
103  const std::string& downsampled_name = "",
104  size_t downsample_threshold = SIZE_MAX) override;
105  bool AddGeometry(const std::string& object_name,
106  const TriangleMeshModel& model) override;
107  bool HasGeometry(const std::string& object_name) const override;
108  void UpdateGeometry(const std::string& object_name,
109  const t::geometry::PointCloud& point_cloud,
110  uint32_t update_flags) override;
111  void RemoveGeometry(const std::string& object_name) override;
112  void ShowGeometry(const std::string& object_name, bool show) override;
113  bool GeometryIsVisible(const std::string& object_name) override;
114  void SetGeometryTransform(const std::string& object_name,
115  const Transform& transform) override;
116  Transform GetGeometryTransform(const std::string& object_name) override;
118  const std::string& object_name) override;
119  void GeometryShadows(const std::string& object_name,
120  bool cast_shadows,
121  bool receive_shadows) override;
122  void SetGeometryCulling(const std::string& object_name,
123  bool enable) override;
124  void SetGeometryPriority(const std::string& object_name,
125  uint8_t priority) override;
126  void OverrideMaterial(const std::string& object_name,
127  const MaterialRecord& material) override;
128  void QueryGeometry(std::vector<std::string>& geometry) override;
129 
130  void OverrideMaterialAll(const MaterialRecord& material,
131  bool shader_only = true) override;
132 
133  // Lighting Environment
134  bool AddPointLight(const std::string& light_name,
135  const Eigen::Vector3f& color,
136  const Eigen::Vector3f& position,
137  float intensity,
138  float falloff,
139  bool cast_shadows) override;
140  bool AddSpotLight(const std::string& light_name,
141  const Eigen::Vector3f& color,
142  const Eigen::Vector3f& position,
143  const Eigen::Vector3f& direction,
144  float intensity,
145  float falloff,
146  float inner_cone_angle,
147  float outer_cone_angle,
148  bool cast_shadows) override;
149  bool AddDirectionalLight(const std::string& light_name,
150  const Eigen::Vector3f& color,
151  const Eigen::Vector3f& direction,
152  float intensity,
153  bool cast_shadows) override;
154  Light& GetLight(const std::string& light_name) override;
155  void RemoveLight(const std::string& light_name) override;
156  void UpdateLight(const std::string& light_name,
157  const Light& light) override;
158  void UpdateLightColor(const std::string& light_name,
159  const Eigen::Vector3f& color) override;
160  void UpdateLightPosition(const std::string& light_name,
161  const Eigen::Vector3f& position) override;
162  void UpdateLightDirection(const std::string& light_name,
163  const Eigen::Vector3f& direction) override;
164  void UpdateLightIntensity(const std::string& light_name,
165  float intensity) override;
166  void UpdateLightFalloff(const std::string& light_name,
167  float falloff) override;
168  void UpdateLightConeAngles(const std::string& light_name,
169  float inner_cone_angle,
170  float outer_cone_angle) override;
171  void EnableLightShadow(const std::string& light_name,
172  bool cast_shadows) override;
173 
174  void SetSunLight(const Eigen::Vector3f& direction,
175  const Eigen::Vector3f& color,
176  float intensity) override;
177  void EnableSunLight(bool enable) override;
178  void EnableSunLightShadows(bool enable) override;
179  void SetSunLightColor(const Eigen::Vector3f& color) override;
180  Eigen::Vector3f GetSunLightColor() override;
181  void SetSunLightIntensity(float intensity) override;
182  float GetSunLightIntensity() override;
183  void SetSunLightDirection(const Eigen::Vector3f& direction) override;
184  Eigen::Vector3f GetSunLightDirection() override;
185  void SetSunAngularRadius(float radius) override;
186  void SetSunHaloSize(float size) override;
187  void SetSunHaloFalloff(float falloff) override;
188 
189  bool SetIndirectLight(const std::string& ibl_name) override;
190  const std::string& GetIndirectLight() override;
191  void EnableIndirectLight(bool enable) override;
192  void SetIndirectLightIntensity(float intensity) override;
193  float GetIndirectLightIntensity() override;
194  void SetIndirectLightRotation(const Transform& rotation) override;
196  void ShowSkybox(bool show) override;
197  bool GetSkyboxVisible() const override;
198  void SetBackground(
199  const Eigen::Vector4f& color,
200  const std::shared_ptr<geometry::Image> image = nullptr) override;
201  void SetBackground(TextureHandle image) override;
202  void EnableGroundPlane(bool enable, GroundPlane plane) override;
203  void SetGroundPlaneColor(const Eigen::Vector4f& color) override;
204 
205  void RenderToImage(std::function<void(std::shared_ptr<geometry::Image>)>
206  callback) override;
207  void RenderToDepthImage(
208  std::function<void(std::shared_ptr<geometry::Image>)> callback)
209  override;
210 
211  void Draw(filament::Renderer& renderer);
212 
213  // NOTE: This method is to work around Filament limitation when rendering to
214  // depth buffer. Materials with SSR require multiple passes which causes a
215  // crash with render to depth since we must disable multiple passes (i.e.,
216  // post-processing) in order to get back valid, un-modified depth values.
217  void HideRefractedMaterials(bool hide = true);
218 
219  // NOTE: Can GetNativeScene be removed?
220  filament::Scene* GetNativeScene() const { return scene_; }
221 
222 private:
223  MaterialInstanceHandle AssignMaterialToFilamentGeometry(
224  filament::RenderableManager::Builder& builder,
225  const MaterialRecord& material);
226  enum BufferReuse { kNo, kYes };
227  bool CreateAndAddFilamentEntity(
228  const std::string& object_name,
229  GeometryBuffersBuilder& buffer_builder,
230  filament::Box& aabb,
233  const MaterialRecord& material,
234  BufferReuse reusing_vertex_buffer = BufferReuse::kNo);
235 
236  filament::Engine& engine_;
237  FilamentResourceManager& resource_mgr_;
238  filament::Scene* scene_ = nullptr;
239 
240  struct TextureMaps {
241  rendering::TextureHandle albedo_map =
243  rendering::TextureHandle normal_map =
245  rendering::TextureHandle ao_rough_metal_map =
247  rendering::TextureHandle reflectance_map =
249  rendering::TextureHandle clear_coat_map =
251  rendering::TextureHandle clear_coat_roughness_map =
253  rendering::TextureHandle anisotropy_map =
255  rendering::TextureHandle gradient_texture =
257  };
258 
259  struct GeometryMaterialInstance {
260  TextureMaps maps;
261  MaterialRecord properties;
262  MaterialInstanceHandle mat_instance;
263  };
264 
265  struct RenderableGeometry {
266  std::string name;
267  bool visible = true;
268  bool was_hidden_before_picking = false;
269  bool cast_shadows = true;
270  bool receive_shadows = true;
271  bool culling_enabled = true;
272  int priority = -1; // default priority
273 
274  GeometryMaterialInstance mat;
275 
276  // Filament resources
277  utils::Entity filament_entity;
278  filament::RenderableManager::PrimitiveType primitive_type;
281  void ReleaseResources(filament::Engine& engine,
282  FilamentResourceManager& manager);
283  };
284 
285  struct LightEntity {
286  bool enabled = true;
287  utils::Entity filament_entity;
288  };
289 
290  // NOTE: ViewContainer and views_ are temporary
291  struct ViewContainer {
292  std::unique_ptr<FilamentView> view;
293  bool is_active = true;
294  int render_count = -1;
295  };
296  std::unordered_map<REHandle_abstract, ViewContainer> views_;
297 
298  std::vector<RenderableGeometry*> GetGeometry(const std::string& object_name,
299  bool warn_if_not_found = true);
300  bool GeometryIsModel(const std::string& object_name) const;
301  LightEntity* GetLightInternal(const std::string& light_name,
302  bool warn_if_not_found = true);
303  void OverrideMaterialInternal(RenderableGeometry* geom,
304  const MaterialRecord& material,
305  bool shader_only = false);
306  void UpdateMaterialProperties(RenderableGeometry& geom);
307  void UpdateDefaultLit(GeometryMaterialInstance& geom_mi);
308  void UpdateDefaultLitSSR(GeometryMaterialInstance& geom_mi);
309  void UpdateDefaultUnlit(GeometryMaterialInstance& geom_mi);
310  void UpdateNormalShader(GeometryMaterialInstance& geom_mi);
311  void UpdateDepthShader(GeometryMaterialInstance& geom_mi);
312  void UpdateDepthValueShader(GeometryMaterialInstance& geom_mi);
313  void UpdateGradientShader(GeometryMaterialInstance& geom_mi);
314  void UpdateSolidColorShader(GeometryMaterialInstance& geom_mi);
315  void UpdateBackgroundShader(GeometryMaterialInstance& geom_mi);
316  void UpdateGroundPlaneShader(GeometryMaterialInstance& geom_mi);
317  void UpdateLineShader(GeometryMaterialInstance& geom_mi);
318  void UpdateUnlitPolygonOffsetShader(GeometryMaterialInstance& geom_mi);
319  utils::EntityInstance<filament::TransformManager>
320  GetGeometryTransformInstance(RenderableGeometry* geom);
321  void CreateSunDirectionalLight();
322  void CreateBackgroundGeometry();
323  void CreateGroundPlaneGeometry();
324 
325  std::unordered_map<std::string, RenderableGeometry> geometries_;
326  std::unordered_map<std::string, LightEntity> lights_;
327  std::unordered_map<std::string, std::vector<std::string>> model_geometries_;
328 
329  Eigen::Vector4f background_color_;
330  std::shared_ptr<geometry::Image> background_image_;
331  std::string ibl_name_;
332  bool ibl_enabled_ = false;
333  bool skybox_enabled_ = false;
334  std::weak_ptr<filament::IndirectLight> indirect_light_;
335  std::weak_ptr<filament::Skybox> skybox_;
336  IndirectLightHandle ibl_handle_;
337  SkyboxHandle skybox_handle_;
338  LightEntity sun_;
339 };
340 
341 } // namespace rendering
342 } // namespace visualization
343 } // namespace open3d
std::shared_ptr< core::Tensor > image
Definition: FilamentRenderer.cpp:183
std::function< void(std::shared_ptr< core::Tensor >)> callback
Definition: FilamentRenderer.cpp:182
math::float4 color
Definition: LineSetBuffers.cpp:45
math::float3 position
Definition: LineSetBuffers.cpp:43
std::vector< UVAtlasVertex > vb
Definition: UVUnwrapping.cpp:27
std::vector< uint8_t > ib
Definition: UVUnwrapping.cpp:29
A bounding box that is aligned along the coordinate axes and defined by the min_bound and max_bound.
Definition: BoundingVolume.h:160
The base geometry class for 3D geometries.
Definition: Geometry3D.h:28
The base geometry class.
Definition: Geometry.h:21
A point cloud contains a list of 3D points.
Definition: PointCloud.h:80
Definition: FilamentResourceManager.h:50
static const TextureHandle kDefaultNormalMap
Definition: FilamentResourceManager.h:71
static const TextureHandle kDefaultTexture
Definition: FilamentResourceManager.h:69
void UpdateLightDirection(const std::string &light_name, const Eigen::Vector3f &direction) override
Definition: FilamentScene.cpp:1378
Eigen::Vector3f GetSunLightColor() override
Definition: FilamentScene.cpp:1499
void UpdateLightIntensity(const std::string &light_name, float intensity) override
Definition: FilamentScene.cpp:1390
const std::string & GetIndirectLight() override
Definition: FilamentScene.cpp:1586
void EnableIndirectLight(bool enable) override
Definition: FilamentScene.cpp:1588
~FilamentScene()
Definition: FilamentScene.cpp:158
void SetActiveCamera(const std::string &camera_name) override
Definition: FilamentScene.cpp:288
void SetSunLightColor(const Eigen::Vector3f &color) override
Definition: FilamentScene.cpp:1492
void RenderToDepthImage(std::function< void(std::shared_ptr< geometry::Image >)> callback) override
Size of image is the size of the window.
Definition: FilamentScene.cpp:1819
void SetGeometryTransform(const std::string &object_name, const Transform &transform) override
Definition: FilamentScene.cpp:726
Eigen::Vector3f GetSunLightDirection() override
Definition: FilamentScene.cpp:1535
bool AddPointLight(const std::string &light_name, const Eigen::Vector3f &color, const Eigen::Vector3f &position, float intensity, float falloff, bool cast_shadows) override
Definition: FilamentScene.cpp:1220
void SetSunLightDirection(const Eigen::Vector3f &direction) override
Definition: FilamentScene.cpp:1507
void Draw(filament::Renderer &renderer)
Definition: FilamentScene.cpp:1897
void HideRefractedMaterials(bool hide=true)
Definition: FilamentScene.cpp:1913
void RenderToImage(std::function< void(std::shared_ptr< geometry::Image >)> callback) override
Size of image is the size of the window.
Definition: FilamentScene.cpp:1813
void EnableSunLight(bool enable) override
Definition: FilamentScene.cpp:1460
void SetBackground(const Eigen::Vector4f &color, const std::shared_ptr< geometry::Image > image=nullptr) override
Definition: FilamentScene.cpp:1679
void UpdateLightColor(const std::string &light_name, const Eigen::Vector3f &color) override
Definition: FilamentScene.cpp:1353
Transform GetIndirectLightRotation() override
Definition: FilamentScene.cpp:1621
void EnableLightShadow(const std::string &light_name, bool cast_shadows) override
Definition: FilamentScene.cpp:1418
bool AddDirectionalLight(const std::string &light_name, const Eigen::Vector3f &color, const Eigen::Vector3f &direction, float intensity, bool cast_shadows) override
Definition: FilamentScene.cpp:1298
void SetSunLightIntensity(float intensity) override
Definition: FilamentScene.cpp:1478
geometry::AxisAlignedBoundingBox GetGeometryBoundingBox(const std::string &object_name) override
Definition: FilamentScene.cpp:756
void UpdateLightPosition(const std::string &light_name, const Eigen::Vector3f &position) override
Definition: FilamentScene.cpp:1364
Transform GetGeometryTransform(const std::string &object_name) override
Definition: FilamentScene.cpp:741
void RemoveGeometry(const std::string &object_name) override
Definition: FilamentScene.cpp:671
bool SetIndirectLight(const std::string &ibl_name) override
Definition: FilamentScene.cpp:1543
void SetGroundPlaneColor(const Eigen::Vector4f &color) override
Definition: FilamentScene.cpp:1787
bool GeometryIsVisible(const std::string &object_name) override
Definition: FilamentScene.cpp:700
Light & GetLight(const std::string &light_name) override
Definition: FilamentScene.cpp:1333
void SetIndirectLightIntensity(float intensity) override
Definition: FilamentScene.cpp:1601
ViewHandle AddView(std::int32_t x, std::int32_t y, std::uint32_t w, std::uint32_t h) override
Definition: FilamentScene.cpp:235
void SetSunAngularRadius(float radius) override
Definition: FilamentScene.cpp:1514
float GetSunLightIntensity() override
Definition: FilamentScene.cpp:1485
void UpdateLightFalloff(const std::string &light_name, float falloff) override
Definition: FilamentScene.cpp:1401
void EnableSunLightShadows(bool enable) override
Definition: FilamentScene.cpp:1471
void RemoveView(const ViewHandle &view_id) override
Definition: FilamentScene.cpp:279
bool GetSkyboxVisible() const override
Definition: FilamentScene.cpp:1645
View * GetView(const ViewHandle &view_id) const override
Definition: FilamentScene.cpp:254
void OverrideMaterialAll(const MaterialRecord &material, bool shader_only=true) override
Definition: FilamentScene.cpp:1210
void SetRenderOnce(const ViewHandle &view_id) override
Definition: FilamentScene.cpp:271
Scene * Copy() override
Definition: FilamentScene.cpp:174
void AddCamera(const std::string &camera_name, std::shared_ptr< Camera > cam) override
Definition: FilamentScene.cpp:283
void SetGeometryCulling(const std::string &object_name, bool enable) override
Definition: FilamentScene.cpp:791
void EnableGroundPlane(bool enable, GroundPlane plane) override
Definition: FilamentScene.cpp:1768
void SetSunHaloSize(float size) override
Definition: FilamentScene.cpp:1521
filament::Scene * GetNativeScene() const
Definition: FilamentScene.h:220
void ShowSkybox(bool show) override
Definition: FilamentScene.cpp:1631
FilamentScene(filament::Engine &engine, FilamentResourceManager &resource_mgr, Renderer &renderer)
Definition: FilamentScene.cpp:147
bool AddGeometry(const std::string &object_name, const geometry::Geometry3D &geometry, const MaterialRecord &material, const std::string &downsampled_name="", size_t downsample_threshold=SIZE_MAX) override
Definition: FilamentScene.cpp:305
void UpdateGeometry(const std::string &object_name, const t::geometry::PointCloud &point_cloud, uint32_t update_flags) override
Definition: FilamentScene.cpp:537
void SetGeometryPriority(const std::string &object_name, uint8_t priority) override
Definition: FilamentScene.cpp:803
void SetViewActive(const ViewHandle &view_id, bool is_active) override
Definition: FilamentScene.cpp:263
void ShowGeometry(const std::string &object_name, bool show) override
Definition: FilamentScene.cpp:686
void RemoveLight(const std::string &light_name) override
Definition: FilamentScene.cpp:1344
void SetSunHaloFalloff(float falloff) override
Definition: FilamentScene.cpp:1528
void QueryGeometry(std::vector< std::string > &geometry) override
Definition: FilamentScene.cpp:1204
void SetSunLight(const Eigen::Vector3f &direction, const Eigen::Vector3f &color, float intensity) override
Definition: FilamentScene.cpp:1449
void SetIndirectLightRotation(const Transform &rotation) override
Definition: FilamentScene.cpp:1614
float GetIndirectLightIntensity() override
Definition: FilamentScene.cpp:1607
void GeometryShadows(const std::string &object_name, bool cast_shadows, bool receive_shadows) override
Definition: FilamentScene.cpp:778
Eigen::Transform< float, 3, Eigen::Affine > Transform
Definition: FilamentScene.h:68
void UpdateLightConeAngles(const std::string &light_name, float inner_cone_angle, float outer_cone_angle) override
Definition: FilamentScene.cpp:1412
void RemoveCamera(const std::string &camera_name) override
Definition: FilamentScene.cpp:286
bool HasGeometry(const std::string &object_name) const override
Definition: FilamentScene.cpp:529
void UpdateLight(const std::string &light_name, const Light &light) override
Definition: FilamentScene.cpp:1339
void OverrideMaterial(const std::string &object_name, const MaterialRecord &material) override
Definition: FilamentScene.cpp:1196
bool AddSpotLight(const std::string &light_name, const Eigen::Vector3f &color, const Eigen::Vector3f &position, const Eigen::Vector3f &direction, float intensity, float falloff, float inner_cone_angle, float outer_cone_angle, bool cast_shadows) override
Definition: FilamentScene.cpp:1256
int size
Definition: FilePCD.cpp:40
std::string name
Definition: FilePCD.cpp:39
Definition: FilamentEngine.h:12
const char const char value recording_handle imu_sample recording_handle uint8_t size_t data_size k4a_record_configuration_t config target_format k4a_capture_t capture_handle k4a_imu_sample_t imu_sample playback_handle k4a_logging_message_cb_t void min_level device_handle k4a_imu_sample_t timeout_in_ms capture_handle capture_handle capture_handle image_handle temperature_c k4a_image_t image_handle uint8_t image_handle image_handle image_handle image_handle uint32_t
Definition: K4aPlugin.cpp:548
const char const char value recording_handle imu_sample recording_handle uint8_t size_t data_size k4a_record_configuration_t config target_format k4a_capture_t capture_handle k4a_imu_sample_t imu_sample playback_handle k4a_logging_message_cb_t void min_level device_handle k4a_imu_sample_t int32_t
Definition: K4aPlugin.cpp:395
REHandle< EntityType::IndexBuffer > IndexBufferHandle
Definition: RendererHandle.h:139
REHandle< EntityType::VertexBuffer > VertexBufferHandle
Definition: RendererHandle.h:138
REHandle< EntityType::MaterialInstance > MaterialInstanceHandle
Definition: RendererHandle.h:135
REHandle< EntityType::Skybox > SkyboxHandle
Definition: RendererHandle.h:132
REHandle< EntityType::IndirectLight > IndirectLightHandle
Definition: RendererHandle.h:131
Definition: PinholeCameraIntrinsic.cpp:16