Open3D (C++ API)  0.17.0
ContinuousConv.h
Go to the documentation of this file.
1 // ----------------------------------------------------------------------------
2 // - Open3D: www.open3d.org -
3 // ----------------------------------------------------------------------------
4 // Copyright (c) 2018-2023 www.open3d.org
5 // SPDX-License-Identifier: MIT
6 // ----------------------------------------------------------------------------
7 
8 #pragma once
9 
10 #include <tbb/parallel_for.h>
11 
12 #include "open3d/ml/impl/continuous_conv/CoordinateTransformation.h"
13 
14 namespace open3d {
15 namespace ml {
16 namespace impl {
17 
20 template <class TFeat,
21  class TOut,
22  class TReal,
23  class TIndex,
24  InterpolationMode INTERPOLATION,
25  CoordinateMapping MAPPING,
26  bool ALIGN_CORNERS,
27  bool INDIVIDUAL_EXTENT,
28  bool ISOTROPIC_EXTENT,
29  bool POINT_IMPORTANCE>
30 void _CConvComputeFeaturesCPU(TOut* out_features,
31  const std::vector<int>& filter_dims,
32  const TFeat* filter,
33  size_t num_out,
34  const TReal* out_positions,
35  size_t num_inp,
36  const TReal* inp_positions,
37  const TFeat* inp_features,
38  const TFeat* inp_importance,
39  size_t neighbors_index_size,
40  const TIndex* neighbors_index,
41  const TFeat* neighbors_importance,
42  const int64_t* neighbors_row_splits,
43  const TReal* extents,
44  const TReal* offsets,
45  bool normalize) {
46  const bool NEIGHBORS_IMPORTANCE = neighbors_importance != nullptr;
47  const int VECSIZE = 32;
48  typedef Eigen::Array<TReal, VECSIZE, 1> Vec_t;
49  typedef InterpolationVec<TReal, VECSIZE, INTERPOLATION> InterpolationVec_t;
50  InterpolationVec_t interpolation;
51 
52  const int in_channels = filter_dims[filter_dims.size() - 2];
53  const int out_channels = filter_dims[filter_dims.size() - 1];
54 
55  int spatial_filter_size = 1;
56  for (int i = 0; i < 3; ++i) spatial_filter_size *= filter_dims[i];
57  Eigen::Array<int, 3, 1> filter_size_xyz(filter_dims[2], filter_dims[1],
58  filter_dims[0]);
59 
60  memset(out_features, 0, sizeof(TOut) * num_out * out_channels);
61 
62  tbb::parallel_for(
63  tbb::blocked_range<size_t>(0, num_out, 32),
64  [&](const tbb::blocked_range<size_t>& r) {
65  int range_length = r.end() - r.begin();
66 
67  Eigen::Matrix<TOut, Eigen::Dynamic, 1> normalizers(range_length,
68  1);
69  normalizers.setZero();
70 
71  Eigen::Matrix<TFeat, Eigen::Dynamic, Eigen::Dynamic> B(
72  in_channels * spatial_filter_size, range_length);
73  B.setZero();
74 
75  typedef Eigen::Array<TFeat, VECSIZE, Eigen::Dynamic> Matrix;
76  Matrix infeat(VECSIZE, in_channels);
77 
78  Eigen::Array<TReal, 3, 1> offsets_(offsets[0], offsets[1],
79  offsets[2]);
80 
81  Eigen::Array<TReal, VECSIZE, 3> inv_extents;
82  if (INDIVIDUAL_EXTENT == false) {
83  if (ISOTROPIC_EXTENT) {
84  inv_extents = 1 / extents[0];
85  } else {
86  inv_extents.col(0) = 1 / extents[0];
87  inv_extents.col(1) = 1 / extents[1];
88  inv_extents.col(2) = 1 / extents[2];
89  }
90  }
91 
92  for (size_t out_idx = r.begin(); out_idx != r.end();
93  ++out_idx) {
94  const int out_col = out_idx - r.begin();
95  const size_t neighbor_start = neighbors_row_splits[out_idx];
96  const size_t neighbor_end =
97  neighbors_row_splits[out_idx + 1];
98 
99  if (INDIVIDUAL_EXTENT) {
100  if (ISOTROPIC_EXTENT) {
101  inv_extents = 1 / extents[out_idx];
102  } else {
103  inv_extents.col(0) = 1 / extents[3 * out_idx + 0];
104  inv_extents.col(1) = 1 / extents[3 * out_idx + 1];
105  inv_extents.col(2) = 1 / extents[3 * out_idx + 2];
106  }
107  }
108 
109  typename InterpolationVec_t::Weight_t interp_weights;
110  typename InterpolationVec_t::Idx_t interp_indices;
111 
112  int vec_valid_count = 0;
113  Vec_t x, y, z;
114 
115  // set to zero to avoid problems with vectors with less than
116  // VECSIZE valid entries
117  x.setZero();
118  y.setZero();
119  z.setZero();
120  for (size_t n = neighbor_start; n < neighbor_end; ++n) {
121  const size_t inp_idx = neighbors_index[n];
122  const int i = vec_valid_count;
123  x(i) = inp_positions[inp_idx * 3 + 0] -
124  out_positions[out_idx * 3 + 0];
125  y(i) = inp_positions[inp_idx * 3 + 1] -
126  out_positions[out_idx * 3 + 1];
127  z(i) = inp_positions[inp_idx * 3 + 2] -
128  out_positions[out_idx * 3 + 2];
129 
130  const TFeat n_importance =
131  (NEIGHBORS_IMPORTANCE ? neighbors_importance[n]
132  : TFeat(1));
133  normalizers(out_col) += TOut(n_importance);
134 
135  for (int ic = 0; ic < in_channels; ++ic)
136  infeat(i, ic) =
137  inp_features[inp_idx * in_channels + ic];
138 
139  TFeat importance(1.0);
140  if (POINT_IMPORTANCE)
141  importance = inp_importance[inp_idx];
142  if (NEIGHBORS_IMPORTANCE) importance *= n_importance;
143 
144  if (POINT_IMPORTANCE || NEIGHBORS_IMPORTANCE) {
145  for (int ic = 0; ic < in_channels; ++ic)
146  infeat(i, ic) *= importance;
147  }
148 
149  ++vec_valid_count;
150  if (vec_valid_count == VECSIZE) {
151  ComputeFilterCoordinates<ALIGN_CORNERS, MAPPING>(
152  x, y, z, filter_size_xyz, inv_extents,
153  offsets_);
154  interpolation.Interpolate(
155  interp_weights, interp_indices, x, y, z,
156  filter_size_xyz, in_channels);
157  for (int k = 0; k < VECSIZE; ++k)
158  for (int j = 0; j < InterpolationVec_t::Size();
159  ++j) {
160  for (int ic = 0; ic < in_channels; ++ic)
161  B(interp_indices(j, k) + ic, out_col) +=
162  TFeat(interp_weights(j, k)) *
163  infeat(k, ic);
164  }
165  vec_valid_count = 0;
166  }
167  }
168  if (vec_valid_count) {
169  ComputeFilterCoordinates<ALIGN_CORNERS, MAPPING>(
170  x, y, z, filter_size_xyz, inv_extents,
171  offsets_);
172  interpolation.Interpolate(interp_weights,
173  interp_indices, x, y, z,
174  filter_size_xyz, in_channels);
175  for (int k = 0; k < vec_valid_count; ++k)
176  for (int j = 0; j < InterpolationVec_t::Size();
177  ++j) {
178  for (int ic = 0; ic < in_channels; ++ic)
179  B(interp_indices(j, k) + ic, out_col) +=
180  TFeat(interp_weights(j, k)) *
181  infeat(k, ic);
182  }
183  }
184 
185  } // out_idx
186 
187  Eigen::Map<const Eigen::Matrix<TFeat, Eigen::Dynamic,
188  Eigen::Dynamic>>
189  A(filter, out_channels,
190  spatial_filter_size * in_channels);
191  Eigen::Map<Eigen::Matrix<TOut, Eigen::Dynamic, Eigen::Dynamic>>
192  C(out_features + (r.begin() * out_channels),
193  out_channels, range_length);
194 
195  C = (A * B).template cast<TOut>();
196  if (normalize) {
197  for (int i = 0; i < range_length; ++i) {
198  if (normalizers(i) != TOut(0))
199  C.col(i) /= normalizers(i);
200  }
201  }
202  });
203 }
204 
279 template <class TFeat, class TOut, class TReal, class TIndex>
280 void CConvComputeFeaturesCPU(TOut* out_features,
281  const std::vector<int>& filter_dims,
282  const TFeat* filter,
283  size_t num_out,
284  const TReal* out_positions,
285  size_t num_inp,
286  const TReal* inp_positions,
287  const TFeat* inp_features,
288  const TFeat* inp_importance,
289  size_t neighbors_index_size,
290  const TIndex* neighbors_index,
291  const TFeat* neighbors_importance,
292  const int64_t* neighbors_row_splits,
293  const TReal* extents,
294  const TReal* offsets,
295  InterpolationMode interpolation,
296  CoordinateMapping coordinate_mapping,
297  bool align_corners,
298  bool individual_extent,
299  bool isotropic_extent,
300  bool normalize) {
301  // Dispatch all template parameter combinations
302  bool has_importance = inp_importance;
303 
304 #define FN_PARAMETERS \
305  out_features, filter_dims, filter, num_out, out_positions, num_inp, \
306  inp_positions, inp_features, inp_importance, neighbors_index_size, \
307  neighbors_index, neighbors_importance, neighbors_row_splits, \
308  extents, offsets, normalize
309 
310 #define CALL_TEMPLATE(INTERPOLATION, MAPPING, ALIGN_CORNERS, \
311  INDIVIDUAL_EXTENT, ISOTROPIC_EXTENT, HAS_IMPORTANCE) \
312  if (INTERPOLATION == interpolation && MAPPING == coordinate_mapping && \
313  ALIGN_CORNERS == align_corners && \
314  INDIVIDUAL_EXTENT == individual_extent && \
315  ISOTROPIC_EXTENT == isotropic_extent && \
316  HAS_IMPORTANCE == has_importance) \
317  _CConvComputeFeaturesCPU<TFeat, TOut, TReal, TIndex, INTERPOLATION, \
318  MAPPING, ALIGN_CORNERS, INDIVIDUAL_EXTENT, \
319  ISOTROPIC_EXTENT, HAS_IMPORTANCE>( \
320  FN_PARAMETERS);
321 
322 #define CALL_TEMPLATE2(INTERPOLATION, MAPPING) \
323  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, true, true) \
324  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, true, false) \
325  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, false, true) \
326  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, false, false) \
327  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, true, true) \
328  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, true, false) \
329  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, false, true) \
330  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, false, false) \
331  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, true, true) \
332  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, true, false) \
333  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, false, true) \
334  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, false, false) \
335  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, true, true) \
336  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, true, false) \
337  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, false, true) \
338  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, false, false)
339 
340 #define CALL_TEMPLATE3(INTERPOLATION) \
341  CALL_TEMPLATE2(INTERPOLATION, CoordinateMapping::BALL_TO_CUBE_RADIAL) \
342  CALL_TEMPLATE2(INTERPOLATION, \
343  CoordinateMapping::BALL_TO_CUBE_VOLUME_PRESERVING) \
344  CALL_TEMPLATE2(INTERPOLATION, CoordinateMapping::IDENTITY)
345 
346 #define CALL_TEMPLATE4 \
347  CALL_TEMPLATE3(InterpolationMode::LINEAR) \
348  CALL_TEMPLATE3(InterpolationMode::LINEAR_BORDER) \
349  CALL_TEMPLATE3(InterpolationMode::NEAREST_NEIGHBOR)
350 
351  CALL_TEMPLATE4
352 
353 #undef CALL_TEMPLATE
354 #undef CALL_TEMPLATE2
355 #undef CALL_TEMPLATE3
356 #undef CALL_TEMPLATE4
357 
358 #undef FN_PARAMETERS
359 }
360 
361 } // namespace impl
362 } // namespace ml
363 } // namespace open3d
CALL_TEMPLATE4
#define CALL_TEMPLATE4
VECSIZE
#define VECSIZE
B
Eigen::Matrix3d B
Definition: PointCloudPlanarPatchDetection.cpp:506
open3d::ml::impl::InterpolationMode
InterpolationMode
Definition: ContinuousConvTypes.h:18
open3d::ml::impl::_CConvComputeFeaturesCPU
void _CConvComputeFeaturesCPU(TOut *out_features, const std::vector< int > &filter_dims, const TFeat *filter, size_t num_out, const TReal *out_positions, size_t num_inp, const TReal *inp_positions, const TFeat *inp_features, const TFeat *inp_importance, size_t neighbors_index_size, const TIndex *neighbors_index, const TFeat *neighbors_importance, const int64_t *neighbors_row_splits, const TReal *extents, const TReal *offsets, bool normalize)
Definition: ContinuousConv.h:30
open3d::ml::impl::CConvComputeFeaturesCPU
void CConvComputeFeaturesCPU(TOut *out_features, const std::vector< int > &filter_dims, const TFeat *filter, size_t num_out, const TReal *out_positions, size_t num_inp, const TReal *inp_positions, const TFeat *inp_features, const TFeat *inp_importance, size_t neighbors_index_size, const TIndex *neighbors_index, const TFeat *neighbors_importance, const int64_t *neighbors_row_splits, const TReal *extents, const TReal *offsets, InterpolationMode interpolation, CoordinateMapping coordinate_mapping, bool align_corners, bool individual_extent, bool isotropic_extent, bool normalize)
Definition: ContinuousConv.h:280
open3d::ml::impl::CoordinateMapping
CoordinateMapping
Definition: ContinuousConvTypes.h:26
open3d
Definition: PinholeCameraIntrinsic.cpp:16
open3d::ml::impl::InterpolationVec
Class for computing interpolation weights.
Definition: CoordinateTransformation.h:185
Generated by doxygen 1.9.1