Open3D (C++ API)  0.12.0
ContinuousConv.h
Go to the documentation of this file.
1 // ----------------------------------------------------------------------------
2 // - Open3D: www.open3d.org -
3 // ----------------------------------------------------------------------------
4 // The MIT License (MIT)
5 //
6 // Copyright (c) 2020 www.open3d.org
7 //
8 // Permission is hereby granted, free of charge, to any person obtaining a copy
9 // of this software and associated documentation files (the "Software"), to deal
10 // in the Software without restriction, including without limitation the rights
11 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12 // copies of the Software, and to permit persons to whom the Software is
13 // furnished to do so, subject to the following conditions:
14 //
15 // The above copyright notice and this permission notice shall be included in
16 // all copies or substantial portions of the Software.
17 //
18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 // IN THE SOFTWARE.
25 // ----------------------------------------------------------------------------
26 
27 #pragma once
28 
29 #include <tbb/parallel_for.h>
30 
31 #include "open3d/ml/impl/continuous_conv/CoordinateTransformation.h"
32 
33 namespace open3d {
34 namespace ml {
35 namespace impl {
36 
39 template <class TReal,
40  class TIndex,
41  InterpolationMode INTERPOLATION,
42  CoordinateMapping MAPPING,
43  bool ALIGN_CORNERS,
44  bool INDIVIDUAL_EXTENT,
45  bool ISOTROPIC_EXTENT,
46  bool POINT_IMPORTANCE>
47 void _CConvComputeFeaturesCPU(TReal* out_features,
48  const std::vector<int>& filter_dims,
49  const TReal* filter,
50  size_t num_out,
51  const TReal* out_positions,
52  size_t num_inp,
53  const TReal* inp_positions,
54  const TReal* inp_features,
55  const TReal* inp_importance,
56  size_t neighbors_index_size,
57  const TIndex* neighbor_index,
58  const TReal* neighbor_importance,
59  const int64_t* neighbors_row_splits,
60  const TReal* extents,
61  const TReal* offsets,
62  bool normalize) {
63  const bool NEIGHBOR_IMPORTANCE = neighbor_importance != nullptr;
64  const int VECSIZE = 32;
65  typedef Eigen::Array<TReal, VECSIZE, 1> Vec_t;
66  typedef InterpolationVec<TReal, VECSIZE, INTERPOLATION> InterpolationVec_t;
67  InterpolationVec_t interpolation;
68 
69  const int in_channels = filter_dims[filter_dims.size() - 2];
70  const int out_channels = filter_dims[filter_dims.size() - 1];
71 
72  int spatial_filter_size = 1;
73  for (int i = 0; i < 3; ++i) spatial_filter_size *= filter_dims[i];
74  Eigen::Array<int, 3, 1> filter_size_xyz(filter_dims[2], filter_dims[1],
75  filter_dims[0]);
76 
77  memset(out_features, 0, sizeof(TReal) * num_out * out_channels);
78 
79  tbb::parallel_for(
80  tbb::blocked_range<size_t>(0, num_out, 32),
81  [&](const tbb::blocked_range<size_t>& r) {
82  int range_length = r.end() - r.begin();
83 
84  Eigen::Matrix<TReal, Eigen::Dynamic, 1> normalizers(
85  range_length, 1);
86  normalizers.setZero();
87 
88  Eigen::Matrix<TReal, Eigen::Dynamic, Eigen::Dynamic> B(
89  in_channels * spatial_filter_size, range_length);
90  B.setZero();
91 
92  typedef Eigen::Array<TReal, VECSIZE, Eigen::Dynamic> Matrix;
93  Matrix infeat(VECSIZE, in_channels);
94 
95  Eigen::Array<TReal, 3, 1> offsets_(offsets[0], offsets[1],
96  offsets[2]);
97 
98  Eigen::Array<TReal, VECSIZE, 3> inv_extents;
99  if (INDIVIDUAL_EXTENT == false) {
100  if (ISOTROPIC_EXTENT) {
101  inv_extents = 1 / extents[0];
102  } else {
103  inv_extents.col(0) = 1 / extents[0];
104  inv_extents.col(1) = 1 / extents[1];
105  inv_extents.col(2) = 1 / extents[2];
106  }
107  }
108 
109  for (size_t out_idx = r.begin(); out_idx != r.end();
110  ++out_idx) {
111  const int out_col = out_idx - r.begin();
112  const size_t neighbor_start = neighbors_row_splits[out_idx];
113  const size_t neighbor_end =
114  neighbors_row_splits[out_idx + 1];
115 
116  if (INDIVIDUAL_EXTENT) {
117  if (ISOTROPIC_EXTENT) {
118  inv_extents = 1 / extents[out_idx];
119  } else {
120  inv_extents.col(0) = 1 / extents[3 * out_idx + 0];
121  inv_extents.col(1) = 1 / extents[3 * out_idx + 1];
122  inv_extents.col(2) = 1 / extents[3 * out_idx + 2];
123  }
124  }
125 
126  typename InterpolationVec_t::Weight_t interp_weights;
127  typename InterpolationVec_t::Idx_t interp_indices;
128 
129  int vec_valid_count = 0;
130  Vec_t x, y, z;
131 
132  // set to zero to avoid problems with vectors with less than
133  // VECSIZE valid entries
134  x.setZero();
135  y.setZero();
136  z.setZero();
137  for (size_t n = neighbor_start; n < neighbor_end; ++n) {
138  const size_t inp_idx = neighbor_index[n];
139  const int i = vec_valid_count;
140  x(i) = inp_positions[inp_idx * 3 + 0] -
141  out_positions[out_idx * 3 + 0];
142  y(i) = inp_positions[inp_idx * 3 + 1] -
143  out_positions[out_idx * 3 + 1];
144  z(i) = inp_positions[inp_idx * 3 + 2] -
145  out_positions[out_idx * 3 + 2];
146 
147  const TReal n_importance =
148  (NEIGHBOR_IMPORTANCE ? neighbor_importance[n]
149  : 1);
150  normalizers(out_col) += n_importance;
151 
152  for (int ic = 0; ic < in_channels; ++ic)
153  infeat(i, ic) =
154  inp_features[inp_idx * in_channels + ic];
155 
156  TReal importance = 1.0;
157  if (POINT_IMPORTANCE)
158  importance = inp_importance[inp_idx];
159  if (NEIGHBOR_IMPORTANCE) importance *= n_importance;
160 
161  if (POINT_IMPORTANCE || NEIGHBOR_IMPORTANCE) {
162  for (int ic = 0; ic < in_channels; ++ic)
163  infeat(i, ic) *= importance;
164  }
165 
166  ++vec_valid_count;
167  if (vec_valid_count == VECSIZE) {
168  ComputeFilterCoordinates<ALIGN_CORNERS, MAPPING>(
169  x, y, z, filter_size_xyz, inv_extents,
170  offsets_);
171  interpolation.Interpolate(
172  interp_weights, interp_indices, x, y, z,
173  filter_size_xyz, in_channels);
174  for (int k = 0; k < VECSIZE; ++k)
175  for (int j = 0; j < InterpolationVec_t::Size();
176  ++j) {
177  for (int ic = 0; ic < in_channels; ++ic)
178  B(interp_indices(j, k) + ic, out_col) +=
179  interp_weights(j, k) *
180  infeat(k, ic);
181  }
182  vec_valid_count = 0;
183  }
184  }
185  if (vec_valid_count) {
186  ComputeFilterCoordinates<ALIGN_CORNERS, MAPPING>(
187  x, y, z, filter_size_xyz, inv_extents,
188  offsets_);
189  interpolation.Interpolate(interp_weights,
190  interp_indices, x, y, z,
191  filter_size_xyz, in_channels);
192  for (int k = 0; k < vec_valid_count; ++k)
193  for (int j = 0; j < InterpolationVec_t::Size();
194  ++j) {
195  for (int ic = 0; ic < in_channels; ++ic)
196  B(interp_indices(j, k) + ic, out_col) +=
197  interp_weights(j, k) *
198  infeat(k, ic);
199  }
200  }
201 
202  } // out_idx
203 
204  Eigen::Map<const Eigen::Matrix<TReal, Eigen::Dynamic,
205  Eigen::Dynamic>>
206  A(filter, out_channels,
207  spatial_filter_size * in_channels);
208  Eigen::Map<Eigen::Matrix<TReal, Eigen::Dynamic, Eigen::Dynamic>>
209  C(out_features + (r.begin() * out_channels),
210  out_channels, range_length);
211 
212  C = A * B;
213  if (normalize) {
214  for (int i = 0; i < range_length; ++i) {
215  if (normalizers(i) != 0) C.col(i) /= normalizers(i);
216  }
217  }
218  });
219 }
220 
290 template <class TReal, class TIndex>
291 void CConvComputeFeaturesCPU(TReal* out_features,
292  const std::vector<int>& filter_dims,
293  const TReal* filter,
294  size_t num_out,
295  const TReal* out_positions,
296  size_t num_inp,
297  const TReal* inp_positions,
298  const TReal* inp_features,
299  const TReal* inp_importance,
300  size_t neighbors_index_size,
301  const TIndex* neighbor_index,
302  const TReal* neighbor_importance,
303  const int64_t* neighbors_row_splits,
304  const TReal* extents,
305  const TReal* offsets,
306  InterpolationMode interpolation,
307  CoordinateMapping coordinate_mapping,
308  bool align_corners,
309  bool individual_extent,
310  bool isotropic_extent,
311  bool normalize) {
312  // Dispatch all template parameter combinations
313  bool has_importance = inp_importance;
314 
315 #define FN_PARAMETERS \
316  out_features, filter_dims, filter, num_out, out_positions, num_inp, \
317  inp_positions, inp_features, inp_importance, neighbors_index_size, \
318  neighbor_index, neighbor_importance, neighbors_row_splits, \
319  extents, offsets, normalize
320 
321 #define CALL_TEMPLATE(INTERPOLATION, MAPPING, ALIGN_CORNERS, \
322  INDIVIDUAL_EXTENT, ISOTROPIC_EXTENT, HAS_IMPORTANCE) \
323  if (INTERPOLATION == interpolation && MAPPING == coordinate_mapping && \
324  ALIGN_CORNERS == align_corners && \
325  INDIVIDUAL_EXTENT == individual_extent && \
326  ISOTROPIC_EXTENT == isotropic_extent && \
327  HAS_IMPORTANCE == has_importance) \
328  _CConvComputeFeaturesCPU<TReal, TIndex, INTERPOLATION, MAPPING, \
329  ALIGN_CORNERS, INDIVIDUAL_EXTENT, \
330  ISOTROPIC_EXTENT, HAS_IMPORTANCE>( \
331  FN_PARAMETERS);
332 
333 #define CALL_TEMPLATE2(INTERPOLATION, MAPPING) \
334  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, true, true) \
335  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, true, false) \
336  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, false, true) \
337  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, false, false) \
338  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, true, true) \
339  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, true, false) \
340  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, false, true) \
341  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, false, false) \
342  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, true, true) \
343  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, true, false) \
344  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, false, true) \
345  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, false, false) \
346  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, true, true) \
347  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, true, false) \
348  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, false, true) \
349  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, false, false)
350 
351 #define CALL_TEMPLATE3(INTERPOLATION) \
352  CALL_TEMPLATE2(INTERPOLATION, CoordinateMapping::BALL_TO_CUBE_RADIAL) \
353  CALL_TEMPLATE2(INTERPOLATION, \
354  CoordinateMapping::BALL_TO_CUBE_VOLUME_PRESERVING) \
355  CALL_TEMPLATE2(INTERPOLATION, CoordinateMapping::IDENTITY)
356 
357 #define CALL_TEMPLATE4 \
358  CALL_TEMPLATE3(InterpolationMode::LINEAR) \
359  CALL_TEMPLATE3(InterpolationMode::LINEAR_BORDER) \
360  CALL_TEMPLATE3(InterpolationMode::NEAREST_NEIGHBOR)
361 
362  CALL_TEMPLATE4
363 
364 #undef CALL_TEMPLATE
365 #undef CALL_TEMPLATE2
366 #undef CALL_TEMPLATE3
367 #undef CALL_TEMPLATE4
368 
369 #undef FN_PARAMETERS
370 }
371 
372 } // namespace impl
373 } // namespace ml
374 } // namespace open3d
open3d::ml::impl::InterpolationMode
InterpolationMode
Definition: ContinuousConvTypes.h:37
open3d::ml::impl::InterpolationVec
Class for computing interpolation weights.
Definition: CoordinateTransformation.h:204
CoordinateTransformation.h
open3d::ml::impl::CConvComputeFeaturesCPU
void CConvComputeFeaturesCPU(TReal *out_features, const std::vector< int > &filter_dims, const TReal *filter, size_t num_out, const TReal *out_positions, size_t num_inp, const TReal *inp_positions, const TReal *inp_features, const TReal *inp_importance, size_t neighbors_index_size, const TIndex *neighbor_index, const TReal *neighbor_importance, const int64_t *neighbors_row_splits, const TReal *extents, const TReal *offsets, InterpolationMode interpolation, CoordinateMapping coordinate_mapping, bool align_corners, bool individual_extent, bool isotropic_extent, bool normalize)
Definition: ContinuousConv.h:291
open3d::ml::impl::CoordinateMapping
CoordinateMapping
Definition: ContinuousConvTypes.h:45
open3d
Definition: PinholeCameraIntrinsic.cpp:35
open3d::ml::impl::_CConvComputeFeaturesCPU
void _CConvComputeFeaturesCPU(TReal *out_features, const std::vector< int > &filter_dims, const TReal *filter, size_t num_out, const TReal *out_positions, size_t num_inp, const TReal *inp_positions, const TReal *inp_features, const TReal *inp_importance, size_t neighbors_index_size, const TIndex *neighbor_index, const TReal *neighbor_importance, const int64_t *neighbors_row_splits, const TReal *extents, const TReal *offsets, bool normalize)
Definition: ContinuousConv.h:47
CALL_TEMPLATE4
#define CALL_TEMPLATE4
Generated by   doxygen 1.8.13