Open3D (C++ API)  0.11.0
ContinuousConvTranspose.h
Go to the documentation of this file.
1 // ----------------------------------------------------------------------------
2 // - Open3D: www.open3d.org -
3 // ----------------------------------------------------------------------------
4 // The MIT License (MIT)
5 //
6 // Copyright (c) 2020 www.open3d.org
7 //
8 // Permission is hereby granted, free of charge, to any person obtaining a copy
9 // of this software and associated documentation files (the "Software"), to deal
10 // in the Software without restriction, including without limitation the rights
11 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12 // copies of the Software, and to permit persons to whom the Software is
13 // furnished to do so, subject to the following conditions:
14 //
15 // The above copyright notice and this permission notice shall be included in
16 // all copies or substantial portions of the Software.
17 //
18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 // IN THE SOFTWARE.
25 // ----------------------------------------------------------------------------
26 
27 #pragma once
28 
29 #include <tbb/parallel_for.h>
30 
31 #include "open3d/ml/impl/continuous_conv/CoordinateTransformation.h"
32 
33 namespace open3d {
34 namespace ml {
35 namespace impl {
36 
39 template <class TReal,
40  class TIndex,
41  InterpolationMode INTERPOLATION,
42  CoordinateMapping MAPPING,
43  bool ALIGN_CORNERS,
44  bool INDIVIDUAL_EXTENT,
45  bool ISOTROPIC_EXTENT,
46  bool NORMALIZE>
47 void _CConvTransposeComputeFeaturesCPU(
48  TReal* out_features,
49  const std::vector<int>& filter_dims,
50  const TReal* filter,
51  size_t num_out,
52  const TReal* out_positions,
53  const TReal* out_importance,
54  size_t num_inp,
55  const TReal* inp_positions,
56  const TReal* inp_features,
57  const TReal* inp_neighbors_importance_sum,
58  const int64_t* inp_neighbors_row_splits,
59  size_t neighbors_index_size,
60  const TIndex* neighbor_index,
61  const TReal* neighbor_importance,
62  const int64_t* neighbors_row_splits,
63  const TReal* extents,
64  const TReal* offsets) {
65  const bool NEIGHBOR_IMPORTANCE = inp_neighbors_importance_sum;
66  const int VECSIZE = 32;
67  typedef Eigen::Array<TReal, VECSIZE, 1> Vec_t;
68  typedef InterpolationVec<TReal, VECSIZE, INTERPOLATION> InterpolationVec_t;
69  InterpolationVec_t interpolation;
70 
71  const int in_channels = filter_dims[filter_dims.size() - 2];
72  const int out_channels = filter_dims[filter_dims.size() - 1];
73 
74  int spatial_filter_size = 1;
75  for (int i = 0; i < 3; ++i) spatial_filter_size *= filter_dims[i];
76  Eigen::Array<int, 3, 1> filter_size_xyz(filter_dims[2], filter_dims[1],
77  filter_dims[0]);
78 
79  memset(out_features, 0, sizeof(TReal) * num_out * out_channels);
80 
81  tbb::parallel_for(
82  tbb::blocked_range<size_t>(0, num_out, 32),
83  [&](const tbb::blocked_range<size_t>& r) {
84  int range_length = r.end() - r.begin();
85 
86  Eigen::Matrix<TReal, Eigen::Dynamic, Eigen::Dynamic> B(
87  in_channels * spatial_filter_size, range_length);
88  B.setZero();
89 
90  typedef Eigen::Array<TReal, VECSIZE, Eigen::Dynamic> Matrix;
91  Matrix infeat(VECSIZE, in_channels);
92 
93  Eigen::Array<TReal, 3, 1> offsets_(offsets[0], offsets[1],
94  offsets[2]);
95 
96  Eigen::Array<TReal, VECSIZE, 3> inv_extents;
97  if (INDIVIDUAL_EXTENT == false) {
98  if (ISOTROPIC_EXTENT) {
99  inv_extents = 1 / extents[0];
100  } else {
101  inv_extents.col(0) = 1 / extents[0];
102  inv_extents.col(1) = 1 / extents[1];
103  inv_extents.col(2) = 1 / extents[2];
104  }
105  }
106 
107  for (size_t out_idx = r.begin(); out_idx != r.end();
108  ++out_idx) {
109  const int out_col = out_idx - r.begin();
110  const size_t neighbor_start = neighbors_row_splits[out_idx];
111  const size_t neighbor_end =
112  (out_idx + 1 < num_out
113  ? neighbors_row_splits[out_idx + 1]
114  : neighbors_index_size);
115 
116  typename InterpolationVec_t::Weight_t interp_weights;
117  typename InterpolationVec_t::Idx_t interp_indices;
118 
119  int vec_valid_count = 0;
120  Vec_t x, y, z;
121 
122  // set to zero to avoid problems with vectors with less than
123  // VECSIZE valid entries
124  x.setZero();
125  y.setZero();
126  z.setZero();
127  for (size_t n = neighbor_start; n < neighbor_end; ++n) {
128  const size_t inp_idx = neighbor_index[n];
129 
130  const int i = vec_valid_count;
131  x(i) = out_positions[out_idx * 3 + 0] -
132  inp_positions[inp_idx * 3 + 0];
133  y(i) = out_positions[out_idx * 3 + 1] -
134  inp_positions[inp_idx * 3 + 1];
135  z(i) = out_positions[out_idx * 3 + 2] -
136  inp_positions[inp_idx * 3 + 2];
137 
138  if (INDIVIDUAL_EXTENT) {
139  if (ISOTROPIC_EXTENT) {
140  inv_extents.row(i) = 1 / extents[inp_idx];
141  } else {
142  inv_extents(i, 0) =
143  1 / extents[3 * inp_idx + 0];
144  inv_extents(i, 1) =
145  1 / extents[3 * inp_idx + 1];
146  inv_extents(i, 2) =
147  1 / extents[3 * inp_idx + 2];
148  }
149  }
150 
151  TReal n_importance = NEIGHBOR_IMPORTANCE
152  ? neighbor_importance[n]
153  : 1;
154  for (int ic = 0; ic < in_channels; ++ic)
155  infeat(i, ic) =
156  inp_features[inp_idx * in_channels + ic] *
157  n_importance;
158 
159  if (NORMALIZE) {
160  TReal normalizer = 1;
161  if (NEIGHBOR_IMPORTANCE) {
162  if (inp_neighbors_importance_sum[inp_idx] != 0)
163  normalizer /= inp_neighbors_importance_sum
164  [inp_idx];
165  } else {
166  size_t num_inp_neighbors;
167  const size_t inp_neighbor_start =
168  inp_neighbors_row_splits[inp_idx];
169  const size_t inp_neighbor_end =
170  inp_neighbors_row_splits[inp_idx + 1];
171  num_inp_neighbors =
172  inp_neighbor_end - inp_neighbor_start;
173  if (num_inp_neighbors > 0)
174  normalizer /= num_inp_neighbors;
175  }
176  for (int ic = 0; ic < in_channels; ++ic)
177  infeat(i, ic) *= normalizer;
178  }
179 
180  ++vec_valid_count;
181  if (vec_valid_count == VECSIZE ||
182  n + 1 == neighbor_end) {
183  ComputeFilterCoordinates<ALIGN_CORNERS, MAPPING>(
184  x, y, z, filter_size_xyz, inv_extents,
185  offsets_);
186  interpolation.Interpolate(
187  interp_weights, interp_indices, x, y, z,
188  filter_size_xyz, in_channels);
189  for (int k = 0; k < vec_valid_count; ++k) {
190  for (int j = 0; j < InterpolationVec_t::Size();
191  ++j) {
192  for (int ic = 0; ic < in_channels; ++ic)
193  B(interp_indices(j, k) + ic, out_col) +=
194  interp_weights(j, k) *
195  infeat(k, ic);
196  }
197  }
198  vec_valid_count = 0;
199  }
200  }
201 
202  } // out_idx
203 
204  Eigen::Map<const Eigen::Matrix<TReal, Eigen::Dynamic,
205  Eigen::Dynamic>>
206  A(filter, out_channels,
207  spatial_filter_size * in_channels);
208  Eigen::Map<Eigen::Matrix<TReal, Eigen::Dynamic, Eigen::Dynamic>>
209  C(out_features + (r.begin() * out_channels),
210  out_channels, range_length);
211 
212  C = A * B;
213  if (out_importance) {
214  for (int i = 0; i < range_length; ++i)
215  C.col(i) *= out_importance[r.begin() + i];
216  }
217  });
218 }
219 
296 template <class TReal, class TIndex>
297 void CConvTransposeComputeFeaturesCPU(TReal* out_features,
298  const std::vector<int>& filter_dims,
299  const TReal* filter,
300  size_t num_out,
301  const TReal* out_positions,
302  const TReal* out_importance,
303  size_t num_inp,
304  const TReal* inp_positions,
305  const TReal* inp_features,
306  const TReal* inp_neighbors_importance_sum,
307  const int64_t* inp_neighbors_row_splits,
308  size_t neighbors_index_size,
309  const TIndex* neighbor_index,
310  const TReal* neighbor_importance,
311  const int64_t* neighbors_row_splits,
312  const TReal* extents,
313  const TReal* offsets,
314  InterpolationMode interpolation,
315  CoordinateMapping coordinate_mapping,
316  bool align_corners,
317  bool individual_extent,
318  bool isotropic_extent,
319  bool normalize) {
320 #define FN_PARAMETERS \
321  out_features, filter_dims, filter, num_out, out_positions, out_importance, \
322  num_inp, inp_positions, inp_features, \
323  inp_neighbors_importance_sum, inp_neighbors_row_splits, \
324  neighbors_index_size, neighbor_index, neighbor_importance, \
325  neighbors_row_splits, extents, offsets
326 
327 #define CALL_TEMPLATE(INTERPOLATION, MAPPING, ALIGN_CORNERS, \
328  INDIVIDUAL_EXTENT, ISOTROPIC_EXTENT, NORMALIZE) \
329  if (INTERPOLATION == interpolation && MAPPING == coordinate_mapping && \
330  ALIGN_CORNERS == align_corners && \
331  INDIVIDUAL_EXTENT == individual_extent && \
332  ISOTROPIC_EXTENT == isotropic_extent && NORMALIZE == normalize) \
333  _CConvTransposeComputeFeaturesCPU< \
334  TReal, TIndex, INTERPOLATION, MAPPING, ALIGN_CORNERS, \
335  INDIVIDUAL_EXTENT, ISOTROPIC_EXTENT, NORMALIZE>( \
336  FN_PARAMETERS);
337 
338 #define CALL_TEMPLATE2(INTERPOLATION, MAPPING) \
339  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, true, true) \
340  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, true, false) \
341  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, false, true) \
342  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, false, false) \
343  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, true, true) \
344  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, true, false) \
345  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, false, true) \
346  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, false, false) \
347  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, true, true) \
348  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, true, false) \
349  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, false, true) \
350  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, false, false) \
351  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, true, true) \
352  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, true, false) \
353  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, false, true) \
354  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, false, false)
355 
356 #define CALL_TEMPLATE3(INTERPOLATION) \
357  CALL_TEMPLATE2(INTERPOLATION, CoordinateMapping::BALL_TO_CUBE_RADIAL) \
358  CALL_TEMPLATE2(INTERPOLATION, \
359  CoordinateMapping::BALL_TO_CUBE_VOLUME_PRESERVING) \
360  CALL_TEMPLATE2(INTERPOLATION, CoordinateMapping::IDENTITY)
361 
362 #define CALL_TEMPLATE4 \
363  CALL_TEMPLATE3(InterpolationMode::LINEAR) \
364  CALL_TEMPLATE3(InterpolationMode::LINEAR_BORDER) \
365  CALL_TEMPLATE3(InterpolationMode::NEAREST_NEIGHBOR)
366 
367  CALL_TEMPLATE4
368 
369 #undef CALL_TEMPLATE
370 #undef CALL_TEMPLATE2
371 #undef CALL_TEMPLATE3
372 #undef CALL_TEMPLATE4
373 
374 #undef FN_PARAMETERS
375 }
376 
377 } // namespace impl
378 } // namespace ml
379 } // namespace open3d
CALL_TEMPLATE4
#define CALL_TEMPLATE4
open3d::ml::impl::InterpolationMode
InterpolationMode
Definition: ContinuousConvTypes.h:37
open3d::ml::impl::InterpolationVec
Class for computing interpolation weights.
Definition: CoordinateTransformation.h:204
CoordinateTransformation.h
open3d::ml::impl::CoordinateMapping
CoordinateMapping
Definition: ContinuousConvTypes.h:45
open3d
Definition: PinholeCameraIntrinsic.cpp:35
open3d::ml::impl::CConvTransposeComputeFeaturesCPU
void CConvTransposeComputeFeaturesCPU(TReal *out_features, const std::vector< int > &filter_dims, const TReal *filter, size_t num_out, const TReal *out_positions, const TReal *out_importance, size_t num_inp, const TReal *inp_positions, const TReal *inp_features, const TReal *inp_neighbors_importance_sum, const int64_t *inp_neighbors_row_splits, size_t neighbors_index_size, const TIndex *neighbor_index, const TReal *neighbor_importance, const int64_t *neighbors_row_splits, const TReal *extents, const TReal *offsets, InterpolationMode interpolation, CoordinateMapping coordinate_mapping, bool align_corners, bool individual_extent, bool isotropic_extent, bool normalize)
Definition: ContinuousConvTranspose.h:297
open3d::ml::impl::_CConvTransposeComputeFeaturesCPU
void _CConvTransposeComputeFeaturesCPU(TReal *out_features, const std::vector< int > &filter_dims, const TReal *filter, size_t num_out, const TReal *out_positions, const TReal *out_importance, size_t num_inp, const TReal *inp_positions, const TReal *inp_features, const TReal *inp_neighbors_importance_sum, const int64_t *inp_neighbors_row_splits, size_t neighbors_index_size, const TIndex *neighbor_index, const TReal *neighbor_importance, const int64_t *neighbors_row_splits, const TReal *extents, const TReal *offsets)
Definition: ContinuousConvTranspose.h:47
Generated by   doxygen 1.8.13