Open3D (C++ API)  0.18.0
ContinuousConvTransposeBackpropFilter.h
Go to the documentation of this file.
1 // ----------------------------------------------------------------------------
2 // - Open3D: www.open3d.org -
3 // ----------------------------------------------------------------------------
4 // Copyright (c) 2018-2023 www.open3d.org
5 // SPDX-License-Identifier: MIT
6 // ----------------------------------------------------------------------------
7 
8 #pragma once
9 
10 #include <tbb/parallel_for.h>
11 
12 #include <mutex>
13 
14 #include "open3d/ml/impl/continuous_conv/CoordinateTransformation.h"
15 
16 namespace open3d {
17 namespace ml {
18 namespace impl {
19 
22 template <class TFeat,
23  class TOut,
24  class TReal,
25  class TIndex,
26  InterpolationMode INTERPOLATION,
27  CoordinateMapping MAPPING,
28  bool ALIGN_CORNERS,
29  bool INDIVIDUAL_EXTENT,
30  bool ISOTROPIC_EXTENT,
31  bool NORMALIZE>
32 void _CConvTransposeBackpropFilterCPU(TOut* filter_backprop,
33  const std::vector<int>& filter_dims,
34  size_t num_out,
35  const TReal* out_positions,
36  const TFeat* out_importance,
37  size_t num_inp,
38  const TReal* inp_positions,
39  const TFeat* inp_features,
40  const TFeat* inp_neighbors_importance_sum,
41  const int64_t* inp_neighbors_row_splits,
42  size_t neighbors_index_size,
43  const TIndex* neighbors_index,
44  const TFeat* neighbors_importance,
45  const int64_t* neighbors_row_splits,
46  const TReal* extents,
47  const TReal* offsets,
48  const TFeat* out_features_gradient) {
49  const bool NEIGHBOR_IMPORTANCE = neighbors_importance;
50  const int VECSIZE = 32;
51  typedef Eigen::Array<TReal, VECSIZE, 1> Vec_t;
52  typedef InterpolationVec<TReal, VECSIZE, INTERPOLATION> InterpolationVec_t;
53  InterpolationVec_t interpolation;
54 
55  const int in_channels = filter_dims[filter_dims.size() - 2];
56  const int out_channels = filter_dims[filter_dims.size() - 1];
57 
58  int spatial_filter_size = 1;
59  for (int i = 0; i < 3; ++i) spatial_filter_size *= filter_dims[i];
60  Eigen::Array<int, 3, 1> filter_size_xyz(filter_dims[2], filter_dims[1],
61  filter_dims[0]);
62 
63  memset(filter_backprop, 0,
64  sizeof(TOut) * spatial_filter_size * in_channels * out_channels);
65  std::mutex filter_backprop_mutex;
66 
67  tbb::parallel_for(
68  tbb::blocked_range<size_t>(0, num_out, 32),
69  [&](const tbb::blocked_range<size_t>& r) {
70  int range_length = r.end() - r.begin();
71 
72  Eigen::Matrix<TFeat, Eigen::Dynamic, Eigen::Dynamic> B(
73  in_channels * spatial_filter_size, range_length);
74  B.setZero();
75  Eigen::Matrix<TFeat, Eigen::Dynamic, Eigen::Dynamic> C(
76  out_channels, range_length);
77 
78  typedef Eigen::Array<TFeat, VECSIZE, Eigen::Dynamic> Matrix;
79  Matrix infeat(VECSIZE, in_channels);
80 
81  Eigen::Array<TReal, 3, 1> offsets_(offsets[0], offsets[1],
82  offsets[2]);
83 
84  Eigen::Array<TReal, VECSIZE, 3> inv_extents;
85  if (INDIVIDUAL_EXTENT == false) {
86  if (ISOTROPIC_EXTENT) {
87  inv_extents = 1 / extents[0];
88  } else {
89  inv_extents.col(0) = 1 / extents[0];
90  inv_extents.col(1) = 1 / extents[1];
91  inv_extents.col(2) = 1 / extents[2];
92  }
93  }
94 
95  for (size_t out_idx = r.begin(); out_idx != r.end();
96  ++out_idx) {
97  const int out_col = out_idx - r.begin();
98  const size_t neighbor_start = neighbors_row_splits[out_idx];
99  const size_t neighbor_end =
100  neighbors_row_splits[out_idx + 1];
101 
102  C.col(out_col) = Eigen::Map<
103  const Eigen::Array<TFeat, Eigen::Dynamic, 1>>(
104  out_features_gradient + out_idx * out_channels,
105  out_channels, 1);
106 
107  typename InterpolationVec_t::Weight_t interp_weights;
108  typename InterpolationVec_t::Idx_t interp_indices;
109 
110  int vec_valid_count = 0;
111  Vec_t x, y, z;
112 
113  // set to zero to avoid problems with vectors with less than
114  // VECSIZE valid entries
115  x.setZero();
116  y.setZero();
117  z.setZero();
118  for (size_t n = neighbor_start; n < neighbor_end; ++n) {
119  const size_t inp_idx = neighbors_index[n];
120 
121  const int i = vec_valid_count;
122  x(i) = out_positions[out_idx * 3 + 0] -
123  inp_positions[inp_idx * 3 + 0];
124  y(i) = out_positions[out_idx * 3 + 1] -
125  inp_positions[inp_idx * 3 + 1];
126  z(i) = out_positions[out_idx * 3 + 2] -
127  inp_positions[inp_idx * 3 + 2];
128 
129  if (INDIVIDUAL_EXTENT) {
130  if (ISOTROPIC_EXTENT) {
131  inv_extents.row(i) = 1 / extents[inp_idx];
132  } else {
133  inv_extents(i, 0) =
134  1 / extents[3 * inp_idx + 0];
135  inv_extents(i, 1) =
136  1 / extents[3 * inp_idx + 1];
137  inv_extents(i, 2) =
138  1 / extents[3 * inp_idx + 2];
139  }
140  }
141 
142  TFeat n_importance = NEIGHBOR_IMPORTANCE
143  ? neighbors_importance[n]
144  : TFeat(1);
145  for (int ic = 0; ic < in_channels; ++ic)
146  infeat(i, ic) =
147  inp_features[inp_idx * in_channels + ic] *
148  n_importance;
149 
150  if (NORMALIZE) {
151  TFeat normalizer(1);
152  if (NEIGHBOR_IMPORTANCE) {
153  if (inp_neighbors_importance_sum[inp_idx] !=
154  TFeat(0))
155  normalizer /= inp_neighbors_importance_sum
156  [inp_idx];
157  } else {
158  size_t num_inp_neighbors;
159  const size_t inp_neighbor_start =
160  inp_neighbors_row_splits[inp_idx];
161  const size_t inp_neighbor_end =
162  inp_neighbors_row_splits[inp_idx + 1];
163  num_inp_neighbors =
164  inp_neighbor_end - inp_neighbor_start;
165  if (num_inp_neighbors > 0)
166  normalizer /= TFeat(num_inp_neighbors);
167  }
168  for (int ic = 0; ic < in_channels; ++ic)
169  infeat(i, ic) *= normalizer;
170  }
171 
172  ++vec_valid_count;
173  if (vec_valid_count == VECSIZE ||
174  n + 1 == neighbor_end) {
175  ComputeFilterCoordinates<ALIGN_CORNERS, MAPPING>(
176  x, y, z, filter_size_xyz, inv_extents,
177  offsets_);
178  interpolation.Interpolate(
179  interp_weights, interp_indices, x, y, z,
180  filter_size_xyz, in_channels);
181  for (int k = 0; k < vec_valid_count; ++k) {
182  for (int j = 0; j < InterpolationVec_t::Size();
183  ++j) {
184  for (int ic = 0; ic < in_channels; ++ic)
185  B(interp_indices(j, k) + ic, out_col) +=
186  TFeat(interp_weights(j, k)) *
187  infeat(k, ic);
188  }
189  }
190  vec_valid_count = 0;
191  }
192  }
193 
194  } // out_idx
195 
196  if (out_importance) {
197  for (size_t out_idx = r.begin(); out_idx != r.end();
198  ++out_idx) {
199  const int out_col = out_idx - r.begin();
200  C.col(out_col) *= out_importance[out_idx];
201  }
202  }
203 
204  Eigen::Matrix<TOut, Eigen::Dynamic, Eigen::Dynamic> A(
205  out_channels, spatial_filter_size * in_channels);
206 
207  A = (C * B.transpose()).template cast<TOut>();
208 
209  {
210  std::lock_guard<std::mutex> lock(filter_backprop_mutex);
211  int linear_i = 0;
212  for (int j = 0; j < spatial_filter_size * in_channels; ++j)
213  for (int i = 0; i < out_channels; ++i, ++linear_i) {
214  filter_backprop[linear_i] += A(i, j);
215  }
216  }
217  });
218 }
219 
302 template <class TFeat, class TOut, class TReal, class TIndex>
303 void CConvTransposeBackpropFilterCPU(TOut* filter_backprop,
304  const std::vector<int>& filter_dims,
305  size_t num_out,
306  const TReal* out_positions,
307  const TFeat* out_importance,
308  size_t num_inp,
309  const TReal* inp_positions,
310  const TFeat* inp_features,
311  const TFeat* inp_neighbors_importance_sum,
312  const int64_t* inp_neighbors_row_splits,
313  size_t neighbors_index_size,
314  const TIndex* neighbors_index,
315  const TFeat* neighbors_importance,
316  const int64_t* neighbors_row_splits,
317  const TReal* extents,
318  const TReal* offsets,
319  const TFeat* out_features_gradient,
320  InterpolationMode interpolation,
321  CoordinateMapping coordinate_mapping,
322  bool align_corners,
323  bool individual_extent,
324  bool isotropic_extent,
325  bool normalize) {
326 #define FN_PARAMETERS \
327  filter_backprop, filter_dims, num_out, out_positions, out_importance, \
328  num_inp, inp_positions, inp_features, \
329  inp_neighbors_importance_sum, inp_neighbors_row_splits, \
330  neighbors_index_size, neighbors_index, neighbors_importance, \
331  neighbors_row_splits, extents, offsets, out_features_gradient
332 
333 #define CALL_TEMPLATE(INTERPOLATION, MAPPING, ALIGN_CORNERS, \
334  INDIVIDUAL_EXTENT, ISOTROPIC_EXTENT, NORMALIZE) \
335  if (INTERPOLATION == interpolation && MAPPING == coordinate_mapping && \
336  ALIGN_CORNERS == align_corners && \
337  INDIVIDUAL_EXTENT == individual_extent && \
338  ISOTROPIC_EXTENT == isotropic_extent && NORMALIZE == normalize) \
339  _CConvTransposeBackpropFilterCPU<TFeat, TOut, TReal, TIndex, \
340  INTERPOLATION, MAPPING, \
341  ALIGN_CORNERS, INDIVIDUAL_EXTENT, \
342  ISOTROPIC_EXTENT, NORMALIZE>( \
343  FN_PARAMETERS);
344 
345 #define CALL_TEMPLATE2(INTERPOLATION, MAPPING) \
346  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, true, true) \
347  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, true, false) \
348  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, false, true) \
349  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, false, false) \
350  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, true, true) \
351  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, true, false) \
352  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, false, true) \
353  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, false, false) \
354  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, true, true) \
355  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, true, false) \
356  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, false, true) \
357  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, false, false) \
358  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, true, true) \
359  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, true, false) \
360  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, false, true) \
361  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, false, false)
362 
363 #define CALL_TEMPLATE3(INTERPOLATION) \
364  CALL_TEMPLATE2(INTERPOLATION, CoordinateMapping::BALL_TO_CUBE_RADIAL) \
365  CALL_TEMPLATE2(INTERPOLATION, \
366  CoordinateMapping::BALL_TO_CUBE_VOLUME_PRESERVING) \
367  CALL_TEMPLATE2(INTERPOLATION, CoordinateMapping::IDENTITY)
368 
369 #define CALL_TEMPLATE4 \
370  CALL_TEMPLATE3(InterpolationMode::LINEAR) \
371  CALL_TEMPLATE3(InterpolationMode::LINEAR_BORDER) \
372  CALL_TEMPLATE3(InterpolationMode::NEAREST_NEIGHBOR)
373 
374  CALL_TEMPLATE4
375 
376 #undef CALL_TEMPLATE
377 #undef CALL_TEMPLATE2
378 #undef CALL_TEMPLATE3
379 #undef CALL_TEMPLATE4
380 
381 #undef FN_PARAMETERS
382 }
383 
384 } // namespace impl
385 } // namespace ml
386 } // namespace open3d
CALL_TEMPLATE4
#define CALL_TEMPLATE4
VECSIZE
#define VECSIZE
B
Eigen::Matrix3d B
Definition: PointCloudPlanarPatchDetection.cpp:506
open3d::ml::impl::_CConvTransposeBackpropFilterCPU
void _CConvTransposeBackpropFilterCPU(TOut *filter_backprop, const std::vector< int > &filter_dims, size_t num_out, const TReal *out_positions, const TFeat *out_importance, size_t num_inp, const TReal *inp_positions, const TFeat *inp_features, const TFeat *inp_neighbors_importance_sum, const int64_t *inp_neighbors_row_splits, size_t neighbors_index_size, const TIndex *neighbors_index, const TFeat *neighbors_importance, const int64_t *neighbors_row_splits, const TReal *extents, const TReal *offsets, const TFeat *out_features_gradient)
Definition: ContinuousConvTransposeBackpropFilter.h:32
open3d::ml::impl::InterpolationMode
InterpolationMode
Definition: ContinuousConvTypes.h:18
open3d::ml::impl::CConvTransposeBackpropFilterCPU
void CConvTransposeBackpropFilterCPU(TOut *filter_backprop, const std::vector< int > &filter_dims, size_t num_out, const TReal *out_positions, const TFeat *out_importance, size_t num_inp, const TReal *inp_positions, const TFeat *inp_features, const TFeat *inp_neighbors_importance_sum, const int64_t *inp_neighbors_row_splits, size_t neighbors_index_size, const TIndex *neighbors_index, const TFeat *neighbors_importance, const int64_t *neighbors_row_splits, const TReal *extents, const TReal *offsets, const TFeat *out_features_gradient, InterpolationMode interpolation, CoordinateMapping coordinate_mapping, bool align_corners, bool individual_extent, bool isotropic_extent, bool normalize)
Definition: ContinuousConvTransposeBackpropFilter.h:303
open3d::ml::impl::CoordinateMapping
CoordinateMapping
Definition: ContinuousConvTypes.h:26
open3d
Definition: PinholeCameraIntrinsic.cpp:16
open3d::ml::impl::InterpolationVec
Class for computing interpolation weights.
Definition: CoordinateTransformation.h:185
Generated by doxygen 1.9.1