Open3D (C++ API)  0.18.0
VideoScaler.h
Go to the documentation of this file.
1 // ----------------------------------------------------------------------------
2 // - Open3D: www.open3d.org -
3 // ----------------------------------------------------------------------------
4 // Copyright (c) 2018-2023 www.open3d.org
5 // SPDX-License-Identifier: MIT
6 // ----------------------------------------------------------------------------
7 // ----------------------------------------------------------------------------
8 // Contains source code from
9 // https://github.com/mpromonet/webrtc-streamer
10 //
11 // This software is in the public domain, furnished "as is", without technical
12 // support, and with no warranty, express or implied, as to its usefulness for
13 // any purpose.
14 // ----------------------------------------------------------------------------
15 //
16 // This is a private header. It shall be hidden from Open3D's public API. Do not
17 // put this in Open3D.h.in.
18 
19 #pragma once
20 
21 #include <api/media_stream_interface.h>
22 #include <media/base/video_broadcaster.h>
23 
25 
26 namespace open3d {
27 namespace visualization {
28 namespace webrtc_server {
29 
30 class VideoScaler : public rtc::VideoSinkInterface<webrtc::VideoFrame>,
31  public rtc::VideoSourceInterface<webrtc::VideoFrame> {
32 public:
33  VideoScaler(rtc::scoped_refptr<BitmapTrackSourceInterface> video_source,
34  const std::map<std::string, std::string> &opts)
35  : video_source_(video_source),
36  width_(0),
37  height_(0),
38  rotation_(webrtc::kVideoRotation_0),
39  roi_x_(0),
40  roi_y_(0),
41  roi_width_(0),
42  roi_height_(0) {
43  if (opts.find("width") != opts.end()) {
44  width_ = std::stoi(opts.at("width"));
45  }
46  if (opts.find("height") != opts.end()) {
47  height_ = std::stoi(opts.at("height"));
48  }
49  if (opts.find("rotation") != opts.end()) {
50  int rotation = std::stoi(opts.at("rotation"));
51  switch (rotation) {
52  case 90:
53  rotation_ = webrtc::kVideoRotation_90;
54  break;
55  case 180:
56  rotation_ = webrtc::kVideoRotation_180;
57  break;
58  case 270:
59  rotation_ = webrtc::kVideoRotation_270;
60  break;
61  }
62  }
63  if (opts.find("roi_x") != opts.end()) {
64  roi_x_ = std::stoi(opts.at("roi_x"));
65  if (roi_x_ < 0) {
66  utility::LogWarning("Ignore roi_x={}, it must be >=0", roi_x_);
67  roi_x_ = 0;
68  }
69  }
70  if (opts.find("roi_y") != opts.end()) {
71  roi_y_ = std::stoi(opts.at("roi_y"));
72  if (roi_y_ < 0) {
73  utility::LogWarning("Ignore roi_y_={}, it must be >=0", roi_y_);
74  roi_y_ = 0;
75  }
76  }
77  if (opts.find("roi_width") != opts.end()) {
78  roi_width_ = std::stoi(opts.at("roi_width"));
79  if (roi_width_ <= 0) {
80  utility::LogWarning("Ignore roi_width={}, it must be >0",
81  roi_width_);
82  roi_width_ = 0;
83  }
84  }
85  if (opts.find("roi_height") != opts.end()) {
86  roi_height_ = std::stoi(opts.at("roi_height"));
87  if (roi_height_ <= 0) {
88  utility::LogWarning("Ignore roi_height={}, it must be >0",
89  roi_height_);
90  roi_height_ = 0;
91  }
92  }
93  }
94 
95  virtual ~VideoScaler() {}
96 
97  void OnFrame(const webrtc::VideoFrame &frame) override {
98  if (roi_x_ >= frame.width()) {
100  "The ROI position protrudes beyond the right edge of the "
101  "image. Ignore roi_x.");
102  roi_x_ = 0;
103  }
104  if (roi_y_ >= frame.height()) {
106  "The ROI position protrudes beyond the right edge of the "
107  "image. Ignore roi_y_.");
108  roi_y_ = 0;
109  }
110  if (roi_width_ != 0 && (roi_width_ + roi_x_) > frame.width()) {
112  "The ROI position protrudes beyond the right edge of the "
113  "image. Ignore roi_width_.");
114  roi_width_ = 0;
115  }
116  if (roi_height_ != 0 && (roi_height_ + roi_y_) > frame.height()) {
118  "The ROI position protrudes beyond the right edge of the "
119  "image. Ignore roi_height_.");
120  roi_height_ = 0;
121  }
122 
123  if (roi_width_ == 0) {
124  roi_width_ = frame.width() - roi_x_;
125  }
126  if (roi_height_ == 0) {
127  roi_height_ = frame.height() - roi_y_;
128  }
129 
130  // source image is croped but destination image size is not set
131  if ((roi_width_ != frame.width() || roi_height_ != frame.height()) &&
132  (height_ == 0 && width_ == 0)) {
133  height_ = roi_height_;
134  width_ = roi_width_;
135  }
136 
137  if ((height_ == 0) && (width_ == 0) &&
138  (rotation_ == webrtc::kVideoRotation_0)) {
139  broadcaster_.OnFrame(frame);
140  } else {
141  int height = height_;
142  int width = width_;
143  if ((height == 0) && (width == 0)) {
144  height = frame.height();
145  width = frame.width();
146  } else if (height == 0) {
147  height = (roi_height_ * width) / roi_width_;
148  } else if (width == 0) {
149  width = (roi_width_ * height) / roi_height_;
150  }
151  rtc::scoped_refptr<webrtc::I420Buffer> scaled_buffer =
152  webrtc::I420Buffer::Create(width, height);
153  if (roi_width_ != frame.width() || roi_height_ != frame.height()) {
154  scaled_buffer->CropAndScaleFrom(
155  *frame.video_frame_buffer()->ToI420(), roi_x_, roi_y_,
156  roi_width_, roi_height_);
157  } else {
158  scaled_buffer->ScaleFrom(*frame.video_frame_buffer()->ToI420());
159  }
160  webrtc::VideoFrame scaledFrame =
161  webrtc::VideoFrame(scaled_buffer, frame.timestamp(),
162  frame.render_time_ms(), rotation_);
163 
164  broadcaster_.OnFrame(scaledFrame);
165  }
166  }
167 
168  void AddOrUpdateSink(rtc::VideoSinkInterface<webrtc::VideoFrame> *sink,
169  const rtc::VideoSinkWants &wants) override {
170  video_source_->AddOrUpdateSink(this, wants);
171 
172  broadcaster_.AddOrUpdateSink(sink, wants);
173  }
174 
176  rtc::VideoSinkInterface<webrtc::VideoFrame> *sink) override {
177  video_source_->RemoveSink(this);
178 
179  broadcaster_.RemoveSink(sink);
180  }
181 
182  int width() { return roi_width_; }
183  int height() { return roi_height_; }
184 
185 private:
186  rtc::scoped_refptr<BitmapTrackSourceInterface> video_source_;
187  rtc::VideoBroadcaster broadcaster_;
188 
189  int width_;
190  int height_;
191  webrtc::VideoRotation rotation_;
192  int roi_x_;
193  int roi_y_;
194  int roi_width_;
195  int roi_height_;
196 };
197 
198 } // namespace webrtc_server
199 } // namespace visualization
200 } // namespace open3d
Rect frame
Definition: BitmapWindowSystem.cpp:30
#define LogWarning(...)
Definition: Logging.h:60
void RemoveSink(rtc::VideoSinkInterface< webrtc::VideoFrame > *sink) override
Definition: VideoScaler.h:175
void OnFrame(const webrtc::VideoFrame &frame) override
Definition: VideoScaler.h:97
virtual ~VideoScaler()
Definition: VideoScaler.h:95
int width()
Definition: VideoScaler.h:182
void AddOrUpdateSink(rtc::VideoSinkInterface< webrtc::VideoFrame > *sink, const rtc::VideoSinkWants &wants) override
Definition: VideoScaler.h:168
VideoScaler(rtc::scoped_refptr< BitmapTrackSourceInterface > video_source, const std::map< std::string, std::string > &opts)
Definition: VideoScaler.h:33
int height()
Definition: VideoScaler.h:183
Definition: PinholeCameraIntrinsic.cpp:16