video_input.cpp 13.6 KB
Newer Older
1
/*
2
 *  Copyright (C) 2004-2020 Savoir-faire Linux Inc.
3
 *
4
 *  Author: Tristan Matthews <tristan.matthews@savoirfairelinux.com>
5
 *  Author: Vivien Didelot <vivien.didelot@savoirfairelinux.com>
6
 *  Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
7 8 9 10 11 12 13 14 15 16 17 18 19
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
20
 *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
21 22
 */

23 24 25 26
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

27
#include "video_input.h"
28

29
#include "media_decoder.h"
30
#include "media_const.h"
31
#include "manager.h"
32
#include "client/videomanager.h"
33
#include "client/ring_signal.h"
34
#include "sinkclient.h"
Tristan Matthews's avatar
Tristan Matthews committed
35
#include "logger.h"
36 37 38
#include "media/media_buffer.h"

#include <libavformat/avio.h>
39

40
#include <string>
41
#include <sstream>
42
#include <cassert>
43
#ifdef _MSC_VER
44 45
#include <io.h> // for access
#else
46
#include <unistd.h>
47
#endif
Adrien Béraud's avatar
Adrien Béraud committed
48 49 50
extern "C" {
#include <libavutil/display.h>
}
51

Adrien Béraud's avatar
Adrien Béraud committed
52
namespace jami { namespace video {
53

54 55 56
static constexpr unsigned default_grab_width = 640;
static constexpr unsigned default_grab_height = 480;

57
VideoInput::VideoInput(VideoInputMode inputMode)
58
    : VideoGenerator::VideoGenerator()
59 60 61
    , loop_(std::bind(&VideoInput::setup, this),
            std::bind(&VideoInput::process, this),
            std::bind(&VideoInput::cleanup, this))
62 63 64 65 66 67 68
{
    inputMode_ = inputMode;
    if (inputMode_ == VideoInputMode::Undefined) {
#if (defined(__ANDROID__) || defined(RING_UWP) || (defined(TARGET_OS_IOS) && TARGET_OS_IOS))
        inputMode_ = VideoInputMode::ManagedByClient;
#else
        inputMode_ = VideoInputMode::ManagedByDaemon;
69
#endif
70 71 72 73 74
    }
    if (inputMode_ == VideoInputMode::ManagedByDaemon) {
        sink_ = Manager::instance().createSinkClient("local");
    }
}
75

76
VideoInput::~VideoInput()
77
{
78
    isStopped_ = true;
79 80 81 82 83
    if (videoManagedByClient()) {
        emitSignal<DRing::VideoSignal::StopCapture>();
        capturing_ = false;
        return;
    }
84
    loop_.join();
85 86
}

87 88
void
VideoInput::startLoop()
89
{
90 91 92 93
    if (videoManagedByClient()) {
        switchDevice();
        return;
    }
94 95
    if (!loop_.isRunning())
        loop_.start();
96 97
}

98 99
void
VideoInput::switchDevice()
100 101
{
    if (switchPending_.exchange(false)) {
Adrien Béraud's avatar
Adrien Béraud committed
102
        JAMI_DBG("Switching input to '%s'", decOpts_.input.c_str());
103
        if (decOpts_.input.empty()) {
104
            capturing_ = false;
105 106 107 108 109
            return;
        }

        emitSignal<DRing::VideoSignal::StopCapture>();
        emitSignal<DRing::VideoSignal::StartCapture>(decOpts_.input);
110
        capturing_ = true;
111
    }
112
}
113

114
int VideoInput::getWidth() const
115 116 117 118 119 120
{
    if (videoManagedByClient()) {
        return decOpts_.width;
    }
    return decoder_->getWidth();
}
121

122
int VideoInput::getHeight() const
123 124 125 126 127 128
{
    if (videoManagedByClient()) {
        return decOpts_.height;
    }
    return decoder_->getHeight();
}
Adrien Béraud's avatar
Adrien Béraud committed
129

130 131
AVPixelFormat VideoInput::getPixelFormat() const
{
132 133 134
    if (!videoManagedByClient()) {
        return decoder_->getPixelFormat();
    }
135 136 137 138 139
    int format;
    std::stringstream ss;
    ss << decOpts_.format;
    ss >> format;
    return (AVPixelFormat)format;
140 141
}

Adrien Béraud's avatar
Adrien Béraud committed
142 143 144
void
VideoInput::setRotation(int angle)
{
145 146 147 148 149 150 151 152
    std::shared_ptr<AVBufferRef> displayMatrix {
        av_buffer_alloc(sizeof(int32_t) * 9),
        [](AVBufferRef* buf){ av_buffer_unref(&buf); }
    };
    if (displayMatrix) {
        av_display_rotation_set(reinterpret_cast<int32_t*>(displayMatrix->data), angle);
        displayMatrix_ = std::move(displayMatrix);
    }
Adrien Béraud's avatar
Adrien Béraud committed
153 154
}

155
bool VideoInput::setup()
156
{
157 158 159 160
    if (not attach(sink_.get())) {
        JAMI_ERR("attach sink failed");
        return false;
    }
161

162 163
    if (!sink_->start())
        JAMI_ERR("start sink failed");
164

165
    JAMI_DBG("VideoInput ready to capture");
Adrien Béraud's avatar
Adrien Béraud committed
166

167
    return true;
168
}
169

170 171
void
VideoInput::process()
172
{
173
    if (switchPending_)
174 175
        createDecoder();

176
    if (not captureFrame()) {
177 178 179
        loop_.stop();
        return;
    }
180 181
}

182 183
void
VideoInput::cleanup()
184
{
185 186 187
    deleteDecoder(); // do it first to let a chance to last frame to be displayed
    detach(sink_.get());
    sink_->stop();
Adrien Béraud's avatar
Adrien Béraud committed
188
    JAMI_DBG("VideoInput closed");
189 190
}

191
bool
192
VideoInput::captureFrame()
193
{
194 195 196 197
    // Return true if capture could continue, false if must be stop
    if (not decoder_)
        return false;

198 199 200 201 202 203 204 205 206
    switch (decoder_->decode()) {
    case MediaDemuxer::Status::EndOfFile:
        createDecoder();
        return static_cast<bool>(decoder_);
    case MediaDemuxer::Status::ReadError:
        JAMI_ERR() << "Failed to decode frame";
        return false;
    default:
        return true;
207
    }
208 209 210 211 212
}

void
VideoInput::createDecoder()
{
213 214 215 216
    deleteDecoder();

    switchPending_ = false;

217
    if (decOpts_.input.empty()) {
218
        foundDecOpts(decOpts_);
219
        return;
220
    }
221

222 223 224
    auto decoder = std::make_unique<MediaDecoder>([this](const std::shared_ptr<MediaFrame>& frame) mutable {
        publishFrame(std::static_pointer_cast<VideoFrame>(frame));
    });
225

226
    if (emulateRate_)
227
        decoder->emulateRate();
228

229 230 231
    decoder->setInterruptCallback(
        [](void* data) -> int { return not static_cast<VideoInput*>(data)->isCapturing(); },
        this);
232

233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252
    bool ready = false, restartSink = false;
    while (!ready && !isStopped_) {
        // Retry to open the video till the input is opened
        auto ret = decoder->openInput(decOpts_);
        ready = ret >= 0;
        if (ret < 0 && -ret != EBUSY) {
            JAMI_ERR("Could not open input \"%s\" with status %i", decOpts_.input.c_str(), ret);
            foundDecOpts(decOpts_);
            return;
        } else if (-ret == EBUSY) {
            // If the device is busy, this means that it can be used by another call.
            // If this is the case, cleanup() can occurs and this will erase shmPath_
            // So, be sure to regenerate a correct shmPath for clients.
            restartSink = true;
        }
        std::this_thread::sleep_for(std::chrono::milliseconds(10));
    }

    if (restartSink && !isStopped_) {
        sink_->start();
253
    }
254 255

    /* Data available, finish the decoding */
256
    if (decoder->setupVideo() < 0) {
Adrien Béraud's avatar
Adrien Béraud committed
257
        JAMI_ERR("decoder IO startup failed");
258
        foundDecOpts(decOpts_);
259 260
        return;
    }
261

262 263
    decoder->decode(); // Populate AVCodecContext fields

264 265 266
    decOpts_.width = decoder->getWidth();
    decOpts_.height = decoder->getHeight();
    decOpts_.framerate = decoder->getFps();
267 268 269 270 271 272 273
    AVPixelFormat fmt = decoder->getPixelFormat();
    if (fmt != AV_PIX_FMT_NONE) {
        decOpts_.pixel_format = av_get_pix_fmt_name(fmt);
    } else {
        JAMI_WARN("Could not determine pixel format, using default");
        decOpts_.pixel_format = av_get_pix_fmt_name(AV_PIX_FMT_YUV420P);
    }
274

275 276 277
    JAMI_DBG("created decoder with video params : size=%dX%d, fps=%lf pix=%s",
             decOpts_.width, decOpts_.height, decOpts_.framerate.real(),
             decOpts_.pixel_format.c_str());
278 279

    decoder_ = std::move(decoder);
280
    foundDecOpts(decOpts_);
281 282 283

    /* Signal the client about readable sink */
    sink_->setFrameSize(decoder_->getWidth(), decoder_->getHeight());
284 285
}

286 287
void
VideoInput::deleteDecoder()
288
{
289 290
    if (not decoder_)
        return;
291
    flushFrames();
292
    decoder_.reset();
293 294
}

295 296 297 298 299 300 301 302 303
void VideoInput::clearOptions()
{
    decOpts_ = {};
    emulateRate_ = false;
}

bool
VideoInput::isCapturing() const noexcept
{
304 305 306
    if (videoManagedByClient()) {
        return capturing_;
    }
307 308 309
    return loop_.isRunning();
}

310 311
bool
VideoInput::initCamera(const std::string& device)
312
{
Adrien Béraud's avatar
Adrien Béraud committed
313
    decOpts_ = jami::getVideoDeviceMonitor().getDeviceParams(device);
314
    return true;
315 316
}

317 318 319 320 321 322
static constexpr unsigned
round2pow(unsigned i, unsigned n)
{
    return (i >> n) << n;
}

323 324
bool
VideoInput::initX11(std::string display)
325
{
326
    size_t space = display.find(' ');
327

328
    clearOptions();
Guillaume Roguez's avatar
Guillaume Roguez committed
329 330
    decOpts_.format = "x11grab";
    decOpts_.framerate = 25;
331

332
    if (space != std::string::npos) {
333 334
        std::istringstream iss(display.substr(space + 1));
        char sep;
335 336 337 338 339
        unsigned w, h;
        iss >> w >> sep >> h;
        // round to 8 pixel block
        decOpts_.width = round2pow(w, 3);
        decOpts_.height = round2pow(h, 3);
Guillaume Roguez's avatar
Guillaume Roguez committed
340
        decOpts_.input = display.erase(space);
341
    } else {
Guillaume Roguez's avatar
Guillaume Roguez committed
342
        decOpts_.input = display;
343
        //decOpts_.video_size = "vga";
344 345 346 347 348 349 350
        decOpts_.width = default_grab_width;
        decOpts_.height = default_grab_height;
    }

    return true;
}

351 352 353 354 355 356 357
bool
VideoInput::initAVFoundation(const std::string& display)
{
    size_t space = display.find(' ');

    clearOptions();
    decOpts_.format = "avfoundation";
358
    decOpts_.pixel_format = "nv12";
359
    decOpts_.name = "Capture screen 0";
360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376
    decOpts_.input = "Capture screen 0";
    decOpts_.framerate = 30;

    if (space != std::string::npos) {
        std::istringstream iss(display.substr(space + 1));
        char sep;
        unsigned w, h;
        iss >> w >> sep >> h;
        decOpts_.width = round2pow(w, 3);
        decOpts_.height = round2pow(h, 3);
    } else {
        decOpts_.width = default_grab_width;
        decOpts_.height = default_grab_height;
    }
    return true;
}

377
bool
378
VideoInput::initGdiGrab(const std::string& params)
379 380 381 382 383 384 385 386 387 388 389 390 391 392
{
    size_t space = params.find(' ');
    clearOptions();
    decOpts_.format = "gdigrab";
    decOpts_.input = "desktop";
    decOpts_.framerate = 30;

    if (space != std::string::npos) {
        std::istringstream iss(params.substr(space + 1));
        char sep;
        unsigned w, h;
        iss >> w >> sep >> h;
        decOpts_.width = round2pow(w, 3);
        decOpts_.height = round2pow(h, 3);
393 394 395 396

        size_t plus = params.find('+');
        std::istringstream dss(params.substr(plus + 1, space - plus));
        dss >> decOpts_.offset_x >> sep >> decOpts_.offset_y;
397 398 399
    } else {
        decOpts_.width = default_grab_width;
        decOpts_.height = default_grab_height;
400 401 402 403 404 405 406 407 408 409 410 411 412
    }

    return true;
}

bool
VideoInput::initFile(std::string path)
{
    size_t dot = path.find_last_of('.');
    std::string ext = dot == std::string::npos ? "" : path.substr(dot + 1);

    /* File exists? */
    if (access(path.c_str(), R_OK) != 0) {
Adrien Béraud's avatar
Adrien Béraud committed
413
        JAMI_ERR("file '%s' unavailable\n", path.c_str());
414 415 416
        return false;
    }

417 418 419 420 421
    // check if file has video, fall back to default device if none
    // FIXME the way this is done is hackish, but it can't be done in createDecoder because that
    // would break the promise returned in switchInput
    DeviceParams p;
    p.input = path;
422
    p.name = path;
423
    auto dec = std::make_unique<MediaDecoder>();
424
    if (dec->openInput(p) < 0 || dec->setupVideo() < 0) {
Adrien Béraud's avatar
Adrien Béraud committed
425
        return initCamera(jami::getVideoDeviceMonitor().getDefaultDevice());
426 427
    }

428 429
    clearOptions();
    emulateRate_ = true;
Guillaume Roguez's avatar
Guillaume Roguez committed
430
    decOpts_.input = path;
431
    decOpts_.name = path;
Guillaume Roguez's avatar
Guillaume Roguez committed
432
    decOpts_.loop = "1";
433

434 435
    // Force 1fps for static image
    if (ext == "jpeg" || ext == "jpg" || ext == "png") {
Guillaume Roguez's avatar
Guillaume Roguez committed
436 437
        decOpts_.format = "image2";
        decOpts_.framerate = 1;
438
    } else {
Adrien Béraud's avatar
Adrien Béraud committed
439
        JAMI_WARN("Guessing file type for %s", path.c_str());
440 441
    }

442
    return false;
443 444
}

445
std::shared_future<DeviceParams>
446 447
VideoInput::switchInput(const std::string& resource)
{
448 449 450
    if (resource == currentResource_)
        return futureDecOpts_;

Adrien Béraud's avatar
Adrien Béraud committed
451
    JAMI_DBG("MRL: '%s'", resource.c_str());
452

453
    if (switchPending_.exchange(true)) {
Adrien Béraud's avatar
Adrien Béraud committed
454
        JAMI_ERR("Video switch already requested");
455
        return {};
456 457
    }

458
    currentResource_ = resource;
459
    decOptsFound_ = false;
460

461 462 463
    std::promise<DeviceParams> p;
    foundDecOpts_.swap(p);

464
    // Switch off video input?
465
    if (resource.empty()) {
466
        clearOptions();
467 468
        futureDecOpts_  = foundDecOpts_.get_future();
        startLoop();
469
        return futureDecOpts_;
470 471 472
    }

    // Supported MRL schemes
473
    static const std::string sep = DRing::Media::VideoProtocolPrefix::SEPARATOR;
474

475 476
    const auto pos = resource.find(sep);
    if (pos == std::string::npos)
477
        return {};
478 479 480

    const auto prefix = resource.substr(0, pos);
    if ((pos + sep.size()) >= resource.size())
481
        return {};
482

483
    const auto suffix = resource.substr(pos + sep.size());
484

485
    bool ready = false;
486

487
    if (prefix == DRing::Media::VideoProtocolPrefix::CAMERA) {
488
        /* Video4Linux2 */
489
        ready = initCamera(suffix);
490
    } else if (prefix == DRing::Media::VideoProtocolPrefix::DISPLAY) {
491
        /* X11 display name */
492 493 494
#ifdef __APPLE__
        ready = initAVFoundation(suffix);
#elif defined(_WIN32)
495
        ready = initGdiGrab(suffix);
496 497
#else
        ready = initX11(suffix);
498
#endif
499
    } else if (prefix == DRing::Media::VideoProtocolPrefix::FILE) {
500
        /* Pathname */
501
        ready = initFile(suffix);
502
    }
503

504 505
    if (ready) {
        foundDecOpts(decOpts_);
506 507
    }
    futureDecOpts_ = foundDecOpts_.get_future().share();
508
    startLoop();
509
    return futureDecOpts_;
510 511
}

512 513
const DeviceParams&
VideoInput::getParams() const
514 515
{ return decOpts_; }

516
MediaStream
517
VideoInput::getInfo() const
518
{
519 520 521 522
    if (!videoManagedByClient()) {
        if (decoder_)
            return decoder_->getStream("v:local");
    }
523 524 525 526
    auto opts = futureDecOpts_.get();
    rational<int> fr(opts.framerate.numerator(), opts.framerate.denominator());
    return MediaStream("v:local", av_get_pix_fmt(opts.pixel_format.c_str()),
        1 / fr, opts.width, opts.height, 0, fr);
527 528
}

529 530 531 532 533 534 535 536 537
void
VideoInput::foundDecOpts(const DeviceParams& params)
{
    if (not decOptsFound_) {
        decOptsFound_ = true;
        foundDecOpts_.set_value(params);
    }
}

Adrien Béraud's avatar
Adrien Béraud committed
538
}} // namespace jami::video