video_input.cpp 16.8 KB
Newer Older
1
/*
2
 *  Copyright (C) 2004-2019 Savoir-faire Linux Inc.
3
 *
4
 *  Author: Tristan Matthews <tristan.matthews@savoirfairelinux.com>
5
 *  Author: Vivien Didelot <vivien.didelot@savoirfairelinux.com>
6 7 8 9 10 11 12 13 14 15 16 17 18
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
19
 *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
20 21
 */

22 23 24 25
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

26
#include "video_input.h"
27

28
#include "media_decoder.h"
29
#include "media_const.h"
30
#include "manager.h"
31
#include "client/videomanager.h"
32
#include "client/ring_signal.h"
33
#include "sinkclient.h"
Tristan Matthews's avatar
Tristan Matthews committed
34
#include "logger.h"
35 36 37
#include "media/media_buffer.h"

#include <libavformat/avio.h>
38

39
#include <string>
40
#include <sstream>
41
#include <cassert>
42
#ifdef _MSC_VER
43 44
#include <io.h> // for access
#else
45
#include <unistd.h>
46
#endif
Adrien Béraud's avatar
Adrien Béraud committed
47 48 49
extern "C" {
#include <libavutil/display.h>
}
50

Adrien Béraud's avatar
Adrien Béraud committed
51
namespace jami { namespace video {
52

53 54 55
static constexpr unsigned default_grab_width = 640;
static constexpr unsigned default_grab_height = 480;

56 57 58
VideoInput::VideoInput()
    : VideoGenerator::VideoGenerator()
    , sink_ {Manager::instance().createSinkClient("local")}
59 60 61
    , loop_(std::bind(&VideoInput::setup, this),
            std::bind(&VideoInput::process, this),
            std::bind(&VideoInput::cleanup, this))
62
#if defined(__ANDROID__) || defined(RING_UWP) || (defined(TARGET_OS_IOS) && TARGET_OS_IOS)
63
    , mutex_(), frame_cv_(), buffers_()
64
#endif
65
{}
66

67
VideoInput::~VideoInput()
68
{
69
#if defined(__ANDROID__) || defined(RING_UWP) || (defined(TARGET_OS_IOS) && TARGET_OS_IOS)
70 71 72 73 74
    /* we need to stop the loop and notify the condition variable
     * to unblock the process loop */
    loop_.stop();
    frame_cv_.notify_one();
#endif
75
    loop_.join();
76 77
}

78
#if defined(__ANDROID__) || defined(RING_UWP) || (defined(TARGET_OS_IOS) && TARGET_OS_IOS)
79 80 81 82 83 84 85 86 87 88 89
bool VideoInput::waitForBufferFull()
{
    for(auto& buffer : buffers_) {
        if (buffer.status == BUFFER_FULL)
            return true;
    }

    /* If the loop is stopped, returned true so we can quit the process loop */
    return !isCapturing();
}

90
void VideoInput::process()
91 92 93 94
{
    foundDecOpts(decOpts_);

    if (switchPending_.exchange(false)) {
Adrien Béraud's avatar
Adrien Béraud committed
95
        JAMI_DBG("Switching input to '%s'", decOpts_.input.c_str());
96 97 98 99 100 101 102 103 104 105 106 107
        if (decOpts_.input.empty()) {
            loop_.stop();
            return;
        }

        emitSignal<DRing::VideoSignal::StopCapture>();
        emitSignal<DRing::VideoSignal::StartCapture>(decOpts_.input);
    }

    std::unique_lock<std::mutex> lck(mutex_);

    frame_cv_.wait(lck, [this] { return waitForBufferFull(); });
108 109 110 111 112 113 114 115 116
    std::weak_ptr<VideoInput> wthis;
    // shared_from_this throws in destructor
    // assumes C++17
    try {
        wthis = shared_from_this();
    } catch (...) {
        return;
    }

Adrien Béraud's avatar
Adrien Béraud committed
117 118 119 120 121
    if (decOpts_.orientation != rotation_) {
        setRotation(decOpts_.orientation);
        rotation_ = decOpts_.orientation;
    }

122 123 124
    for (auto& buffer : buffers_) {
        if (buffer.status == BUFFER_FULL && buffer.index == publish_index_) {
            auto& frame = getNewFrame();
125
            AVPixelFormat format = getPixelFormat();
126

127 128
            if (auto displayMatrix = displayMatrix_)
                av_frame_new_side_data_from_buf(frame.pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(displayMatrix.get()));
Adrien Béraud's avatar
Adrien Béraud committed
129

130 131
            buffer.status = BUFFER_PUBLISHED;
            frame.setFromMemory((uint8_t*)buffer.data, format, decOpts_.width, decOpts_.height,
132 133 134 135 136 137
                                [wthis](uint8_t* ptr) {
                                    if (auto sthis = wthis.lock())
                                        sthis->releaseBufferCb(ptr);
                                    else
                                        std::free(ptr);
                                });
138 139 140 141 142 143 144 145
            publish_index_++;
            lck.unlock();
            publishFrame();
            break;
        }
    }
}

Adrien Béraud's avatar
Adrien Béraud committed
146 147 148
void
VideoInput::setRotation(int angle)
{
149 150 151 152 153 154 155 156
    std::shared_ptr<AVBufferRef> displayMatrix {
        av_buffer_alloc(sizeof(int32_t) * 9),
        [](AVBufferRef* buf){ av_buffer_unref(&buf); }
    };
    if (displayMatrix) {
        av_display_rotation_set(reinterpret_cast<int32_t*>(displayMatrix->data), angle);
        displayMatrix_ = std::move(displayMatrix);
    }
Adrien Béraud's avatar
Adrien Béraud committed
157 158
}

159
void VideoInput::cleanup()
160 161 162 163 164 165 166 167 168 169 170 171
{
    emitSignal<DRing::VideoSignal::StopCapture>();

    if (detach(sink_.get()))
        sink_->stop();

    std::lock_guard<std::mutex> lck(mutex_);
    for (auto& buffer : buffers_) {
        if (buffer.status == BUFFER_AVAILABLE ||
            buffer.status == BUFFER_FULL) {
            freeOneBuffer(buffer);
        } else if (buffer.status != BUFFER_NOT_ALLOCATED) {
Adrien Béraud's avatar
Adrien Béraud committed
172
            JAMI_ERR("Failed to free buffer [%p]", buffer.data);
173 174
        }
    }
Adrien Béraud's avatar
Adrien Béraud committed
175 176

    setRotation(0);
177
}
178
#else
179

180 181
void
VideoInput::process()
182
{
183
    if (switchPending_)
184 185
        createDecoder();

186
    if (not captureFrame()) {
187 188 189
        loop_.stop();
        return;
    }
190 191
}

192 193
void
VideoInput::cleanup()
194
{
195 196 197
    deleteDecoder(); // do it first to let a chance to last frame to be displayed
    detach(sink_.get());
    sink_->stop();
Adrien Béraud's avatar
Adrien Béraud committed
198
    JAMI_DBG("VideoInput closed");
199 200
}

201 202 203 204
#endif
bool VideoInput::setup()
{
    if (not attach(sink_.get())) {
Adrien Béraud's avatar
Adrien Béraud committed
205
        JAMI_ERR("attach sink failed");
206 207 208 209
        return false;
    }

    if (!sink_->start())
Adrien Béraud's avatar
Adrien Béraud committed
210
        JAMI_ERR("start sink failed");
211

Adrien Béraud's avatar
Adrien Béraud committed
212
    JAMI_DBG("VideoInput ready to capture");
213 214 215
    return true;
}

Guillaume Roguez's avatar
Guillaume Roguez committed
216 217 218 219 220 221
void VideoInput::clearOptions()
{
    decOpts_ = {};
    emulateRate_ = false;
}

222 223
bool
VideoInput::isCapturing() const noexcept
224
{
225
    return loop_.isRunning();
226 227 228 229
}

bool VideoInput::captureFrame()
{
230 231 232 233 234
    // Return true if capture could continue, false if must be stop

    if (not decoder_)
        return false;

235 236
    auto& frame = getNewFrame();
    const auto ret = decoder_->decode(frame);
237
    switch (ret) {
238
        case MediaDecoder::Status::ReadError:
239
            return false;
240

241 242 243 244
        // try to keep decoding
        case MediaDecoder::Status::DecodeError:
            return true;

245 246 247
        case MediaDecoder::Status::RestartRequired:
            createDecoder();
#ifdef RING_ACCEL
Adrien Béraud's avatar
Adrien Béraud committed
248
            JAMI_WARN("Disabling hardware decoding due to previous failure");
249 250 251 252
            decoder_->enableAccel(false);
#endif
            return static_cast<bool>(decoder_);

253
        // End of streamed file
254
        case MediaDecoder::Status::EOFError:
255
            createDecoder();
256
            return static_cast<bool>(decoder_);
257

258 259
        case MediaDecoder::Status::FrameFinished:
            publishFrame();
260
            return true;
261 262 263 264 265
        // continue decoding
        case MediaDecoder::Status::Success:
        default:
            return true;
    }
266 267
}

268
#if defined(__ANDROID__) || defined(RING_UWP) || (defined(TARGET_OS_IOS) && TARGET_OS_IOS)
269 270 271 272 273 274
int VideoInput::allocateOneBuffer(struct VideoFrameBuffer& b, int length)
{
    b.data = std::malloc(length);
    if (b.data) {
        b.status = BUFFER_AVAILABLE;
        b.length = length;
Adrien Béraud's avatar
Adrien Béraud committed
275
        JAMI_DBG("Allocated buffer [%p]", b.data);
276 277 278
        return 0;
    }

Adrien Béraud's avatar
Adrien Béraud committed
279
    JAMI_DBG("Failed to allocate memory for one buffer");
280 281 282 283 284
    return -ENOMEM;
}

void VideoInput::freeOneBuffer(struct VideoFrameBuffer& b)
{
Adrien Béraud's avatar
Adrien Béraud committed
285
    JAMI_DBG("Free buffer [%p]", b.data);
286 287 288 289 290 291
    std::free(b.data);
    b.data = nullptr;
    b.length = 0;
    b.status = BUFFER_NOT_ALLOCATED;
}

292
void VideoInput::releaseBufferCb(uint8_t* ptr)
293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321
{
    std::lock_guard<std::mutex> lck(mutex_);

    for(auto &buffer : buffers_) {
        if (buffer.data == ptr) {
            buffer.status = BUFFER_AVAILABLE;
            if (!isCapturing())
                freeOneBuffer(buffer);
            break;
        }
    }
}

void*
VideoInput::obtainFrame(int length)
{
    std::lock_guard<std::mutex> lck(mutex_);

    /* allocate buffers. This is done there because it's only when the Android
     * application requests a buffer that we know its size
     */
    for(auto& buffer : buffers_) {
        if (buffer.status == BUFFER_NOT_ALLOCATED) {
            allocateOneBuffer(buffer, length);
        }
    }

    /* search for an available frame */
    for(auto& buffer : buffers_) {
Philippe Gorley's avatar
Philippe Gorley committed
322
        if (buffer.length == static_cast<size_t>(length) && buffer.status == BUFFER_AVAILABLE) {
323 324 325 326 327
            buffer.status = BUFFER_CAPTURING;
            return buffer.data;
        }
    }

Adrien Béraud's avatar
Adrien Béraud committed
328
    JAMI_WARN("No buffer found");
329 330 331 332 333 334 335 336 337 338
    return nullptr;
}

void
VideoInput::releaseFrame(void *ptr)
{
    std::lock_guard<std::mutex> lck(mutex_);
    for(auto& buffer : buffers_) {
        if (buffer.data  == ptr) {
            if (buffer.status != BUFFER_CAPTURING)
Adrien Béraud's avatar
Adrien Béraud committed
339
                JAMI_ERR("Released a buffer with status %d, expected %d",
340 341 342 343 344 345 346 347 348 349 350 351 352 353
                         buffer.status, BUFFER_CAPTURING);
            if (isCapturing()) {
                buffer.status = BUFFER_FULL;
                buffer.index = capture_index_++;
                frame_cv_.notify_one();
            } else {
                freeOneBuffer(buffer);
            }
            break;
        }
    }
}
#endif

354 355 356
void
VideoInput::createDecoder()
{
357 358 359 360
    deleteDecoder();

    switchPending_ = false;

361
    if (decOpts_.input.empty()) {
362
        foundDecOpts(decOpts_);
363
        return;
364
    }
365

366
    auto decoder = std::unique_ptr<MediaDecoder>(new MediaDecoder());
367

368
    if (emulateRate_)
369
        decoder->emulateRate();
370

371 372 373
    decoder->setInterruptCallback(
        [](void* data) -> int { return not static_cast<VideoInput*>(data)->isCapturing(); },
        this);
374

375
    if (decoder->openInput(decOpts_) < 0) {
Adrien Béraud's avatar
Adrien Béraud committed
376
        JAMI_ERR("Could not open input \"%s\"", decOpts_.input.c_str());
377
        foundDecOpts(decOpts_);
378 379
        return;
    }
380 381

    /* Data available, finish the decoding */
382
    if (decoder->setupFromVideoData() < 0) {
Adrien Béraud's avatar
Adrien Béraud committed
383
        JAMI_ERR("decoder IO startup failed");
384
        foundDecOpts(decOpts_);
385 386
        return;
    }
387 388 389 390 391

    decOpts_.width = decoder->getWidth();
    decOpts_.height = decoder->getHeight();
    decOpts_.framerate = decoder->getFps();

Adrien Béraud's avatar
Adrien Béraud committed
392
    JAMI_DBG("created decoder with video params : size=%dX%d, fps=%lf",
393 394 395
             decOpts_.width, decOpts_.height, decOpts_.framerate.real());

    decoder_ = std::move(decoder);
396
    foundDecOpts(decOpts_);
397 398 399

    /* Signal the client about readable sink */
    sink_->setFrameSize(decoder_->getWidth(), decoder_->getHeight());
400 401
}

402 403
void
VideoInput::deleteDecoder()
404
{
405 406
    if (not decoder_)
        return;
407
    flushFrames();
408
    decoder_.reset();
409 410
}

411 412
bool
VideoInput::initCamera(const std::string& device)
413
{
Adrien Béraud's avatar
Adrien Béraud committed
414
    decOpts_ = jami::getVideoDeviceMonitor().getDeviceParams(device);
415
    return true;
416 417
}

418 419 420 421 422 423
static constexpr unsigned
round2pow(unsigned i, unsigned n)
{
    return (i >> n) << n;
}

424 425
bool
VideoInput::initX11(std::string display)
426
{
427
    size_t space = display.find(' ');
428

429
    clearOptions();
Guillaume Roguez's avatar
Guillaume Roguez committed
430 431
    decOpts_.format = "x11grab";
    decOpts_.framerate = 25;
432

433
    if (space != std::string::npos) {
434 435
        std::istringstream iss(display.substr(space + 1));
        char sep;
436 437 438 439 440
        unsigned w, h;
        iss >> w >> sep >> h;
        // round to 8 pixel block
        decOpts_.width = round2pow(w, 3);
        decOpts_.height = round2pow(h, 3);
Guillaume Roguez's avatar
Guillaume Roguez committed
441
        decOpts_.input = display.erase(space);
442
    } else {
Guillaume Roguez's avatar
Guillaume Roguez committed
443
        decOpts_.input = display;
444
        //decOpts_.video_size = "vga";
445 446 447 448 449 450 451
        decOpts_.width = default_grab_width;
        decOpts_.height = default_grab_height;
    }

    return true;
}

452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476
bool
VideoInput::initAVFoundation(const std::string& display)
{
    size_t space = display.find(' ');

    clearOptions();
    decOpts_.format = "avfoundation";
    decOpts_.input = "Capture screen 0";
    decOpts_.framerate = 30;

    if (space != std::string::npos) {
        std::istringstream iss(display.substr(space + 1));
        char sep;
        unsigned w, h;
        iss >> w >> sep >> h;
        decOpts_.width = round2pow(w, 3);
        decOpts_.height = round2pow(h, 3);
    } else {
        decOpts_.width = default_grab_width;
        decOpts_.height = default_grab_height;
    }
    return true;
}


477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492
bool
VideoInput::initGdiGrab(std::string params)
{
    size_t space = params.find(' ');
    clearOptions();
    decOpts_.format = "gdigrab";
    decOpts_.input = "desktop";
    decOpts_.framerate = 30;

    if (space != std::string::npos) {
        std::istringstream iss(params.substr(space + 1));
        char sep;
        unsigned w, h;
        iss >> w >> sep >> h;
        decOpts_.width = round2pow(w, 3);
        decOpts_.height = round2pow(h, 3);
493 494 495 496

        size_t plus = params.find('+');
        std::istringstream dss(params.substr(plus + 1, space - plus));
        dss >> decOpts_.offset_x >> sep >> decOpts_.offset_y;
497 498 499
    } else {
        decOpts_.width = default_grab_width;
        decOpts_.height = default_grab_height;
500 501 502 503 504 505 506 507 508 509 510 511 512
    }

    return true;
}

bool
VideoInput::initFile(std::string path)
{
    size_t dot = path.find_last_of('.');
    std::string ext = dot == std::string::npos ? "" : path.substr(dot + 1);

    /* File exists? */
    if (access(path.c_str(), R_OK) != 0) {
Adrien Béraud's avatar
Adrien Béraud committed
513
        JAMI_ERR("file '%s' unavailable\n", path.c_str());
514 515 516
        return false;
    }

517 518 519 520 521 522 523
    // check if file has video, fall back to default device if none
    // FIXME the way this is done is hackish, but it can't be done in createDecoder because that
    // would break the promise returned in switchInput
    DeviceParams p;
    p.input = path;
    auto dec = std::make_unique<MediaDecoder>();
    if (dec->openInput(p) < 0 || dec->setupFromVideoData() < 0) {
Adrien Béraud's avatar
Adrien Béraud committed
524
        return initCamera(jami::getVideoDeviceMonitor().getDefaultDevice());
525 526
    }

527 528
    clearOptions();
    emulateRate_ = true;
Guillaume Roguez's avatar
Guillaume Roguez committed
529 530
    decOpts_.input = path;
    decOpts_.loop = "1";
531

532 533
    // Force 1fps for static image
    if (ext == "jpeg" || ext == "jpg" || ext == "png") {
Guillaume Roguez's avatar
Guillaume Roguez committed
534 535
        decOpts_.format = "image2";
        decOpts_.framerate = 1;
536
    } else {
Adrien Béraud's avatar
Adrien Béraud committed
537
        JAMI_WARN("Guessing file type for %s", path.c_str());
538 539
    }

540
    return false;
541 542
}

543
std::shared_future<DeviceParams>
544 545
VideoInput::switchInput(const std::string& resource)
{
546 547 548
    if (resource == currentResource_)
        return futureDecOpts_;

Adrien Béraud's avatar
Adrien Béraud committed
549
    JAMI_DBG("MRL: '%s'", resource.c_str());
550

551
    if (switchPending_) {
Adrien Béraud's avatar
Adrien Béraud committed
552
        JAMI_ERR("Video switch already requested");
553
        return {};
554 555
    }

556
    currentResource_ = resource;
557
    decOptsFound_ = false;
558

559 560 561
    std::promise<DeviceParams> p;
    foundDecOpts_.swap(p);

562
    // Switch off video input?
563
    if (resource.empty()) {
564
        clearOptions();
565
        switchPending_ = true;
566 567
        if (!loop_.isRunning())
            loop_.start();
568 569
        futureDecOpts_   = foundDecOpts_.get_future();
        return futureDecOpts_;
570 571 572
    }

    // Supported MRL schemes
573
    static const std::string sep = DRing::Media::VideoProtocolPrefix::SEPARATOR;
574

575 576
    const auto pos = resource.find(sep);
    if (pos == std::string::npos)
577
        return {};
578 579 580

    const auto prefix = resource.substr(0, pos);
    if ((pos + sep.size()) >= resource.size())
581
        return {};
582

583
    const auto suffix = resource.substr(pos + sep.size());
584

585
    bool ready = false;
586

587
    if (prefix == DRing::Media::VideoProtocolPrefix::CAMERA) {
588
        /* Video4Linux2 */
589
        ready = initCamera(suffix);
590
    } else if (prefix == DRing::Media::VideoProtocolPrefix::DISPLAY) {
591
        /* X11 display name */
592 593 594
#ifdef __APPLE__
        ready = initAVFoundation(suffix);
#elif defined(_WIN32)
595
        ready = initGdiGrab(suffix);
596 597
#else
        ready = initX11(suffix);
598
#endif
599
    } else if (prefix == DRing::Media::VideoProtocolPrefix::FILE) {
600
        /* Pathname */
601
        ready = initFile(suffix);
602
    }
603

604 605
    if (ready) {
        foundDecOpts(decOpts_);
606
    }
607

608 609 610 611 612
    switchPending_ = true;
    if (!loop_.isRunning())
        loop_.start();
    futureDecOpts_ = foundDecOpts_.get_future().share();
    return futureDecOpts_;
613 614
}

615
#if defined(__ANDROID__) || defined(RING_UWP) || (defined(TARGET_OS_IOS) && TARGET_OS_IOS)
616 617 618 619 620 621
int VideoInput::getWidth() const
{ return decOpts_.width; }

int VideoInput::getHeight() const
{ return decOpts_.height; }

622
AVPixelFormat VideoInput::getPixelFormat() const
623 624 625 626 627 628
{
    int format;
    std::stringstream ss;
    ss << decOpts_.format;
    ss >> format;

629
    return (AVPixelFormat)format;
630 631
}
#else
632
int VideoInput::getWidth() const
633
{ return decoder_->getWidth(); }
634

635
int VideoInput::getHeight() const
636
{ return decoder_->getHeight(); }
637

638
AVPixelFormat VideoInput::getPixelFormat() const
639
{ return decoder_->getPixelFormat(); }
640
#endif
641

642 643 644
DeviceParams VideoInput::getParams() const
{ return decOpts_; }

645
MediaStream
646
VideoInput::getInfo() const
647 648 649 650
{
    return decoder_->getStream("v:local");
}

651 652 653 654 655 656 657 658 659
void
VideoInput::foundDecOpts(const DeviceParams& params)
{
    if (not decOptsFound_) {
        decOptsFound_ = true;
        foundDecOpts_.set_value(params);
    }
}

Adrien Béraud's avatar
Adrien Béraud committed
660
}} // namespace jami::video