video_input.cpp 16.3 KB
Newer Older
1
/*
2
 *  Copyright (C) 2004-2019 Savoir-faire Linux Inc.
3
 *
4
 *  Author: Tristan Matthews <tristan.matthews@savoirfairelinux.com>
5
 *  Author: Vivien Didelot <vivien.didelot@savoirfairelinux.com>
6 7 8 9 10 11 12 13 14 15 16 17 18
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
19
 *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
20 21
 */

22 23 24 25
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

26
#include "video_input.h"
27

28
#include "media_decoder.h"
29
#include "media_const.h"
30
#include "manager.h"
31
#include "client/videomanager.h"
32
#include "client/ring_signal.h"
33
#include "sinkclient.h"
Tristan Matthews's avatar
Tristan Matthews committed
34
#include "logger.h"
35 36 37
#include "media/media_buffer.h"

#include <libavformat/avio.h>
38

39
#include <string>
40
#include <sstream>
41
#include <cassert>
42
#ifdef _MSC_VER
43 44
#include <io.h> // for access
#else
45
#include <unistd.h>
46
#endif
Adrien Béraud's avatar
Adrien Béraud committed
47 48 49
extern "C" {
#include <libavutil/display.h>
}
50

Adrien Béraud's avatar
Adrien Béraud committed
51
namespace jami { namespace video {
52

53 54 55
static constexpr unsigned default_grab_width = 640;
static constexpr unsigned default_grab_height = 480;

56 57 58
VideoInput::VideoInput()
    : VideoGenerator::VideoGenerator()
    , sink_ {Manager::instance().createSinkClient("local")}
59 60 61
    , loop_(std::bind(&VideoInput::setup, this),
            std::bind(&VideoInput::process, this),
            std::bind(&VideoInput::cleanup, this))
62
#if defined(__ANDROID__) || defined(RING_UWP) || (defined(TARGET_OS_IOS) && TARGET_OS_IOS)
63
    , mutex_(), frame_cv_(), buffers_()
64
#endif
65
{}
66

67
VideoInput::~VideoInput()
68
{
69
#if defined(__ANDROID__) || defined(RING_UWP) || (defined(TARGET_OS_IOS) && TARGET_OS_IOS)
70 71 72 73 74
    /* we need to stop the loop and notify the condition variable
     * to unblock the process loop */
    loop_.stop();
    frame_cv_.notify_one();
#endif
75
    loop_.join();
76 77
}

78
#if defined(__ANDROID__) || defined(RING_UWP) || (defined(TARGET_OS_IOS) && TARGET_OS_IOS)
79 80 81 82 83 84 85 86 87 88 89
bool VideoInput::waitForBufferFull()
{
    for(auto& buffer : buffers_) {
        if (buffer.status == BUFFER_FULL)
            return true;
    }

    /* If the loop is stopped, returned true so we can quit the process loop */
    return !isCapturing();
}

90
void VideoInput::process()
91 92 93 94
{
    foundDecOpts(decOpts_);

    if (switchPending_.exchange(false)) {
Adrien Béraud's avatar
Adrien Béraud committed
95
        JAMI_DBG("Switching input to '%s'", decOpts_.input.c_str());
96 97 98 99 100 101 102 103 104 105 106 107
        if (decOpts_.input.empty()) {
            loop_.stop();
            return;
        }

        emitSignal<DRing::VideoSignal::StopCapture>();
        emitSignal<DRing::VideoSignal::StartCapture>(decOpts_.input);
    }

    std::unique_lock<std::mutex> lck(mutex_);

    frame_cv_.wait(lck, [this] { return waitForBufferFull(); });
108 109 110 111 112 113 114 115 116
    std::weak_ptr<VideoInput> wthis;
    // shared_from_this throws in destructor
    // assumes C++17
    try {
        wthis = shared_from_this();
    } catch (...) {
        return;
    }

Adrien Béraud's avatar
Adrien Béraud committed
117 118 119 120 121
    if (decOpts_.orientation != rotation_) {
        setRotation(decOpts_.orientation);
        rotation_ = decOpts_.orientation;
    }

122 123 124
    for (auto& buffer : buffers_) {
        if (buffer.status == BUFFER_FULL && buffer.index == publish_index_) {
            auto& frame = getNewFrame();
125
            AVPixelFormat format = getPixelFormat();
126

127 128
            if (auto displayMatrix = displayMatrix_)
                av_frame_new_side_data_from_buf(frame.pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(displayMatrix.get()));
Adrien Béraud's avatar
Adrien Béraud committed
129

130 131
            buffer.status = BUFFER_PUBLISHED;
            frame.setFromMemory((uint8_t*)buffer.data, format, decOpts_.width, decOpts_.height,
132 133 134 135 136 137
                                [wthis](uint8_t* ptr) {
                                    if (auto sthis = wthis.lock())
                                        sthis->releaseBufferCb(ptr);
                                    else
                                        std::free(ptr);
                                });
138 139 140 141 142 143 144 145
            publish_index_++;
            lck.unlock();
            publishFrame();
            break;
        }
    }
}

Adrien Béraud's avatar
Adrien Béraud committed
146 147 148
void
VideoInput::setRotation(int angle)
{
149 150 151 152 153 154 155 156
    std::shared_ptr<AVBufferRef> displayMatrix {
        av_buffer_alloc(sizeof(int32_t) * 9),
        [](AVBufferRef* buf){ av_buffer_unref(&buf); }
    };
    if (displayMatrix) {
        av_display_rotation_set(reinterpret_cast<int32_t*>(displayMatrix->data), angle);
        displayMatrix_ = std::move(displayMatrix);
    }
Adrien Béraud's avatar
Adrien Béraud committed
157 158
}

159
void VideoInput::cleanup()
160 161 162 163 164 165 166 167 168 169 170 171
{
    emitSignal<DRing::VideoSignal::StopCapture>();

    if (detach(sink_.get()))
        sink_->stop();

    std::lock_guard<std::mutex> lck(mutex_);
    for (auto& buffer : buffers_) {
        if (buffer.status == BUFFER_AVAILABLE ||
            buffer.status == BUFFER_FULL) {
            freeOneBuffer(buffer);
        } else if (buffer.status != BUFFER_NOT_ALLOCATED) {
Adrien Béraud's avatar
Adrien Béraud committed
172
            JAMI_ERR("Failed to free buffer [%p]", buffer.data);
173 174
        }
    }
Adrien Béraud's avatar
Adrien Béraud committed
175 176

    setRotation(0);
177
}
178
#else
179

180 181
void
VideoInput::process()
182
{
183
    if (switchPending_)
184 185
        createDecoder();

186
    if (not captureFrame()) {
187 188 189
        loop_.stop();
        return;
    }
190 191
}

192 193
void
VideoInput::cleanup()
194
{
195 196 197
    deleteDecoder(); // do it first to let a chance to last frame to be displayed
    detach(sink_.get());
    sink_->stop();
Adrien Béraud's avatar
Adrien Béraud committed
198
    JAMI_DBG("VideoInput closed");
199 200
}

201 202 203 204
#endif
bool VideoInput::setup()
{
    if (not attach(sink_.get())) {
Adrien Béraud's avatar
Adrien Béraud committed
205
        JAMI_ERR("attach sink failed");
206 207 208 209
        return false;
    }

    if (!sink_->start())
Adrien Béraud's avatar
Adrien Béraud committed
210
        JAMI_ERR("start sink failed");
211

Adrien Béraud's avatar
Adrien Béraud committed
212
    JAMI_DBG("VideoInput ready to capture");
213 214 215
    return true;
}

Guillaume Roguez's avatar
Guillaume Roguez committed
216 217 218 219 220 221
void VideoInput::clearOptions()
{
    decOpts_ = {};
    emulateRate_ = false;
}

222 223
bool
VideoInput::isCapturing() const noexcept
224
{
225
    return loop_.isRunning();
226 227 228 229
}

bool VideoInput::captureFrame()
{
230 231 232 233
    // Return true if capture could continue, false if must be stop
    if (not decoder_)
        return false;

234 235 236 237 238 239 240 241 242
    switch (decoder_->decode()) {
    case MediaDemuxer::Status::EndOfFile:
        createDecoder();
        return static_cast<bool>(decoder_);
    case MediaDemuxer::Status::ReadError:
        JAMI_ERR() << "Failed to decode frame";
        return false;
    default:
        return true;
243
    }
244 245
}

246
#if defined(__ANDROID__) || defined(RING_UWP) || (defined(TARGET_OS_IOS) && TARGET_OS_IOS)
247 248 249 250 251 252
int VideoInput::allocateOneBuffer(struct VideoFrameBuffer& b, int length)
{
    b.data = std::malloc(length);
    if (b.data) {
        b.status = BUFFER_AVAILABLE;
        b.length = length;
Adrien Béraud's avatar
Adrien Béraud committed
253
        JAMI_DBG("Allocated buffer [%p]", b.data);
254 255 256
        return 0;
    }

Adrien Béraud's avatar
Adrien Béraud committed
257
    JAMI_DBG("Failed to allocate memory for one buffer");
258 259 260 261 262
    return -ENOMEM;
}

void VideoInput::freeOneBuffer(struct VideoFrameBuffer& b)
{
Adrien Béraud's avatar
Adrien Béraud committed
263
    JAMI_DBG("Free buffer [%p]", b.data);
264 265 266 267 268 269
    std::free(b.data);
    b.data = nullptr;
    b.length = 0;
    b.status = BUFFER_NOT_ALLOCATED;
}

270
void VideoInput::releaseBufferCb(uint8_t* ptr)
271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299
{
    std::lock_guard<std::mutex> lck(mutex_);

    for(auto &buffer : buffers_) {
        if (buffer.data == ptr) {
            buffer.status = BUFFER_AVAILABLE;
            if (!isCapturing())
                freeOneBuffer(buffer);
            break;
        }
    }
}

void*
VideoInput::obtainFrame(int length)
{
    std::lock_guard<std::mutex> lck(mutex_);

    /* allocate buffers. This is done there because it's only when the Android
     * application requests a buffer that we know its size
     */
    for(auto& buffer : buffers_) {
        if (buffer.status == BUFFER_NOT_ALLOCATED) {
            allocateOneBuffer(buffer, length);
        }
    }

    /* search for an available frame */
    for(auto& buffer : buffers_) {
Philippe Gorley's avatar
Philippe Gorley committed
300
        if (buffer.length == static_cast<size_t>(length) && buffer.status == BUFFER_AVAILABLE) {
301 302 303 304 305
            buffer.status = BUFFER_CAPTURING;
            return buffer.data;
        }
    }

Adrien Béraud's avatar
Adrien Béraud committed
306
    JAMI_WARN("No buffer found");
307 308 309 310 311 312 313 314 315 316
    return nullptr;
}

void
VideoInput::releaseFrame(void *ptr)
{
    std::lock_guard<std::mutex> lck(mutex_);
    for(auto& buffer : buffers_) {
        if (buffer.data  == ptr) {
            if (buffer.status != BUFFER_CAPTURING)
Adrien Béraud's avatar
Adrien Béraud committed
317
                JAMI_ERR("Released a buffer with status %d, expected %d",
318 319 320 321 322 323 324 325 326 327 328 329 330 331
                         buffer.status, BUFFER_CAPTURING);
            if (isCapturing()) {
                buffer.status = BUFFER_FULL;
                buffer.index = capture_index_++;
                frame_cv_.notify_one();
            } else {
                freeOneBuffer(buffer);
            }
            break;
        }
    }
}
#endif

332 333 334
void
VideoInput::createDecoder()
{
335 336 337 338
    deleteDecoder();

    switchPending_ = false;

339
    if (decOpts_.input.empty()) {
340
        foundDecOpts(decOpts_);
341
        return;
342
    }
343

344 345 346
    auto decoder = std::make_unique<MediaDecoder>([this](const std::shared_ptr<MediaFrame>& frame) mutable {
        publishFrame(std::static_pointer_cast<VideoFrame>(frame));
    });
347

348
    if (emulateRate_)
349
        decoder->emulateRate();
350

351 352 353
    decoder->setInterruptCallback(
        [](void* data) -> int { return not static_cast<VideoInput*>(data)->isCapturing(); },
        this);
354

355
    if (decoder->openInput(decOpts_) < 0) {
Adrien Béraud's avatar
Adrien Béraud committed
356
        JAMI_ERR("Could not open input \"%s\"", decOpts_.input.c_str());
357
        foundDecOpts(decOpts_);
358 359
        return;
    }
360 361

    /* Data available, finish the decoding */
362
    if (decoder->setupVideo() < 0) {
Adrien Béraud's avatar
Adrien Béraud committed
363
        JAMI_ERR("decoder IO startup failed");
364
        foundDecOpts(decOpts_);
365 366
        return;
    }
367 368 369 370 371

    decOpts_.width = decoder->getWidth();
    decOpts_.height = decoder->getHeight();
    decOpts_.framerate = decoder->getFps();

Adrien Béraud's avatar
Adrien Béraud committed
372
    JAMI_DBG("created decoder with video params : size=%dX%d, fps=%lf",
373 374 375
             decOpts_.width, decOpts_.height, decOpts_.framerate.real());

    decoder_ = std::move(decoder);
376
    foundDecOpts(decOpts_);
377 378 379

    /* Signal the client about readable sink */
    sink_->setFrameSize(decoder_->getWidth(), decoder_->getHeight());
380 381
}

382 383
void
VideoInput::deleteDecoder()
384
{
385 386
    if (not decoder_)
        return;
387
    flushFrames();
388
    decoder_.reset();
389 390
}

391 392
bool
VideoInput::initCamera(const std::string& device)
393
{
Adrien Béraud's avatar
Adrien Béraud committed
394
    decOpts_ = jami::getVideoDeviceMonitor().getDeviceParams(device);
395
    return true;
396 397
}

398 399 400 401 402 403
static constexpr unsigned
round2pow(unsigned i, unsigned n)
{
    return (i >> n) << n;
}

404 405
bool
VideoInput::initX11(std::string display)
406
{
407
    size_t space = display.find(' ');
408

409
    clearOptions();
Guillaume Roguez's avatar
Guillaume Roguez committed
410 411
    decOpts_.format = "x11grab";
    decOpts_.framerate = 25;
412

413
    if (space != std::string::npos) {
414 415
        std::istringstream iss(display.substr(space + 1));
        char sep;
416 417 418 419 420
        unsigned w, h;
        iss >> w >> sep >> h;
        // round to 8 pixel block
        decOpts_.width = round2pow(w, 3);
        decOpts_.height = round2pow(h, 3);
Guillaume Roguez's avatar
Guillaume Roguez committed
421
        decOpts_.input = display.erase(space);
422
    } else {
Guillaume Roguez's avatar
Guillaume Roguez committed
423
        decOpts_.input = display;
424
        //decOpts_.video_size = "vga";
425 426 427 428 429 430 431
        decOpts_.width = default_grab_width;
        decOpts_.height = default_grab_height;
    }

    return true;
}

432 433 434 435 436 437 438
bool
VideoInput::initAVFoundation(const std::string& display)
{
    size_t space = display.find(' ');

    clearOptions();
    decOpts_.format = "avfoundation";
439
    decOpts_.pixel_format = "nv12";
440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457
    decOpts_.input = "Capture screen 0";
    decOpts_.framerate = 30;

    if (space != std::string::npos) {
        std::istringstream iss(display.substr(space + 1));
        char sep;
        unsigned w, h;
        iss >> w >> sep >> h;
        decOpts_.width = round2pow(w, 3);
        decOpts_.height = round2pow(h, 3);
    } else {
        decOpts_.width = default_grab_width;
        decOpts_.height = default_grab_height;
    }
    return true;
}


458
bool
459
VideoInput::initGdiGrab(const std::string& params)
460 461 462 463 464 465 466 467 468 469 470 471 472 473
{
    size_t space = params.find(' ');
    clearOptions();
    decOpts_.format = "gdigrab";
    decOpts_.input = "desktop";
    decOpts_.framerate = 30;

    if (space != std::string::npos) {
        std::istringstream iss(params.substr(space + 1));
        char sep;
        unsigned w, h;
        iss >> w >> sep >> h;
        decOpts_.width = round2pow(w, 3);
        decOpts_.height = round2pow(h, 3);
474 475 476 477

        size_t plus = params.find('+');
        std::istringstream dss(params.substr(plus + 1, space - plus));
        dss >> decOpts_.offset_x >> sep >> decOpts_.offset_y;
478 479 480
    } else {
        decOpts_.width = default_grab_width;
        decOpts_.height = default_grab_height;
481 482 483 484 485 486 487 488 489 490 491 492 493
    }

    return true;
}

bool
VideoInput::initFile(std::string path)
{
    size_t dot = path.find_last_of('.');
    std::string ext = dot == std::string::npos ? "" : path.substr(dot + 1);

    /* File exists? */
    if (access(path.c_str(), R_OK) != 0) {
Adrien Béraud's avatar
Adrien Béraud committed
494
        JAMI_ERR("file '%s' unavailable\n", path.c_str());
495 496 497
        return false;
    }

498 499 500 501 502 503
    // check if file has video, fall back to default device if none
    // FIXME the way this is done is hackish, but it can't be done in createDecoder because that
    // would break the promise returned in switchInput
    DeviceParams p;
    p.input = path;
    auto dec = std::make_unique<MediaDecoder>();
504
    if (dec->openInput(p) < 0 || dec->setupVideo() < 0) {
Adrien Béraud's avatar
Adrien Béraud committed
505
        return initCamera(jami::getVideoDeviceMonitor().getDefaultDevice());
506 507
    }

508 509
    clearOptions();
    emulateRate_ = true;
Guillaume Roguez's avatar
Guillaume Roguez committed
510 511
    decOpts_.input = path;
    decOpts_.loop = "1";
512

513 514
    // Force 1fps for static image
    if (ext == "jpeg" || ext == "jpg" || ext == "png") {
Guillaume Roguez's avatar
Guillaume Roguez committed
515 516
        decOpts_.format = "image2";
        decOpts_.framerate = 1;
517
    } else {
Adrien Béraud's avatar
Adrien Béraud committed
518
        JAMI_WARN("Guessing file type for %s", path.c_str());
519 520
    }

521
    return false;
522 523
}

524
std::shared_future<DeviceParams>
525 526
VideoInput::switchInput(const std::string& resource)
{
527 528 529
    if (resource == currentResource_)
        return futureDecOpts_;

Adrien Béraud's avatar
Adrien Béraud committed
530
    JAMI_DBG("MRL: '%s'", resource.c_str());
531

532
    if (switchPending_) {
Adrien Béraud's avatar
Adrien Béraud committed
533
        JAMI_ERR("Video switch already requested");
534
        return {};
535 536
    }

537
    currentResource_ = resource;
538
    decOptsFound_ = false;
539

540 541 542
    std::promise<DeviceParams> p;
    foundDecOpts_.swap(p);

543
    // Switch off video input?
544
    if (resource.empty()) {
545
        clearOptions();
546
        switchPending_ = true;
547 548
        if (!loop_.isRunning())
            loop_.start();
549 550
        futureDecOpts_   = foundDecOpts_.get_future();
        return futureDecOpts_;
551 552 553
    }

    // Supported MRL schemes
554
    static const std::string sep = DRing::Media::VideoProtocolPrefix::SEPARATOR;
555

556 557
    const auto pos = resource.find(sep);
    if (pos == std::string::npos)
558
        return {};
559 560 561

    const auto prefix = resource.substr(0, pos);
    if ((pos + sep.size()) >= resource.size())
562
        return {};
563

564
    const auto suffix = resource.substr(pos + sep.size());
565

566
    bool ready = false;
567

568
    if (prefix == DRing::Media::VideoProtocolPrefix::CAMERA) {
569
        /* Video4Linux2 */
570
        ready = initCamera(suffix);
571
    } else if (prefix == DRing::Media::VideoProtocolPrefix::DISPLAY) {
572
        /* X11 display name */
573 574 575
#ifdef __APPLE__
        ready = initAVFoundation(suffix);
#elif defined(_WIN32)
576
        ready = initGdiGrab(suffix);
577 578
#else
        ready = initX11(suffix);
579
#endif
580
    } else if (prefix == DRing::Media::VideoProtocolPrefix::FILE) {
581
        /* Pathname */
582
        ready = initFile(suffix);
583
    }
584

585 586
    if (ready) {
        foundDecOpts(decOpts_);
587
    }
588

589 590 591 592 593
    switchPending_ = true;
    if (!loop_.isRunning())
        loop_.start();
    futureDecOpts_ = foundDecOpts_.get_future().share();
    return futureDecOpts_;
594 595
}

596
#if defined(__ANDROID__) || defined(RING_UWP) || (defined(TARGET_OS_IOS) && TARGET_OS_IOS)
597 598 599 600 601 602
int VideoInput::getWidth() const
{ return decOpts_.width; }

int VideoInput::getHeight() const
{ return decOpts_.height; }

603
AVPixelFormat VideoInput::getPixelFormat() const
604 605 606 607 608 609
{
    int format;
    std::stringstream ss;
    ss << decOpts_.format;
    ss >> format;

610
    return (AVPixelFormat)format;
611 612
}
#else
613
int VideoInput::getWidth() const
614
{ return decoder_->getWidth(); }
615

616
int VideoInput::getHeight() const
617
{ return decoder_->getHeight(); }
618

619
AVPixelFormat VideoInput::getPixelFormat() const
620
{ return decoder_->getPixelFormat(); }
621
#endif
622

623 624 625
DeviceParams VideoInput::getParams() const
{ return decOpts_; }

626
MediaStream
627
VideoInput::getInfo() const
628 629 630 631
{
    return decoder_->getStream("v:local");
}

632 633 634 635 636 637 638 639 640
void
VideoInput::foundDecOpts(const DeviceParams& params)
{
    if (not decOptsFound_) {
        decOptsFound_ = true;
        foundDecOpts_.set_value(params);
    }
}

Adrien Béraud's avatar
Adrien Béraud committed
641
}} // namespace jami::video